Merge pull request #212 from openziti/release-next

v0.16.5

CHANGELOG.md

@@ -1,16 +1,136 @@
+# Release 0.16.5
+
+## Breaking Changes
+
+## What's New
+  * fix [edge#338 configurable IP range for DNS services](https://github.com/openziti/edge/issues/338)
+    The IP range for hostname-configured services defaults to 100.64/10, and can be changed with the
+    `-d` command-line option.
+  * fix [edge#336 Startup Race Condition](https://github.com/openziti/edge/issues/336)
+  * fix api-session and session deletes from ziti CLI
+  * update ziti-fabric-test loop3
+  * allow specifying policy semantic from CLI when creating policies
+  * new eventing features
+  * Posture Check Configuration API
+
+## Event Changes
+### Event Configuration
+Handlers can now be configured for events via the config file. Here is an example configuration:
+
+    events:
+        jsonLogger:
+            subscriptions:
+                - type: metrics
+                  sourceFilter: .*
+                  metricFilter: .*egress.*m1_rate*
+                - type: fabric.sessions
+                  include:
+                    - created
+                - type: edge.sessions
+                  include:
+                    - created
+                - type: fabric.usage
+            handler:
+                type: file
+                format: json
+                path: /tmp/ziti-events.log
+
+Each section can include any number of event subscriptions and a single handler. The supported event types are:
+
+* metrics
+* fabric.sessions
+* fabric.usage
+* edge.sessions
+
+### Event Handlers
+There are two new handlers which can be used to output events.
+
+#### File Handler
+Sends events to disk. Supported options:
+
+* type: file
+* format: json|plain
+* path: Path to the target file
+* bufferSize: size of event queue. When this fills up, processes submitting events will block
+* maxsizemb: max size of the file before it gets rolled. default is 10MB
+* maxbackups: max number of rolling files to keep. default is 0 (keeps all)
+
+#### Stdout Handler
+Sends events to stdout. Supported options:
+
+* type: stdout
+* format: json|plain
+* bufferSize: size of event queue. When this fills up, processes submitting events will block
+
+### Usage Events
+There is a new Usage event type which has been derived from the metrics events. This allows handlers to see discrete usage events, rather than a collection of them.
+
+The Usage event looks like:
+
+    type UsageEvent struct {
+        Namespace        string `json:"namespace"`
+        EventType        string `json:"event_type"`
+        SourceId         string `json:"source_id"`
+        SessionId        string `json:"session_id"`
+        Usage            uint64 `json:"usage"`
+        IntervalStartUTC int64  `json:"interval_start_utc"`
+        IntervalLength   uint64 `json:"interval_length"`
+    }
+
+### Metrics Events
+Metrics events can now be filtered. Metrics events processed through the new event framework no longer sending metrics messages directly. Rather, a flattened (and more easily filterable) event type is provided. The new event type looks like:
+
+    type MetricsEvent struct {
+        Namespace    string
+        SourceId     string
+        Timestamp    *timestamp.Timestamp
+        Tags         map[string]string
+        IntMetrics   map[string]int64
+        FloatMetrics map[string]float64
+        MetricGroup  map[string]string
+    }
+
+## Posture Check Configuration API
+
+Edge API endpoints have been added to configure posture checks. Posture
+checks not currently enforced. However it is possible for integrations
+to being developed against the API.
+
+This section contains an overview of the new endpoints. See the OpenAPI
+2.0 API definition (swagger.yml) for complete details.
+
+### New Endpoints
+
+- `GET /posture-checks` - retrieve a list of existing Posture Checks
+- `POST /posture-checks` - create a new Posture Check
+- `PUT/PATCH /posture-checks/<id>` - update an existing Posture Check
+- `DELETE /posture-checks/<id>` - delete an existing Posture Check
+- `GET /posture-check-types` - retrieve a list of existing Posture Check
+  Types
+- `GET /posture-check-types/<id>` - retrieve a an existing Posture Check
+  Type
+- `GET /identities/<id>/posture-data` - retrieve the Posture Data for a
+  specific identity
+
+### Modified Endpoints
+
+- `GET/POST /service-policies` - now accepts/returns field
+  `postureCheckRoles` and `postureCheckRolesDisplay`
+- `PUT/PATCH /service-policies/<id>` - now accepts `postureCheckRoles`
+
 # Release 0.16.4
 
 ## Breaking CLI Change
-  * The `ziti edge enroll` and `ziti-tunnel enroll` subcommands no longer require a --jwt argument. Instead the JWT can be supplied as the first argument. So `ziti edge enroll --jwt /path/to/my.jwt` would become `ziti edge enroll /path/to/my.jwt`. For now the --jwt flag is still accepted as well. 
+  * The `ziti edge enroll` and `ziti-tunnel enroll` subcommands no longer require a --jwt argument. Instead the JWT can be supplied as the first argument. So `ziti edge enroll --jwt /path/to/my.jwt` would become `ziti edge enroll /path/to/my.jwt`. For now the --jwt flag is still accepted as well.
 
 ## Deprecations
-  * The `ziti-enroller` command is deprecated and will be removed in a future release. It will now display a DEPRECATION warning when it is run 
+  * The `ziti-enroller` command is deprecated and will be removed in a future release. It will now display a DEPRECATION warning when it is run
 
 ## What's New
   * [ziti#192 CAs default to 10yr expiration](https://github.com/openziti/ziti/issues/192)
   * Allow specifying edge config file when using ziti-fabric-test loop2
-  * Add grouping data to streaming metrics, so values can be associated with their source metric  
-  * New WSS underlay to support Edge Router connections from Browser-based webapps using the ziti-sdk-js  
+  * Add grouping data to streaming metrics, so values can be associated with their source metric
+  * New WSS underlay to support Edge Router connections from Browser-based webapps using the ziti-sdk-js
   * [ziti#151 enroll subcommand w/out args should print help](https://github.com/openziti/ziti/issues/192)
   * Fix processing of `--configTypes all` in `ziti edge list services`
   * Addressable Terminators and the eXtensible Terminator Validation framework
@@ -24,44 +144,44 @@ The following should be added to controller configuration files when using the E
     terminator:
       validators:
         edge: edge
-        
+
 This config stanza enables validating edge terminators. If this stanza is not added, everything will continue to work, but different Edge identities will be allowed to bind to the same terminator identity, which is generally not a valid state.
 
 # SDK API changes
 The `Context.ListenWithOptions` method now takes a `ListenOptions` which are defined in the `ziti` package, instead of the `edge` package.
 
-There's a new `Context.DialWithOptions` method which takes a `DialOptions` struct.  
+There's a new `Context.DialWithOptions` method which takes a `DialOptions` struct.
 
 ## Addressable Terminators and Terminator Validation
 This release contains two new features related to terminators. The first allows you to connect to a subset of terminators for a service. The second allows developers to plugin validation for terminators with different validation logic per binding type.
 
 ### Addressable Terminators
 Terminators define how network traffic for Ziti services exits the fabric and makes its way to the application providing/hosting the service. Each terminator for a service specifies the following:
- 
+
  1. The router at which traffic terminates
  2. The binding, which specifies the Xgress component responsible for providing the connection to the hosting application
- 3. The address, which the Xgress component can use to create or lookup the connection 
- 
+ 3. The address, which the Xgress component can use to create or lookup the connection
+
  There are currently two kinds of termination. Router terminated services make outbound connections to the hosting applications. SDK hosted servers make inbound connections from the SDK to the router.
-      
+
 Now that Ziti supports multiple terminators we may want to be able to connect to a specific hosting application. This can be used to allow a service to cover many endpoints, each of which can be connected to individually. Some common use cases for this might be a peer-to-peer application, like a voip service, or a service like SSH covering multiple machines.
 
-What we don't want in these cases is to have to create a new service for each voip client or each new machine that we want to ssh to. 
-We also want to make sure that if an application is making multiple connections (and thus multiple terminators) for redundancy or load balancing purposes, that we can address the application, rather than an individual terminator. 
+What we don't want in these cases is to have to create a new service for each voip client or each new machine that we want to ssh to.
+We also want to make sure that if an application is making multiple connections (and thus multiple terminators) for redundancy or load balancing purposes, that we can address the application, rather than an individual terminator.
 
-To this end, terminators now have two new fields: 
+To this end, terminators now have two new fields:
 
 1. Identity - defines the name by which a terminator can be addressed (along with other terminators using the same identity)
 1. IdentitySecret - allows verifying that terminators using the same identity are from the same application.
 
 Notes
- 
+
 1. Identity here may be related to the concept of the Edge Identity, but is not required to be.
-2. How IdentitySecret is used to validate a terminator is up to the terminator valiator for the binding. The edge has a terminator validator which uses the client's certs to ensure that all terminators for a given terminator identity belong to the same edge identity.  
+2. How IdentitySecret is used to validate a terminator is up to the terminator valiator for the binding. The edge has a terminator validator which uses the client's certs to ensure that all terminators for a given terminator identity belong to the same edge identity.
 
-The identity allows the service to be addressed with that identity. 
+The identity allows the service to be addressed with that identity.
 
-This can now be used in the fabric by prefixing the service name with the identity, separated by the `@` symbol. For example, if a service `ssh` has a terminator with identity `web-server`, it could be dialed using `web-server@ssh`. 
+This can now be used in the fabric by prefixing the service name with the identity, separated by the `@` symbol. For example, if a service `ssh` has a terminator with identity `web-server`, it could be dialed using `web-server@ssh`.
 
 The Edge SDK also supports dialing and binding using identity addressing. The `Context` type now supports a new `DialWithOptions` method, which can be used to specify an identity.
 
@@ -86,7 +206,7 @@ or
 For non-Edge SDK terminators the identity can be provided on the command line when creating the terminator.
 
 #### Example
-There is a simple application in the sdk-golang repository `example/call` which shows how identity addressing can be used to implement something like a VoIP service.  
+There is a simple application in the sdk-golang repository `example/call` which shows how identity addressing can be used to implement something like a VoIP service.
 
 ### xvt (eXtensible Termrinator Validation)
 The fabric now supports pluggable validators for terminators. Validators implement the following interface:
@@ -98,14 +218,14 @@ The fabric now supports pluggable validators for terminators. Validators impleme
 Validators can be registered with the controller on startup. For example, here is how the edge terminator validator is registered:
 
 	xtv.RegisterValidator("edge", env.NewEdgeTerminatorValidator(c.AppEnv))
-	
+
 Validators can then be tied to a binding in the config file:
 
     terminator:
       validators:
         edge: edge
 
-In the example above, terminators with the binding `edge` will use the validator which was registered under the name `edge`. The binding is the key and the validator name is the value. 
+In the example above, terminators with the binding `edge` will use the validator which was registered under the name `edge`. The binding is the key and the validator name is the value.
 
 # Release 0.16.2
 
@@ -119,7 +239,7 @@ In the example above, terminators with the binding `edge` will use the validator
     * [Fix service policy denormalization migration](https://github.com/openziti/edge/issues/291)
   * [sdk-golang#84](https://github.com/openziti/sdk-golang/issues/84) Fixes go routine leak that would slowly kill SDK application (i.e. ziti-probe)
   * REST API doc via ReDoc available at `https://<host>:<port>/docs`
-  
+
 # Release 0.16.1
 
 * What's New

etc/ctrl.with.edge.yml

@@ -50,6 +50,24 @@ metrics:
     #url:                http://localhost:8086
     #database:           ziti
 
+#events:
+#  jsonLogger:
+#    subscriptions:
+#      - type: metrics
+#        sourceFilter: .*
+#        metricFilter: .*egress.*m1_rate*
+#      - type: fabric.sessions
+#        include:
+#          - created
+#      - type: edge.sessions
+#        include:
+#          - created
+#      - type: fabric.usage
+#    handler:
+#      type: file
+#      format: json
+#      path: /tmp/ziti-events.log
+
 terminator:
   validators:
     edge: edge

go.mod

@@ -2,15 +2,15 @@ module github.com/openziti/ziti
 
 go 1.14
 
-//replace github.com/openziti/foundation => ../foundation
+// replace github.com/openziti/foundation => ../foundation
 
 // replace github.com/michaelquigley/dilithium => ../../q/research/dilithium
 
-//replace github.com/openziti/fabric => ../fabric
+// replace github.com/openziti/fabric => ../fabric
 
-//replace github.com/openziti/sdk-golang => ../sdk-golang
+// replace github.com/openziti/sdk-golang => ../sdk-golang
 
-//replace github.com/openziti/edge => ../edge
+// replace github.com/openziti/edge => ../edge
 
 require (
 	github.com/Jeffail/gabs v1.4.0
@@ -21,15 +21,16 @@ require (
 	github.com/go-ole/go-ole v1.2.4 // indirect
 	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
 	github.com/golang/protobuf v1.4.2
+	github.com/google/go-cmp v0.5.2
 	github.com/gorilla/mux v1.8.0
 	github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d
 	github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0
 	github.com/keybase/go-ps v0.0.0-20190827175125-91aafc93ba19
 	github.com/michaelquigley/pfxlog v0.0.0-20190813191113-2be43bd0dccc
-	github.com/openziti/edge v0.16.38
-	github.com/openziti/fabric v0.14.8
-	github.com/openziti/foundation v0.14.8
-	github.com/openziti/sdk-golang v0.13.52
+	github.com/openziti/edge v0.16.48
+	github.com/openziti/fabric v0.14.10
+	github.com/openziti/foundation v0.14.10
+	github.com/openziti/sdk-golang v0.13.55
 	github.com/pborman/uuid v1.2.0
 	github.com/pkg/errors v0.9.1
 	github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0
@@ -44,7 +45,7 @@ require (
 	github.com/stretchr/testify v1.6.1
 	github.com/urfave/negroni v1.0.0
 	golang.org/x/net v0.0.0-20200707034311-ab3426394381
-	google.golang.org/grpc v1.25.1
+	google.golang.org/grpc v1.27.0
 	gopkg.in/AlecAivazis/survey.v1 v1.8.7
 	gopkg.in/resty.v1 v1.12.0
 	gopkg.in/yaml.v2 v2.3.0

go.sum

@@ -38,6 +38,9 @@ github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:l
 github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg=
 github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 h1:4daAzAu0S6Vi7/lbWECcX0j45yZReDZ56BQsrVBOEEY=
 github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg=
+github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef h1:46PFijGLmAjMPwCCCo7Jf0W6f9slllCkkv7vyc1yOSg=
+github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
+github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/biogo/store v0.0.0-20190426020002-884f370e325d h1:vu2gsANkGtqYaQNXhmAAiJ7b1eKTbX3/aPAvDCasBgE=
@@ -77,7 +80,7 @@ github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDD
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg=
 github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o=
-github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
@@ -178,6 +181,7 @@ github.com/go-openapi/validate v0.19.10 h1:tG3SZ5DC5KF4cyt7nqLVcQXGj5A7mpaYkAcNP
 github.com/go-openapi/validate v0.19.10/go.mod h1:RKEZTUWDkxKQxN2jDT7ZnZi2bhZlbNMAuKvKB+IaGx8=
 github.com/go-openapi/validate v0.19.11 h1:8lCr0b9lNWKjVjW/hSZZvltUy+bULl7vbnCTsOzlhPo=
 github.com/go-openapi/validate v0.19.11/go.mod h1:Rzou8hA/CBw8donlS6WNEUQupNvUZ0waH08tGe6kAQ4=
+github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0=
@@ -230,6 +234,7 @@ github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrU
 github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
 github.com/golang/protobuf v1.4.0 h1:oOuy+ugB+P/kBdUnG5QaMXSIyJ1q38wWSojYCb3z5VQ=
 github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
 github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
@@ -242,6 +247,7 @@ github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
@@ -284,9 +290,13 @@ github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839/go.mod h1
 github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU=
 github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGARJA=
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc=
 github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
+github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/jsimonetti/rtnetlink v0.0.0-20190606172950-9527aa82566a h1:84IpUNXj4mCR9CuCEvSiCArMbzr/TMbuPIadKDwypkI=
 github.com/jsimonetti/rtnetlink v0.0.0-20190606172950-9527aa82566a/go.mod h1:Oz+70psSo5OFh8DBl0Zv2ACw7Esh6pPUphlvZG9x7uw=
 github.com/jsimonetti/rtnetlink v0.0.0-20200117123717-f846d4f6c1f4 h1:nwOc1YaOrYJ37sEBrtWZrdqzK22hiJs3GpDmP3sR2Yw=
@@ -298,8 +308,8 @@ github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 h1:iQTw/8FWTuc7uia
 github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8=
 github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4=
 github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA=
-github.com/kataras/go-events v0.0.3-0.20170604004442-17d67be645c3 h1:RGJQh17kB5HdupMRzKz89QzkYBnLRJJ+6SCurbvd0hw=
-github.com/kataras/go-events v0.0.3-0.20170604004442-17d67be645c3/go.mod h1:6IxMW59VJdEIqj3bjFGJvGLRdb0WHtrlxPZy9qXctcg=
+github.com/kataras/go-events v0.0.3-0.20201007151548-c411dc70c0a6 h1:XXnySN8wVet8S4UlcXHjh8Oa7618Eu7z92HxU5cIfhA=
+github.com/kataras/go-events v0.0.3-0.20201007151548-c411dc70c0a6/go.mod h1:6IxMW59VJdEIqj3bjFGJvGLRdb0WHtrlxPZy9qXctcg=
 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
 github.com/keybase/go-ps v0.0.0-20190827175125-91aafc93ba19 h1:WjT3fLi9n8YWh/Ih8Q1LHAPsTqGddPcHqscN+PJ3i68=
@@ -328,8 +338,8 @@ github.com/labstack/gommon v0.3.0 h1:JEeO0bvc78PKdyHxloTKiF8BD5iGrH8T6MSeGvSgob0
 github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k=
 github.com/lucas-clemente/quic-go v0.17.1 h1:ezsH76xpn6hKugfsXUy6voIJBFmAOwnM/Oy9F4b/n+M=
 github.com/lucas-clemente/quic-go v0.17.1/go.mod h1:I0+fcNTdb9eS1ZcjQZbDVPGchJ86chcIxPALn9lEJqE=
-github.com/lucas-clemente/quic-go v0.18.0 h1:JhQDdqxdwdmGdKsKgXi1+coHRoGhvU6z0rNzOJqZ/4o=
-github.com/lucas-clemente/quic-go v0.18.0/go.mod h1:yXttHsSNxQi8AWijC/vLP+OJczXqzHSOcJrM5ITUlCg=
+github.com/lucas-clemente/quic-go v0.18.1 h1:DMR7guC0NtVS8zNZR3IO7NARZvZygkSC56GGtC6cyys=
+github.com/lucas-clemente/quic-go v0.18.1/go.mod h1:yXttHsSNxQi8AWijC/vLP+OJczXqzHSOcJrM5ITUlCg=
 github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
@@ -340,6 +350,8 @@ github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.7.1 h1:mdxE1MF9o53iCb2Ghj1VfWvh7ZOwHpnVG/xwXrV90U8=
 github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
+github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA=
+github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE=
 github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0=
 github.com/marten-seemann/qpack v0.1.0/go.mod h1:LFt1NU/Ptjip0C2CPkhimBz5CGE3WGDAUWqna+CNTrI=
@@ -392,6 +404,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/natefinch/lumberjack v2.0.0+incompatible h1:4QJd3OLAMgj7ph+yZTuX13Ld4UpgHp07nNdFX7mqFfM=
+github.com/natefinch/lumberjack v2.0.0+incompatible/go.mod h1:Wi9p2TTF5DG5oU+6YfsmYQpsTIOm0B1VNzQg9Mw6nPk=
 github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo=
 github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM=
 github.com/netfoundry/secretstream v0.1.2 h1:NgqrYytDnjKbOfWI29TT0SJM+RwB3yf9MIkJVJaU+J0=
@@ -418,14 +432,14 @@ github.com/onsi/gomega v1.8.1/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoT
 github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
 github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8=
-github.com/openziti/edge v0.16.38 h1:zV7P4cK8LO8NyGEiJlJo8Bxf0TjgI+BwRxdrhUg2pqg=
-github.com/openziti/edge v0.16.38/go.mod h1:zaRj81nISMQrkegOfJeOAwhOlDJRTTxJkQOEC5DVrZo=
-github.com/openziti/fabric v0.14.8 h1:ATV0EZwISvhhMiNl1VkYZ6Ia/rAnqc+Rnahx5QbJXQc=
-github.com/openziti/fabric v0.14.8/go.mod h1:fzxNGqL20fBWrvrMu+5UxjZVLaCrZmt/zcvgqD+zzjg=
-github.com/openziti/foundation v0.14.8 h1:WgQMZxXrMZHvqF9vhq541Djm7UjXZr+WMiAaFDskm48=
-github.com/openziti/foundation v0.14.8/go.mod h1:b8Ox/vvvdObz1JoEno0z7KN1cx0echQ7eP/tOmjbB+4=
-github.com/openziti/sdk-golang v0.13.52 h1:DXsjYx8Sfv8crmyamHGq+7E5hhZIIwfpsojjj9lU9xk=
-github.com/openziti/sdk-golang v0.13.52/go.mod h1:iYYT+vXQm4eMDsFcZ2ClKzEwT8eickOakV3FecVKMOw=
+github.com/openziti/edge v0.16.48 h1:h4Kjas4/btQwyMnO116T1isFYchl14OmjHyvwY+ipIY=
+github.com/openziti/edge v0.16.48/go.mod h1:F9MaTSc0oas0H6DdKL6EbVCccwL98jq+A4iRz7W9M78=
+github.com/openziti/fabric v0.14.10 h1:H+5l0K9OrmpSA+N9mETmhxgG0vxmThQs6ho7W9d904c=
+github.com/openziti/fabric v0.14.10/go.mod h1:v1eqj73dzfdBZJw48z8xMYqH4vgoWkpnhkg+qq1Nkb4=
+github.com/openziti/foundation v0.14.10 h1:y5V1s/Iz2Ulsm06tMe9QhNr6YqAHFmznN7gYsI03j4Y=
+github.com/openziti/foundation v0.14.10/go.mod h1:tmvDNdFW5EKKYZM4s3zyiwarvkLLpnDL+WBvtREtYbY=
+github.com/openziti/sdk-golang v0.13.55 h1:X2mXG8rLqvK8czV8LLjkgIn6wZ99m8RJiqaJVJRWue8=
+github.com/openziti/sdk-golang v0.13.55/go.mod h1:SZo6goNAsvEu1kglZnC5fFRnr/C64eJl4G2OoU5JihA=
 github.com/orcaman/concurrent-map v0.0.0-20190826125027-8c72a8bb44f6 h1:lNCW6THrCKBiJBpz8kbVGjC7MgdCGKwuvBgc7LoD6sw=
 github.com/orcaman/concurrent-map v0.0.0-20190826125027-8c72a8bb44f6/go.mod h1:Lu3tH6HLW3feq74c2GC+jIMS/K2CFcDWnWD9XkenwhI=
 github.com/pborman/uuid v1.2.0 h1:J7Q5mO4ysT1dv8hyrUGHb9+ooztCXu1D8MY8DZYsu3g=
@@ -434,6 +448,8 @@ github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pelletier/go-toml v1.4.0 h1:u3Z1r+oOXJIkxqw34zVhyPgjBsm6X2wn21NWs/HfSeg=
 github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo=
+github.com/pelletier/go-toml v1.7.0 h1:7utD74fnzVc/cpcyy8sjrlFr5vYpypUixARcHIMIGuI=
+github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -571,6 +587,8 @@ go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qL
 go.mongodb.org/mongo-driver v1.3.0/go.mod h1:MSWZXKOynuguX+JSvwP8i+58jYCXxbia8HS3gZBapIE=
 go.mongodb.org/mongo-driver v1.3.4 h1:zs/dKNwX0gYUtzwrN9lLiR15hCO0nDwQj5xXx+vjCdE=
 go.mongodb.org/mongo-driver v1.3.4/go.mod h1:MSWZXKOynuguX+JSvwP8i+58jYCXxbia8HS3gZBapIE=
+go.mongodb.org/mongo-driver v1.4.2 h1:WlnEglfTg/PfPq4WXs2Vkl/5ICC6hoG8+r+LraPmGk4=
+go.mongodb.org/mongo-driver v1.4.2/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc=
 go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
@@ -598,6 +616,8 @@ golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899 h1:DZhuSZLsGlFL4CmhA8BcRA0mnthyA/nZ00AqCUo7vHg=
+golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -648,6 +668,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEha
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 h1:qwRHBd0NqMbJxfbotnDhm2ByMI1Shq4Y6oRJo21SGJA=
+golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180606202747-9527bec2660b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -737,6 +759,8 @@ google.golang.org/genproto v0.0.0-20190306203927-b5d61aea6440/go.mod h1:VzzqZJRn
 google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE=
 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
 google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
 google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio=
 google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
@@ -744,15 +768,19 @@ google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZi
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
 google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
-google.golang.org/grpc v1.25.1 h1:wdKvqQk7IttEw92GoRyKG2IDrUIpgpj6H6m81yfeMW0=
-google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.27.0 h1:rRYRFMVgRv6E0D70Skyfsr28tDXIuuPZyWGMPdMcnXg=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
 google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
 google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM=
 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 gopkg.in/AlecAivazis/survey.v1 v1.8.7 h1:oBJqtgsyBLg9K5FK9twNUbcPnbCPoh+R9a+7nag3qJM=
 gopkg.in/AlecAivazis/survey.v1 v1.8.7/go.mod h1:iBNOmqKz/NUbZx3bA+4hAGLRC7fSK7tgtVDT4tB22XA=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
@@ -765,6 +793,8 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
 gopkg.in/resty.v1 v1.12.0 h1:CuXP0Pjfw9rOuY6EP+UvtNvt5DSqHpIxILZKT/quCZI=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=

ziti-controller/subcmd/run.go

@@ -18,10 +18,10 @@ package subcmd
 
 import (
 	"fmt"
-	"github.com/openziti/ziti/common/version"
-	"github.com/openziti/ziti/ziti/agent"
 	"github.com/openziti/edge/controller/server"
 	"github.com/openziti/fabric/controller"
+	"github.com/openziti/ziti/common/version"
+	"github.com/openziti/ziti/ziti/agent"
 	"github.com/spf13/cobra"
 )
 
@@ -45,7 +45,7 @@ func run(cmd *cobra.Command, args []string) {
 		}
 
 		var c *controller.Controller
-		if c, err = controller.NewController(config,  version.GetCmdBuildInfo()); err != nil {
+		if c, err = controller.NewController(config, version.GetCmdBuildInfo()); err != nil {
 			fmt.Printf("unable to create fabric controller %+v\n", err)
 			panic(err)
 		}
@@ -57,8 +57,9 @@ func run(cmd *cobra.Command, args []string) {
 		}
 
 		ec.SetHostController(c)
+		ec.Initialize()
 
-		go ec.InitAndRun()
+		go ec.RunAndWait()
 
 		if err = c.Run(); err != nil {
 			panic(err)

ziti-fabric-test/main.go

@@ -28,6 +28,7 @@ import (
 	"github.com/openziti/ziti/ziti-fabric-test/subcmd"
 	_ "github.com/openziti/ziti/ziti-fabric-test/subcmd/client"
 	_ "github.com/openziti/ziti/ziti-fabric-test/subcmd/loop2"
+	_ "github.com/openziti/ziti/ziti-fabric-test/subcmd/loop3"
 	"github.com/sirupsen/logrus"
 	"math/rand"
 )

ziti-fabric-test/subcmd/loop3/dialer.go

@@ -19,12 +19,13 @@ package loop3
 import (
 	"fmt"
 	"github.com/michaelquigley/pfxlog"
-	loop3_pb "github.com/openziti/ziti/ziti-fabric-test/subcmd/loop3/pb"
 	"github.com/openziti/fabric/router/xgress_transport"
 	"github.com/openziti/foundation/identity/dotziti"
 	"github.com/openziti/foundation/identity/identity"
 	"github.com/openziti/foundation/transport"
 	"github.com/openziti/sdk-golang/ziti"
+	"github.com/openziti/sdk-golang/ziti/config"
+	loop3_pb "github.com/openziti/ziti/ziti-fabric-test/subcmd/loop3/pb"
 	"github.com/spf13/cobra"
 	"net"
 	"strings"
@@ -32,25 +33,41 @@ import (
 )
 
 func init() {
-	dialerCmd.Flags().StringVarP(&dialerCmdIdentity, "identity", "i", "default", ".ziti/identities.yml name")
-	dialerCmd.Flags().StringVarP(&dialerCmdEndpoint, "endpoint", "e", "tls:127.0.0.1:7001", "Endpoint address")
-	dialerCmd.Flags().BoolVarP(&dialerCmdDirect, "direct", "d", false, "Transmit direct (no ingress)")
-	dialerCmd.Flags().StringVarP(&dialerCmdService, "service", "s", "loop", "Service name for ingress")
-	loop3Cmd.AddCommand(dialerCmd)
+	dialerCmd := newDialerCmd()
+	loop3Cmd.AddCommand(dialerCmd.cmd)
 }
 
-var dialerCmd = &cobra.Command{
-	Use:   "dialer <scenarioFile>",
-	Short: "Start loop3 dialer",
-	Args:  cobra.ExactArgs(1),
-	Run:   dialer,
+type dialerCmd struct {
+	cmd            *cobra.Command
+	identity       string
+	endpoint       string
+	direct         bool
+	service        string
+	edgeConfigFile string
 }
-var dialerCmdIdentity string
-var dialerCmdEndpoint string
-var dialerCmdDirect bool
-var dialerCmdService string
 
-func dialer(_ *cobra.Command, args []string) {
+func newDialerCmd() *dialerCmd {
+	result := &dialerCmd{
+		cmd: &cobra.Command{
+			Use:   "dialer <scenarioFile>",
+			Short: "Start loop2 dialer",
+			Args:  cobra.ExactArgs(1),
+		},
+	}
+
+	result.cmd.Run = result.run
+
+	flags := result.cmd.Flags()
+	flags.StringVarP(&result.identity, "identity", "i", "default", ".ziti/identities.yml name")
+	flags.StringVarP(&result.endpoint, "endpoint", "e", "tls:127.0.0.1:7001", "Endpoint address")
+	flags.BoolVarP(&result.direct, "direct", "d", false, "Transmit direct (no ingress)")
+	flags.StringVarP(&result.service, "service", "s", "loop", "Service name for ingress")
+	flags.StringVarP(&result.edgeConfigFile, "config-file", "c", "", "Edge SDK config file")
+
+	return result
+}
+
+func (cmd *dialerCmd) run(_ *cobra.Command, args []string) {
 	log := pfxlog.Logger()
 
 	scenario, err := LoadScenario(args[0])
@@ -59,67 +76,51 @@ func dialer(_ *cobra.Command, args []string) {
 	}
 	log.Debug(scenario)
 
-	resultChs := make(map[string]chan *loop3_pb.Result)
+	if scenario.Metrics != nil {
+		closer := make(chan struct{})
+		if err := StartMetricsReporter(cmd.edgeConfigFile, scenario.Metrics, closer); err != nil {
+			panic(err)
+		}
+		defer close(closer)
+	}
+
+	resultChs := make(map[string]chan *Result)
 	for _, workload := range scenario.Workloads {
 		log.Infof("executing workload [%s] with concurrency [%d]", workload.Name, workload.Concurrency)
 
+		var conns []net.Conn
 		for i := 0; i < int(workload.Concurrency); i++ {
+			conns = append(conns, cmd.connect())
+		}
+
+		for i, conn := range conns {
 			name := fmt.Sprintf("%s:%d", workload.Name, i)
-			resultCh := make(chan *loop3_pb.Result, 1)
+			resultCh := make(chan *Result, 1)
 			resultChs[name] = resultCh
 
 			go func() {
-				var conn net.Conn
-				if strings.HasPrefix(dialerCmdEndpoint, "edge:") {
-					context := ziti.NewContext()
-					service := strings.TrimPrefix(dialerCmdEndpoint, "edge:")
-					conn, err = context.Dial(service)
-					if err != nil {
-						panic(err)
-					}
-				} else {
-					endpoint, err := transport.ParseAddress(dialerCmdEndpoint)
-					if err != nil {
-						panic(err)
-					}
-
-					_, id, err := dotziti.LoadIdentity(dialerCmdIdentity)
-					if err != nil {
-						panic(err)
-					}
-
-					if dialerCmdDirect {
-						if conn, err = dialDirect(endpoint, id); err != nil {
-							panic(err)
-						}
-					} else {
-						serviceId := &identity.TokenId{Token: dialerCmdService}
-						if conn, err = dialIngress(endpoint, id, serviceId); err != nil {
-							panic(err)
-						}
-					}
-				}
-
 				workload := scenario.Workloads[0]
 				local := &loop3_pb.Test{
-					Name:            name,
-					TxRequests:      workload.Dialer.TxRequests,
-					TxPacing:        workload.Dialer.TxPacing,
-					TxMaxJitter:     workload.Dialer.TxMaxJitter,
-					RxRequests:      workload.Listener.TxRequests,
-					RxTimeout:       workload.Dialer.RxTimeout,
-					PayloadMinBytes: workload.Dialer.PayloadMinBytes,
-					PayloadMaxBytes: workload.Dialer.PayloadMaxBytes,
+					Name:             name,
+					TxRequests:       workload.Dialer.TxRequests,
+					TxPacing:         workload.Dialer.TxPacing,
+					TxMaxJitter:      workload.Dialer.TxMaxJitter,
+					RxRequests:       workload.Listener.TxRequests,
+					RxTimeout:        workload.Dialer.RxTimeout,
+					PayloadMinBytes:  workload.Dialer.PayloadMinBytes,
+					PayloadMaxBytes:  workload.Dialer.PayloadMaxBytes,
+					LatencyFrequency: workload.Dialer.LatencyFrequency,
 				}
 				remote := &loop3_pb.Test{
-					Name:            name,
-					TxRequests:      workload.Listener.TxRequests,
-					TxPacing:        workload.Listener.TxPacing,
-					TxMaxJitter:     workload.Listener.TxMaxJitter,
-					RxRequests:      workload.Dialer.TxRequests,
-					RxTimeout:       workload.Listener.RxTimeout,
-					PayloadMinBytes: workload.Listener.PayloadMinBytes,
-					PayloadMaxBytes: workload.Listener.PayloadMaxBytes,
+					Name:             name,
+					TxRequests:       workload.Listener.TxRequests,
+					TxPacing:         workload.Listener.TxPacing,
+					TxMaxJitter:      workload.Listener.TxMaxJitter,
+					RxRequests:       workload.Dialer.TxRequests,
+					RxTimeout:        workload.Listener.RxTimeout,
+					PayloadMinBytes:  workload.Listener.PayloadMinBytes,
+					PayloadMaxBytes:  workload.Listener.PayloadMaxBytes,
+					LatencyFrequency: workload.Listener.LatencyFrequency,
 				}
 
 				if proto, err := newProtocol(conn); err == nil {
@@ -162,6 +163,60 @@ func dialer(_ *cobra.Command, args []string) {
 	}
 }
 
+func (cmd *dialerCmd) connect() net.Conn {
+	log := pfxlog.Logger()
+
+	start := time.Now()
+
+	var conn net.Conn
+	var err error
+	if strings.HasPrefix(cmd.endpoint, "edge:") {
+		var context ziti.Context
+		if cmd.edgeConfigFile != "" {
+			zitiCfg, err := config.NewFromFile(cmd.edgeConfigFile)
+			if err != nil {
+				log.Fatalf("failed to load ziti configuration from %s: %v", cmd.edgeConfigFile, err)
+			}
+			context = ziti.NewContextWithConfig(zitiCfg)
+		} else {
+			context = ziti.NewContext()
+		}
+
+		service := strings.TrimPrefix(cmd.endpoint, "edge:")
+		conn, err = context.DialWithOptions(service, &ziti.DialOptions{
+			ConnectTimeout: time.Second * 30,
+		})
+		if err != nil {
+			panic(err)
+		}
+	} else {
+		endpoint, err := transport.ParseAddress(cmd.endpoint)
+		if err != nil {
+			panic(err)
+		}
+
+		_, id, err := dotziti.LoadIdentity(cmd.identity)
+		if err != nil {
+			panic(err)
+		}
+
+		if cmd.direct {
+			if conn, err = dialDirect(endpoint, id); err != nil {
+				panic(err)
+			}
+		} else {
+			serviceId := &identity.TokenId{Token: cmd.service}
+			if conn, err = dialIngress(endpoint, id, serviceId); err != nil {
+				panic(err)
+			}
+		}
+	}
+
+	ConnectionTime.Update(time.Now().Sub(start))
+
+	return conn
+}
+
 func dialDirect(endpoint transport.Address, id *identity.TokenId) (net.Conn, error) {
 	peer, err := endpoint.Dial("loop", id, nil)
 	if err != nil {

ziti-fabric-test/subcmd/loop3/generator.go

@@ -17,27 +17,29 @@
 package loop3
 
 import (
-	"github.com/openziti/foundation/util/info"
 	"crypto/sha512"
 	"github.com/michaelquigley/pfxlog"
+	"github.com/openziti/foundation/util/info"
 	"math/rand"
 )
 
 type generator struct {
-	count   int
-	minSize int
-	maxSize int
-	blocks  chan *Block
-	pool    [][]byte
+	count       int
+	minSize     int
+	maxSize     int
+	latencyFreq int
+	blocks      chan *Block
+	pool        [][]byte
 }
 
-func newGenerator(count, minSize, maxSize int) *generator {
+func newGenerator(count, minSize, maxSize, latencyFreq int) *generator {
 	g := &generator{
-		count:   count,
-		minSize: minSize,
-		maxSize: maxSize,
-		blocks:  make(chan *Block, 128),
-		pool:    newPool(),
+		count:       count,
+		minSize:     minSize,
+		maxSize:     maxSize,
+		latencyFreq: latencyFreq,
+		blocks:      make(chan *Block),
+		pool:        newPool(),
 	}
 	return g
 }
@@ -62,8 +64,13 @@ func (g *generator) run() {
 			}
 		}
 		hash := sha512.Sum512(data)
+		blockType := BlockTypePlain
+		if g.latencyFreq > 0 && i%g.latencyFreq == 0 {
+			blockType = BlockTypeLatencyRequest
+		}
 		g.blocks <- &Block{
-			Sequence: int32(i),
+			Type:     blockType,
+			Sequence: uint32(i),
 			Data:     data,
 			Hash:     hash[:],
 		}

ziti-fabric-test/subcmd/loop3/listener.go

@@ -17,44 +17,98 @@
 package loop3
 
 import (
-	"github.com/openziti/ziti/ziti-fabric-test/subcmd/loop3/pb"
+	"github.com/michaelquigley/pfxlog"
 	"github.com/openziti/foundation/identity/dotziti"
 	"github.com/openziti/foundation/identity/identity"
 	"github.com/openziti/foundation/transport"
-	"github.com/michaelquigley/pfxlog"
+	"github.com/openziti/sdk-golang/ziti"
+	"github.com/openziti/sdk-golang/ziti/config"
 	"github.com/spf13/cobra"
+	"net"
+	"strings"
 )
 
 func init() {
-	listenerCmd.Flags().StringVarP(&listenerCmdIdentity, "identity", "i", "default", ".ziti/identities.yml name")
-	listenerCmd.Flags().StringVarP(&listenerCmdBindAddress, "bind", "b", "tcp:127.0.0.1:8171", "Listener bind address")
-	loop3Cmd.AddCommand(listenerCmd)
+	listenerCmd := newListenerCmd()
+	loop3Cmd.AddCommand(listenerCmd.cmd)
 }
 
-var listenerCmd = &cobra.Command{
-	Use:   "listener",
-	Short: "Start loop3 listener",
-	Args:  cobra.ExactArgs(0),
-	Run:   listener,
+type listenerCmd struct {
+	cmd            *cobra.Command
+	identity       string
+	bindAddress    string
+	edgeConfigFile string
 }
-var listenerCmdIdentity string
-var listenerCmdBindAddress string
 
-func listener(_ *cobra.Command, _ []string) {
-	_, id, err := dotziti.LoadIdentity(listenerCmdIdentity)
+func newListenerCmd() *listenerCmd {
+	result := &listenerCmd{
+		cmd: &cobra.Command{
+			Use:   "listener",
+			Short: "Start loop2 listener",
+			Args:  cobra.ExactArgs(0),
+		},
+	}
+
+	result.cmd.Run = result.run
+
+	flags := result.cmd.Flags()
+	flags.StringVarP(&result.identity, "identity", "i", "default", ".ziti/identities.yml name")
+	flags.StringVarP(&result.bindAddress, "bind", "b", "tcp:127.0.0.1:8171", "Listener bind address")
+	flags.StringVarP(&result.edgeConfigFile, "config-file", "c", "", "Edge SDK config file")
+
+	return result
+}
+
+func (cmd *listenerCmd) run(_ *cobra.Command, _ []string) {
+	if strings.HasPrefix(cmd.bindAddress, "edge") {
+		cmd.listenEdge()
+	} else {
+		_, id, err := dotziti.LoadIdentity(cmd.identity)
+		if err != nil {
+			panic(err)
+		}
+
+		bindAddress, err := transport.ParseAddress(cmd.bindAddress)
+		if err != nil {
+			panic(err)
+		}
+
+		cmd.listen(bindAddress, id)
+	}
+}
+
+func (cmd *listenerCmd) listenEdge() {
+	log := pfxlog.ContextLogger(cmd.bindAddress)
+	defer log.Error("exited")
+	log.Info("started")
+
+	var context ziti.Context
+	if cmd.edgeConfigFile != "" {
+		zitiCfg, err := config.NewFromFile(cmd.edgeConfigFile)
+		if err != nil {
+			log.Fatalf("failed to load ziti configuration from %s: %v", cmd.edgeConfigFile, err)
+		}
+		context = ziti.NewContextWithConfig(zitiCfg)
+	} else {
+		context = ziti.NewContext()
+	}
+
+	service := strings.TrimPrefix(cmd.bindAddress, "edge:")
+	listener, err := context.Listen(service)
 	if err != nil {
 		panic(err)
 	}
 
-	bindAddress, err := transport.ParseAddress(listenerCmdBindAddress)
-	if err != nil {
-		panic(err)
+	for {
+		if conn, err := listener.Accept(); err != nil {
+			panic(err)
+		} else {
+			go cmd.handle(conn, cmd.bindAddress)
+		}
 	}
-
-	listen(bindAddress, id)
 }
 
-func listen(bind transport.Address, i *identity.TokenId) {
+func (cmd *listenerCmd) listen(bind transport.Address, i *identity.TokenId) {
 	log := pfxlog.ContextLogger(bind.String())
 	defer log.Error("exited")
 	log.Info("started")
@@ -69,7 +123,7 @@ func listen(bind transport.Address, i *identity.TokenId) {
 		select {
 		case peer := <-incoming:
 			if peer != nil {
-				go handle(peer)
+				go cmd.handle(peer.Conn(), peer.Detail().String())
 			} else {
 				return
 			}
@@ -77,17 +131,17 @@ func listen(bind transport.Address, i *identity.TokenId) {
 	}
 }
 
-func handle(peer transport.Connection) {
-	log := pfxlog.ContextLogger(peer.Detail().String())
-	if proto, err := newProtocol(peer.Conn()); err == nil {
+func (cmd *listenerCmd) handle(conn net.Conn, context string) {
+	log := pfxlog.ContextLogger(context)
+	if proto, err := newProtocol(conn); err == nil {
 		if test, err := proto.rxTest(); err == nil {
-			var result *loop3_pb.Result
+			var result *Result
 			if err := proto.run(test); err == nil {
-				result = &loop3_pb.Result{Success: true}
+				result = &Result{Success: true}
 			} else {
-				result = &loop3_pb.Result{Success: false, Message: err.Error()}
+				result = &Result{Success: false, Message: err.Error()}
 			}
-			if err := proto.txResult(result); err != nil {
+			if err := result.Tx(proto); err != nil {
 				log.Errorf("unable to tx result (%s)", err)
 			}
 		}

ziti-fabric-test/subcmd/loop3/messages.go

@@ -0,0 +1,213 @@
+package loop3
+
+import (
+	"bytes"
+	"encoding/binary"
+	"github.com/michaelquigley/pfxlog"
+	"github.com/openziti/foundation/util/info"
+	"github.com/pkg/errors"
+	"time"
+)
+
+type Message interface {
+	Tx(p *protocol) error
+	Rx(p *protocol) error
+}
+
+var success = []byte{1}
+var failure = []byte{0}
+
+type Result struct {
+	Success bool
+	Message string
+}
+
+func (r *Result) getSuccessBytes() []byte {
+	if r.Success {
+		return success
+	}
+	return failure
+}
+
+func (r *Result) Tx(p *protocol) error {
+	dataLen := 1 + len(r.Message)
+	if err := p.txHeader(dataLen); err != nil {
+		return err
+	}
+
+	if _, err := p.peer.Write(r.getSuccessBytes()); err != nil {
+		return err
+	}
+
+	if len(r.Message) > 0 {
+		if _, err := p.peer.Write([]byte(r.Message)); err != nil {
+			return err
+		}
+	}
+
+	MsgTxRate.Mark(1)
+	BytesTxRate.Mark(int64(4 + 4 + dataLen))
+
+	if r.Success {
+		pfxlog.ContextLogger(p.test.Name).Infof("<- [result+]")
+	} else {
+		pfxlog.ContextLogger(p.test.Name).Infof("<- [result-]")
+	}
+
+	return nil
+}
+
+func (r *Result) Rx(p *protocol) error {
+	msgLen, err := p.rxHeader()
+	if err != nil {
+		return err
+	}
+	if msgLen < 1 {
+		return errors.Errorf("not enough data to deserialize Result need at least one byte")
+	}
+	body, err := p.rxMsgBody(msgLen)
+	if err != nil {
+		return err
+	}
+	r.Success = body[0] == 1
+	r.Message = string(body[1:])
+
+	MsgRxRate.Mark(1)
+	BytesRxRate.Mark(int64(4 + 4 + msgLen))
+
+	if r.Success {
+		pfxlog.ContextLogger(p.test.Name).Infof("<- [result+]")
+	} else {
+		pfxlog.ContextLogger(p.test.Name).Infof("<- [result-]")
+	}
+
+	return nil
+}
+
+const (
+	BlockTypePlain           byte = 1
+	BlockTypeLatencyRequest       = 2
+	BlockTypeLatencyResponse      = 3
+)
+
+type Block struct {
+	Type      byte
+	Sequence  uint32
+	Hash      []byte
+	Data      []byte
+	Timestamp time.Time
+}
+
+func (block *Block) getTimestampBytes() ([]byte, error) {
+	if block.Type == BlockTypeLatencyRequest {
+		block.Timestamp = time.Now()
+	}
+
+	tsBuf := bytes.Buffer{}
+
+	if block.Type != BlockTypePlain {
+		ts, err := block.Timestamp.MarshalBinary()
+		if err != nil {
+			return nil, err
+		}
+
+		if len(ts) > 32 {
+			panic(errors.Errorf("unexpected timestamp length: %v", len(ts)))
+		}
+
+		tsBuf.WriteByte(byte(len(ts)))
+		tsBuf.Write(ts)
+	}
+	return tsBuf.Bytes(), nil
+}
+
+func (block *Block) Tx(p *protocol) error {
+	tsBytes, err := block.getTimestampBytes()
+	if err != nil {
+		return err
+	}
+
+	dataLen := 1 /* block type */ + len(tsBytes) + 4 /* sequence bytes */ + len(block.Hash) + len(block.Data)
+
+	if err := p.txHeader(dataLen); err != nil {
+		return err
+	}
+
+	if _, err := p.peer.Write([]byte{block.Type}); err != nil {
+		return err
+	}
+
+	if len(tsBytes) > 0 {
+		if _, err := p.peer.Write(tsBytes); err != nil {
+			return err
+		}
+	}
+
+	seqBytes := make([]byte, 4)
+	binary.LittleEndian.PutUint32(seqBytes, block.Sequence)
+
+	if _, err := p.peer.Write(seqBytes); err != nil {
+		return err
+	}
+
+	if _, err := p.peer.Write(block.Hash); err != nil {
+		return err
+	}
+
+	if _, err := p.peer.Write(block.Data); err != nil {
+		return err
+	}
+
+	MsgTxRate.Mark(1)
+	BytesTxRate.Mark(int64(8 + dataLen))
+
+	pfxlog.ContextLogger(p.test.Name).Infof("-> #%d (%s)", block.Sequence, info.ByteCount(int64(len(block.Data))))
+
+	return nil
+}
+
+func (block *Block) Rx(p *protocol) error {
+	length, err := p.rxHeader()
+	if err != nil {
+		return err
+	}
+
+	body, err := p.rxMsgBody(length)
+	if err != nil {
+		return err
+	}
+
+	buf := bytes.NewBuffer(body)
+	block.Type, err = buf.ReadByte()
+	if err != nil {
+		return err
+	}
+
+	if block.Type != BlockTypePlain {
+		tsLen, err := buf.ReadByte()
+		if err != nil {
+			return err
+		}
+		if err := block.Timestamp.UnmarshalBinary(buf.Next(int(tsLen))); err != nil {
+			return err
+		}
+	}
+
+	seqBytes := buf.Next(4)
+	block.Sequence = binary.LittleEndian.Uint32(seqBytes)
+
+	block.Hash = buf.Next(64)
+	block.Data = buf.Bytes()
+
+	MsgRxRate.Mark(1)
+	BytesRxRate.Mark(int64(8 + length))
+
+	if block.Type == BlockTypeLatencyResponse {
+		elapsed := time.Now().Sub(block.Timestamp)
+		MsgLatency.Update(elapsed)
+	}
+
+	pfxlog.ContextLogger(p.test.Name).Infof("<- #%d (%s)", block.Sequence, info.ByteCount(int64(len(block.Data))))
+
+	return nil
+}

ziti-fabric-test/subcmd/loop3/messages_test.go

@@ -0,0 +1,69 @@
+package loop3
+
+import (
+	"bytes"
+	"crypto/sha512"
+	"github.com/google/go-cmp/cmp"
+	loop3_pb "github.com/openziti/ziti/ziti-fabric-test/subcmd/loop3/pb"
+	"github.com/stretchr/testify/require"
+	"math/rand"
+	"reflect"
+	"testing"
+)
+
+type testPeer struct {
+	bytes.Buffer
+}
+
+func (t *testPeer) Close() error {
+	return nil
+}
+
+func Test_MessageSerDeser(t *testing.T) {
+	req := require.New(t)
+	data := make([]byte, 4192)
+	rand.Read(data)
+
+	hash := sha512.Sum512(data)
+
+	block := &Block{
+		Type:     BlockTypePlain,
+		Sequence: 10,
+		Hash:     hash[:],
+		Data:     data,
+	}
+
+	testBuf := &testPeer{}
+
+	p := &protocol{
+		peer: testBuf,
+		test: &loop3_pb.Test{
+			Name: "test",
+		},
+	}
+
+	req.NoError(block.Tx(p))
+
+	readBlock := &Block{}
+	req.NoError(readBlock.Rx(p))
+
+	req.True(reflect.DeepEqual(block, readBlock), cmp.Diff(block, readBlock))
+
+	data = make([]byte, 4192)
+	rand.Read(data)
+	hash = sha512.Sum512(data)
+
+	block = &Block{
+		Type:     BlockTypeLatencyRequest,
+		Sequence: 10,
+		Hash:     hash[:],
+		Data:     data,
+	}
+
+	req.NoError(block.Tx(p))
+
+	readBlock = &Block{}
+	req.NoError(readBlock.Rx(p))
+
+	req.Equal("", cmp.Diff(block, readBlock))
+}

ziti-fabric-test/subcmd/loop3/metrics.go

@@ -0,0 +1,193 @@
+package loop3
+
+import (
+	"encoding/binary"
+	"github.com/golang/protobuf/proto"
+	"github.com/golang/protobuf/ptypes"
+	"github.com/michaelquigley/pfxlog"
+	"github.com/openziti/fabric/pb/mgmt_pb"
+	"github.com/openziti/sdk-golang/ziti"
+	"github.com/openziti/sdk-golang/ziti/config"
+	"github.com/pkg/errors"
+	"github.com/rcrowley/go-metrics"
+	"time"
+)
+
+var registry = metrics.NewRegistry()
+
+var ConnectionTime = metrics.NewTimer()
+var MsgTxRate = metrics.NewMeter()
+var MsgRxRate = metrics.NewMeter()
+var BytesTxRate = metrics.NewMeter()
+var BytesRxRate = metrics.NewMeter()
+var MsgLatency = metrics.NewTimer()
+
+func init() {
+	register := func(name string, metric interface{}) {
+		if err := registry.Register(name, metric); err != nil {
+			panic(err)
+		}
+	}
+
+	register("tx.msg.rate", MsgTxRate)
+	register("tx.bytes.rate", BytesTxRate)
+	register("rx.msg.rate", MsgRxRate)
+	register("rx.bytes.rate", BytesRxRate)
+	register("msg.latency", MsgLatency)
+	register("conn.time", ConnectionTime)
+}
+
+func StartMetricsReporter(configFile string, metrics *Metrics, closer chan struct{}) error {
+	if metrics.ReportInterval == 0 {
+		return errors.New("metrics report interval must be greater than 0")
+	}
+
+	reporter := &zitiMetricsReporter{
+		configFile: configFile,
+		clientId:   metrics.ClientId,
+		closer:     closer,
+		service:    metrics.Service,
+	}
+
+	go reporter.run(metrics.ReportInterval)
+	return nil
+}
+
+type zitiMetricsReporter struct {
+	configFile string
+	clientId   string
+	closer     chan struct{}
+	service    string
+}
+
+func (r *zitiMetricsReporter) run(reportInterval time.Duration) {
+	log := pfxlog.Logger()
+	log.Infof("reporting metrics every %v", reportInterval)
+
+	ticker := time.NewTicker(reportInterval)
+	defer ticker.Stop()
+
+	var client ziti.Context
+	if r.configFile != "" {
+		sdkConfig, err := config.NewFromFile(r.configFile)
+		if err != nil {
+			panic(err)
+		}
+		client = ziti.NewContextWithConfig(sdkConfig)
+	} else {
+		client = ziti.NewContext()
+	}
+	conn, err := client.Dial(r.service)
+
+	if err != nil {
+		panic(err)
+	}
+
+	defer func() { _ = conn.Close() }()
+
+	lenBuf := make([]byte, 4)
+
+	for {
+		select {
+		case <-ticker.C:
+			event := r.createMetricsEvent()
+			buf, err := proto.Marshal(event)
+			if err != nil {
+				log.WithError(err).Error("unable to marshal streaming metrics")
+			} else {
+				length := len(buf)
+				binary.LittleEndian.PutUint32(lenBuf, uint32(length))
+				log.Infof("sending metrics message with len %v, %+v", length, lenBuf)
+				if _, err := conn.Write(lenBuf); err != nil {
+					panic(err)
+				}
+				if _, err := conn.Write(buf); err != nil {
+					panic(err)
+				}
+				log.Info("reported metrics")
+			}
+		case <-r.closer:
+			log.Info("stopping metrics reporter")
+			return
+		}
+	}
+}
+
+func (r *zitiMetricsReporter) createMetricsEvent() *mgmt_pb.StreamMetricsEvent {
+	event := &mgmt_pb.StreamMetricsEvent{
+		Timestamp:    ptypes.TimestampNow(),
+		SourceId:     r.clientId,
+		IntMetrics:   map[string]int64{},
+		FloatMetrics: map[string]float64{},
+		MetricGroup:  map[string]string{},
+	}
+
+	registry.Each(func(name string, i interface{}) {
+		switch metric := i.(type) {
+		case metrics.Gauge:
+			r.addIntMetric(event, name, "value", metric.Value())
+		case metrics.Meter:
+			r.addMeterToEvent(event, name, metric)
+		case metrics.Histogram:
+			r.addHistogramToEvent(event, name, metric)
+		case metrics.Timer:
+			r.addTimerToEvent(event, name, metric)
+		}
+	})
+
+	return event
+}
+
+func (r *zitiMetricsReporter) addMeterToEvent(event *mgmt_pb.StreamMetricsEvent, name string, metric metrics.Meter) {
+	r.addIntMetric(event, name, "count", metric.Count())
+	r.addFloatMetric(event, name, "mean_rate", metric.RateMean())
+	r.addFloatMetric(event, name, "m1_rate", metric.Rate1())
+	r.addFloatMetric(event, name, "m5_rate", metric.Rate5())
+	r.addFloatMetric(event, name, "m15_rate", metric.Rate15())
+}
+
+func (r *zitiMetricsReporter) addHistogramToEvent(event *mgmt_pb.StreamMetricsEvent, name string, metric metrics.Histogram) {
+	r.addIntMetric(event, name, "count", metric.Count())
+	r.addIntMetric(event, name, "min", metric.Min())
+	r.addIntMetric(event, name, "max", metric.Max())
+	r.addFloatMetric(event, name, "mean", metric.Mean())
+	r.addFloatMetric(event, name, "std_dev", metric.StdDev())
+	r.addFloatMetric(event, name, "variance", metric.Variance())
+	r.addFloatMetric(event, name, "p50", metric.Percentile(.50))
+	r.addFloatMetric(event, name, "p75", metric.Percentile(.75))
+	r.addFloatMetric(event, name, "p95", metric.Percentile(.95))
+	r.addFloatMetric(event, name, "p99", metric.Percentile(.99))
+	r.addFloatMetric(event, name, "p999", metric.Percentile(.999))
+	r.addFloatMetric(event, name, "p9999", metric.Percentile(.9999))
+}
+
+func (r *zitiMetricsReporter) addTimerToEvent(event *mgmt_pb.StreamMetricsEvent, name string, metric metrics.Timer) {
+	r.addIntMetric(event, name, "count", metric.Count())
+	r.addFloatMetric(event, name, "mean_rate", metric.RateMean())
+	r.addFloatMetric(event, name, "m1_rate", metric.Rate1())
+	r.addFloatMetric(event, name, "m5_rate", metric.Rate5())
+	r.addFloatMetric(event, name, "m15_rate", metric.Rate15())
+	r.addIntMetric(event, name, "min", metric.Min())
+	r.addIntMetric(event, name, "max", metric.Max())
+	r.addFloatMetric(event, name, "mean", metric.Mean())
+	r.addFloatMetric(event, name, "std_dev", metric.StdDev())
+	r.addFloatMetric(event, name, "variance", metric.Variance())
+	r.addFloatMetric(event, name, "p50", metric.Percentile(.50))
+	r.addFloatMetric(event, name, "p75", metric.Percentile(.75))
+	r.addFloatMetric(event, name, "p95", metric.Percentile(.95))
+	r.addFloatMetric(event, name, "p99", metric.Percentile(.99))
+	r.addFloatMetric(event, name, "p999", metric.Percentile(.999))
+	r.addFloatMetric(event, name, "p9999", metric.Percentile(.9999))
+}
+
+func (r *zitiMetricsReporter) addIntMetric(event *mgmt_pb.StreamMetricsEvent, base, name string, val int64) {
+	fullName := base + "." + name
+	event.IntMetrics[fullName] = val
+	event.MetricGroup[fullName] = base
+}
+
+func (r *zitiMetricsReporter) addFloatMetric(event *mgmt_pb.StreamMetricsEvent, base, name string, val float64) {
+	fullName := base + "." + name
+	event.FloatMetrics[fullName] = val
+	event.MetricGroup[fullName] = base
+}

ziti-fabric-test/subcmd/loop3/pb/loop.pb.go

@@ -1,205 +0,0 @@
-/*
-	Copyright NetFoundry, Inc.
-
-	Licensed under the Apache License, Version 2.0 (the "License");
-	you may not use this file except in compliance with the License.
-	You may obtain a copy of the License at
-
-	https://www.apache.org/licenses/LICENSE-2.0
-
-	Unless required by applicable law or agreed to in writing, software
-	distributed under the License is distributed on an "AS IS" BASIS,
-	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-	See the License for the specific language governing permissions and
-	limitations under the License.
-*/
-
-// Code generated by protoc-gen-go. DO NOT EDIT.
-// source: loop.proto
-
-package loop3_pb
-
-import (
-	fmt "fmt"
-	proto "github.com/golang/protobuf/proto"
-	math "math"
-)
-
-// Reference imports to suppress errors if they are not otherwise used.
-var _ = proto.Marshal
-var _ = fmt.Errorf
-var _ = math.Inf
-
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the proto package it is being compiled against.
-// A compilation error at this line likely means your copy of the
-// proto package needs to be updated.
-const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
-
-type Test struct {
-	Name                 string   `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
-	TxRequests           int32    `protobuf:"varint,2,opt,name=txRequests,proto3" json:"txRequests,omitempty"`
-	TxPacing             int32    `protobuf:"varint,3,opt,name=txPacing,proto3" json:"txPacing,omitempty"`
-	TxMaxJitter          int32    `protobuf:"varint,4,opt,name=txMaxJitter,proto3" json:"txMaxJitter,omitempty"`
-	RxRequests           int32    `protobuf:"varint,5,opt,name=rxRequests,proto3" json:"rxRequests,omitempty"`
-	RxTimeout            int32    `protobuf:"varint,6,opt,name=rxTimeout,proto3" json:"rxTimeout,omitempty"`
-	PayloadMinBytes      int32    `protobuf:"varint,7,opt,name=payloadMinBytes,proto3" json:"payloadMinBytes,omitempty"`
-	PayloadMaxBytes      int32    `protobuf:"varint,8,opt,name=payloadMaxBytes,proto3" json:"payloadMaxBytes,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
-}
-
-func (m *Test) Reset()         { *m = Test{} }
-func (m *Test) String() string { return proto.CompactTextString(m) }
-func (*Test) ProtoMessage()    {}
-func (*Test) Descriptor() ([]byte, []int) {
-	return fileDescriptor_b6a086c469bfcb2d, []int{0}
-}
-
-func (m *Test) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_Test.Unmarshal(m, b)
-}
-func (m *Test) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_Test.Marshal(b, m, deterministic)
-}
-func (m *Test) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_Test.Merge(m, src)
-}
-func (m *Test) XXX_Size() int {
-	return xxx_messageInfo_Test.Size(m)
-}
-func (m *Test) XXX_DiscardUnknown() {
-	xxx_messageInfo_Test.DiscardUnknown(m)
-}
-
-var xxx_messageInfo_Test proto.InternalMessageInfo
-
-func (m *Test) GetName() string {
-	if m != nil {
-		return m.Name
-	}
-	return ""
-}
-
-func (m *Test) GetTxRequests() int32 {
-	if m != nil {
-		return m.TxRequests
-	}
-	return 0
-}
-
-func (m *Test) GetTxPacing() int32 {
-	if m != nil {
-		return m.TxPacing
-	}
-	return 0
-}
-
-func (m *Test) GetTxMaxJitter() int32 {
-	if m != nil {
-		return m.TxMaxJitter
-	}
-	return 0
-}
-
-func (m *Test) GetRxRequests() int32 {
-	if m != nil {
-		return m.RxRequests
-	}
-	return 0
-}
-
-func (m *Test) GetRxTimeout() int32 {
-	if m != nil {
-		return m.RxTimeout
-	}
-	return 0
-}
-
-func (m *Test) GetPayloadMinBytes() int32 {
-	if m != nil {
-		return m.PayloadMinBytes
-	}
-	return 0
-}
-
-func (m *Test) GetPayloadMaxBytes() int32 {
-	if m != nil {
-		return m.PayloadMaxBytes
-	}
-	return 0
-}
-
-type Result struct {
-	Success              bool     `protobuf:"varint,1,opt,name=success,proto3" json:"success,omitempty"`
-	Message              string   `protobuf:"bytes,2,opt,name=message,proto3" json:"message,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
-}
-
-func (m *Result) Reset()         { *m = Result{} }
-func (m *Result) String() string { return proto.CompactTextString(m) }
-func (*Result) ProtoMessage()    {}
-func (*Result) Descriptor() ([]byte, []int) {
-	return fileDescriptor_b6a086c469bfcb2d, []int{1}
-}
-
-func (m *Result) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_Result.Unmarshal(m, b)
-}
-func (m *Result) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_Result.Marshal(b, m, deterministic)
-}
-func (m *Result) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_Result.Merge(m, src)
-}
-func (m *Result) XXX_Size() int {
-	return xxx_messageInfo_Result.Size(m)
-}
-func (m *Result) XXX_DiscardUnknown() {
-	xxx_messageInfo_Result.DiscardUnknown(m)
-}
-
-var xxx_messageInfo_Result proto.InternalMessageInfo
-
-func (m *Result) GetSuccess() bool {
-	if m != nil {
-		return m.Success
-	}
-	return false
-}
-
-func (m *Result) GetMessage() string {
-	if m != nil {
-		return m.Message
-	}
-	return ""
-}
-
-func init() {
-	proto.RegisterType((*Test)(nil), "loop3.pb.Test")
-	proto.RegisterType((*Result)(nil), "loop3.pb.Result")
-}
-
-func init() { proto.RegisterFile("loop.proto", fileDescriptor_b6a086c469bfcb2d) }
-
-var fileDescriptor_b6a086c469bfcb2d = []byte{
-	// 233 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x5c, 0x90, 0x31, 0x6b, 0xc3, 0x30,
-	0x10, 0x85, 0x49, 0xea, 0x38, 0xf6, 0x75, 0x28, 0xdc, 0x24, 0x4a, 0x29, 0x21, 0x53, 0xa6, 0x2c,
-	0x59, 0x3b, 0x75, 0x2c, 0x04, 0x8a, 0xc8, 0x1f, 0x50, 0xdc, 0x23, 0x18, 0x6c, 0x4b, 0x95, 0x4e,
-	0xa0, 0xfc, 0x85, 0xfe, 0xea, 0xda, 0x97, 0xba, 0x76, 0x3b, 0xdd, 0xbd, 0xef, 0x3d, 0xde, 0xf0,
-	0x00, 0x1a, 0x6b, 0xdd, 0xde, 0x79, 0xcb, 0x16, 0x8b, 0xe1, 0x3f, 0xec, 0xdd, 0x79, 0xfb, 0xb5,
-	0x84, 0xec, 0x44, 0x81, 0x11, 0x21, 0xeb, 0x4c, 0x4b, 0x6a, 0xb1, 0x59, 0xec, 0x4a, 0x2d, 0x3f,
-	0x3e, 0x03, 0x70, 0xd2, 0xf4, 0x19, 0xfb, 0x40, 0x50, 0xcb, 0xde, 0x59, 0xe9, 0x19, 0xc1, 0x47,
-	0x28, 0x38, 0xbd, 0x9b, 0xaa, 0xee, 0x2e, 0xea, 0x4e, 0xdc, 0x5f, 0x8d, 0x1b, 0xb8, 0xe7, 0x74,
-	0x34, 0xe9, 0xad, 0x66, 0x26, 0xaf, 0x32, 0xb1, 0xe7, 0x68, 0x68, 0xf7, 0x53, 0xfb, 0xea, 0xd6,
-	0x3e, 0x11, 0x7c, 0x82, 0xd2, 0xa7, 0x53, 0xdd, 0x92, 0x8d, 0xac, 0x72, 0xb1, 0x27, 0x80, 0x3b,
-	0x78, 0x70, 0xe6, 0xda, 0x58, 0xf3, 0x71, 0xac, 0xbb, 0xd7, 0x2b, 0x53, 0x50, 0x6b, 0xc9, 0xfc,
-	0xc7, 0xf3, 0xa4, 0x49, 0xb7, 0x64, 0xf1, 0x37, 0xf9, 0x83, 0xb7, 0x2f, 0x90, 0x6b, 0x0a, 0xb1,
-	0x61, 0x54, 0xb0, 0x0e, 0xb1, 0xaa, 0x28, 0x04, 0x19, 0xa4, 0xd0, 0xa3, 0x1c, 0x9c, 0xb6, 0xbf,
-	0xe6, 0x42, 0x32, 0x48, 0xa9, 0x47, 0x79, 0xce, 0x65, 0xdb, 0xc3, 0x77, 0x00, 0x00, 0x00, 0xff,
-	0xff, 0xa7, 0xa6, 0xb7, 0x1f, 0x69, 0x01, 0x00, 0x00,
-}

ziti-fabric-test/subcmd/loop3/pb/loop3.pb.go

@@ -0,0 +1,150 @@
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// source: loop3.proto
+
+package loop3_pb
+
+import (
+	fmt "fmt"
+	proto "github.com/golang/protobuf/proto"
+	math "math"
+)
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the proto package it is being compiled against.
+// A compilation error at this line likely means your copy of the
+// proto package needs to be updated.
+const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
+
+type Test struct {
+	Name                 string   `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
+	TxRequests           int32    `protobuf:"varint,2,opt,name=txRequests,proto3" json:"txRequests,omitempty"`
+	TxPacing             int32    `protobuf:"varint,3,opt,name=txPacing,proto3" json:"txPacing,omitempty"`
+	TxMaxJitter          int32    `protobuf:"varint,4,opt,name=txMaxJitter,proto3" json:"txMaxJitter,omitempty"`
+	RxRequests           int32    `protobuf:"varint,5,opt,name=rxRequests,proto3" json:"rxRequests,omitempty"`
+	RxTimeout            int32    `protobuf:"varint,6,opt,name=rxTimeout,proto3" json:"rxTimeout,omitempty"`
+	PayloadMinBytes      int32    `protobuf:"varint,7,opt,name=payloadMinBytes,proto3" json:"payloadMinBytes,omitempty"`
+	PayloadMaxBytes      int32    `protobuf:"varint,8,opt,name=payloadMaxBytes,proto3" json:"payloadMaxBytes,omitempty"`
+	LatencyFrequency     int32    `protobuf:"varint,9,opt,name=latencyFrequency,proto3" json:"latencyFrequency,omitempty"`
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *Test) Reset()         { *m = Test{} }
+func (m *Test) String() string { return proto.CompactTextString(m) }
+func (*Test) ProtoMessage()    {}
+func (*Test) Descriptor() ([]byte, []int) {
+	return fileDescriptor_fb557b01f1ee4776, []int{0}
+}
+
+func (m *Test) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_Test.Unmarshal(m, b)
+}
+func (m *Test) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_Test.Marshal(b, m, deterministic)
+}
+func (m *Test) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_Test.Merge(m, src)
+}
+func (m *Test) XXX_Size() int {
+	return xxx_messageInfo_Test.Size(m)
+}
+func (m *Test) XXX_DiscardUnknown() {
+	xxx_messageInfo_Test.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_Test proto.InternalMessageInfo
+
+func (m *Test) GetName() string {
+	if m != nil {
+		return m.Name
+	}
+	return ""
+}
+
+func (m *Test) GetTxRequests() int32 {
+	if m != nil {
+		return m.TxRequests
+	}
+	return 0
+}
+
+func (m *Test) GetTxPacing() int32 {
+	if m != nil {
+		return m.TxPacing
+	}
+	return 0
+}
+
+func (m *Test) GetTxMaxJitter() int32 {
+	if m != nil {
+		return m.TxMaxJitter
+	}
+	return 0
+}
+
+func (m *Test) GetRxRequests() int32 {
+	if m != nil {
+		return m.RxRequests
+	}
+	return 0
+}
+
+func (m *Test) GetRxTimeout() int32 {
+	if m != nil {
+		return m.RxTimeout
+	}
+	return 0
+}
+
+func (m *Test) GetPayloadMinBytes() int32 {
+	if m != nil {
+		return m.PayloadMinBytes
+	}
+	return 0
+}
+
+func (m *Test) GetPayloadMaxBytes() int32 {
+	if m != nil {
+		return m.PayloadMaxBytes
+	}
+	return 0
+}
+
+func (m *Test) GetLatencyFrequency() int32 {
+	if m != nil {
+		return m.LatencyFrequency
+	}
+	return 0
+}
+
+func init() {
+	proto.RegisterType((*Test)(nil), "loop3.pb.Test")
+}
+
+func init() {
+	proto.RegisterFile("loop3.proto", fileDescriptor_fb557b01f1ee4776)
+}
+
+var fileDescriptor_fb557b01f1ee4776 = []byte{
+	// 217 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x64, 0x90, 0xc1, 0x4a, 0x86, 0x40,
+	0x14, 0x85, 0xd1, 0xd4, 0xf4, 0xba, 0x28, 0xee, 0x6a, 0x88, 0x08, 0x69, 0x25, 0x2d, 0xda, 0xf8,
+	0x06, 0x2d, 0x5a, 0x04, 0x42, 0x88, 0x2f, 0x30, 0xda, 0x10, 0x03, 0x3a, 0x63, 0xe3, 0x15, 0xc6,
+	0x17, 0xeb, 0xf9, 0xd2, 0x51, 0xd2, 0xfa, 0x77, 0xf7, 0x7c, 0xe7, 0xe3, 0x30, 0x0c, 0xa4, 0x9d,
+	0xd6, 0x43, 0xf1, 0x3c, 0x18, 0x4d, 0x1a, 0xe3, 0x3d, 0x34, 0x8f, 0xdf, 0x3e, 0x04, 0xb5, 0x18,
+	0x09, 0x11, 0x02, 0xc5, 0x7b, 0xc1, 0xbc, 0xcc, 0xcb, 0x93, 0xca, 0xdd, 0xf8, 0x00, 0x40, 0xb6,
+	0x12, 0x5f, 0xd3, 0x22, 0x8c, 0xcc, 0x5f, 0x9a, 0xb0, 0x3a, 0x11, 0xbc, 0x83, 0x98, 0xec, 0x3b,
+	0x6f, 0xa5, 0xfa, 0x64, 0x57, 0xae, 0xfd, 0xcd, 0x98, 0x41, 0x4a, 0xb6, 0xe4, 0xf6, 0x4d, 0x12,
+	0x09, 0xc3, 0x02, 0x57, 0x9f, 0xd1, 0xba, 0x6e, 0x8e, 0xf5, 0x70, 0x5b, 0x3f, 0x08, 0xde, 0x43,
+	0x62, 0x6c, 0x2d, 0x7b, 0xa1, 0x27, 0x62, 0x91, 0xab, 0x0f, 0x80, 0x39, 0xdc, 0x0c, 0x7c, 0xee,
+	0x34, 0xff, 0x28, 0xa5, 0x7a, 0x99, 0x49, 0x8c, 0xec, 0xda, 0x39, 0xff, 0xf1, 0xd9, 0xe4, 0x76,
+	0x33, 0xe3, 0xbf, 0xe6, 0x8e, 0xf1, 0x09, 0x6e, 0x3b, 0x4e, 0x42, 0xb5, 0xf3, 0xab, 0x59, 0x5f,
+	0xb1, 0x1c, 0x2c, 0x71, 0xea, 0x05, 0x6f, 0x22, 0xf7, 0x93, 0xc5, 0x4f, 0x00, 0x00, 0x00, 0xff,
+	0xff, 0x39, 0x97, 0xca, 0x77, 0x58, 0x01, 0x00, 0x00,
+}

ziti-fabric-test/subcmd/loop3/pb/loop3.proto

@@ -11,9 +11,5 @@ message Test {
     int32 rxTimeout = 6;
     int32 payloadMinBytes = 7;
     int32 payloadMaxBytes = 8;
-}
-
-message Result {
-    bool success = 1;
-    string message = 2;
+    int32 latencyFrequency = 9;
 }

ziti-fabric-test/subcmd/loop3/protocol.go

@@ -17,8 +17,6 @@
 package loop3
 
 import (
-	"github.com/openziti/ziti/ziti-fabric-test/subcmd/loop3/pb"
-	"github.com/openziti/foundation/util/info"
 	"bytes"
 	"crypto/sha512"
 	"encoding/binary"
@@ -26,70 +24,35 @@ import (
 	"fmt"
 	"github.com/golang/protobuf/proto"
 	"github.com/michaelquigley/pfxlog"
+	"github.com/openziti/ziti/ziti-fabric-test/subcmd/loop3/pb"
 	"github.com/pkg/errors"
-	"github.com/rcrowley/go-metrics"
 	"io"
 	"math/rand"
 	"time"
 )
 
-type Block struct {
-	Sequence int32  `protobuf:"varint,1,opt,name=sequence,proto3" json:"sequence,omitempty"`
-	Data     []byte `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"`
-	Hash     []byte `protobuf:"bytes,3,opt,name=hash,proto3" json:"hash,omitempty"`
-}
-
 type protocol struct {
 	test        *loop3_pb.Test
 	txGenerator *generator
-	rxSequence  int32
+	rxSequence  uint32
 	peer        io.ReadWriteCloser
 	rxBlocks    chan *Block
 	txCount     int32
 	rxCount     int32
+	latencies   chan *time.Time
 	errors      chan error
 }
 
-var rxRate = metrics.NewMeter()
-var txRate = metrics.NewMeter()
-
-func init() {
-	log := pfxlog.Logger()
-	if err := metrics.Register("rxRate", rxRate); err != nil {
-		log.WithError(err).Error("failed to register rx rate metrics")
-	}
-	if err := metrics.Register("txRate", txRate); err != nil {
-		log.WithError(err).Error("failed to register tx rate metrics")
-	}
-
-	go func() {
-		mbs := float64(1024 * 1024)
-		for range time.Tick(time.Second * 5) {
-			metrics.Each(func(name string, i interface{}) {
-				switch metric := i.(type) {
-				case metrics.Meter:
-					if metric.Rate1() > mbs {
-						log.Infof("%v - count: %v, 1m: %.2f MB/s, 5m: %.2f MB/s, avg: %.2f, MB/s",
-							name, metric.Count(), metric.Rate1()/mbs, metric.Rate5()/mbs, metric.RateMean()/mbs)
-					} else {
-						log.Infof("%v - count: %v, 1m: %.2f KB/s, 5m: %.2f KB/s, avg: %.2f /KB/s",
-							name, metric.Count(), metric.Rate1()/1024, metric.Rate5()/1024, metric.RateMean()/1024)
-					}
-				}
-			})
-		}
-	}()
-}
-
 var MagicHeader = []byte{0xCA, 0xFE, 0xF0, 0x0D}
 
 func newProtocol(peer io.ReadWriteCloser) (*protocol, error) {
 	p := &protocol{
 		rxSequence: 0,
 		peer:       peer,
-		rxBlocks:   make(chan *Block, 128),
+		rxBlocks:   make(chan *Block),
 		txCount:    0,
 		rxCount:    0,
+		latencies:  make(chan *time.Time, 1024),
 		errors:     make(chan error, 10240),
 	}
 	return p, nil
@@ -97,7 +60,7 @@ func newProtocol(peer io.ReadWriteCloser) (*protocol, error) {
 
 func (p *protocol) run(test *loop3_pb.Test) error {
 	p.test = test
-	p.txGenerator = newGenerator(int(test.TxRequests), int(test.PayloadMinBytes), int(test.PayloadMaxBytes))
+	p.txGenerator = newGenerator(int(test.TxRequests), int(test.PayloadMinBytes), int(test.PayloadMaxBytes), int(test.LatencyFrequency))
 	go p.txGenerator.run()
 
 	rxerDone := make(chan bool)
@@ -126,13 +89,28 @@ func (p *protocol) txer(done chan bool) {
 	defer func() { done <- true }()
 	defer log.Debug("complete")
 
+	var latency *time.Time
+
 	for p.txCount < p.test.TxRequests {
 		select {
 		case block := <-p.txGenerator.blocks:
 			if block != nil {
-				if err := p.txBlock(block); err == nil {
-					p.txCount++
 
+				if block.Type == BlockTypePlain {
+					select {
+					case latency = <-p.latencies:
+					default:
+						latency = nil
+					}
+
+					if latency != nil {
+						block.Type = BlockTypeLatencyResponse
+						block.Timestamp = *latency
+					}
+				}
+
+				if err := block.Tx(p); err == nil {
+					p.txCount++
 					if p.test.TxPacing > 0 {
 						jitter := 0
 						if p.test.TxMaxJitter > 0 {
@@ -169,6 +147,15 @@ func (p *protocol) rxer(done chan bool) {
 			log.Error(err)
 			return
 		}
+
+		if block.Type == BlockTypeLatencyRequest {
+			select {
+			case p.latencies <- &block.Timestamp:
+			default:
+				log.Warn("latency channel out of room")
+			}
+		}
+
 		p.rxBlocks <- block
 		p.rxCount++
 	}
@@ -243,112 +230,50 @@ func (p *protocol) rxTest() (*loop3_pb.Test, error) {
 }
 
 func (p *protocol) rxBlock() (*Block, error) {
-	if err := p.rxMagicHeader(); err != nil {
+	block := &Block{}
+	if err := block.Rx(p); err != nil {
 		return nil, err
 	}
-	seq, err := p.rxInt32()
-	if err != nil {
-		return nil, err
-	}
-
-	data, err := p.rxByteArray()
-	if err != nil {
-		return nil, err
-	}
-
-	hash, err := p.rxByteArray()
-	if err != nil {
-		return nil, err
-	}
-
-	block := &Block{
-		Sequence: seq,
-		Data:     data,
-		Hash:     hash,
-	}
-	pfxlog.ContextLogger(p.test.Name).Debugf("<- #%d (%s)", block.Sequence, info.ByteCount(int64(len(block.Data))))
-
-	// header (4) + seq (4) + data (4 + data len) + hash (4 + hash len)
-	rxRate.Mark(int64(16 + len(block.Data) + len(block.Hash)))
-
 	return block, nil
 }
 
-func (p *protocol) txResult(result *loop3_pb.Result) error {
-	if err := p.txPb(result); err != nil {
-		return err
-	}
-	if result.Success {
-		pfxlog.ContextLogger(p.test.Name).Infof("<- [result+]")
-	} else {
-		pfxlog.ContextLogger(p.test.Name).Infof("<- [result-]")
-	}
-	return nil
-}
-
-func (p *protocol) rxResult() (*loop3_pb.Result, error) {
-	result := &loop3_pb.Result{}
-	if err := p.rxPb(result); err != nil {
+func (p *protocol) rxResult() (*Result, error) {
+	result := &Result{}
+	if err := result.Rx(p); err != nil {
 		return nil, err
 	}
-	if result.Success {
-		pfxlog.ContextLogger(p.test.Name).Infof("<- [result+]")
-	} else {
-		pfxlog.ContextLogger(p.test.Name).Infof("<- [result-]")
-	}
 	return result, nil
 }
 
-func (p *protocol) txBlock(block *Block) error {
-	if err := p.txMagicHeader(); err != nil {
-		return err
-	}
-	if err := p.txInt32(block.Sequence); err != nil {
-		return err
-	}
-	if err := p.txByteArray(block.Data); err != nil {
-		return err
-	}
-	if err := p.txByteArray(block.Hash); err != nil {
-		return err
-	}
-
-	// header (4) + seq (4) + data (4 + data len) + hash (4 + hash len)
-	txRate.Mark(int64(16 + len(block.Data) + len(block.Hash)))
-
-	pfxlog.ContextLogger(p.test.Name).Debugf("-> #%d (%s)", block.Sequence, info.ByteCount(int64(len(block.Data))))
-	return nil
-}
-
 func (p *protocol) txPb(pb proto.Message) error {
 	data, err := proto.Marshal(pb)
+
 	if err != nil {
 		return err
 	}
 	if err = p.txMagicHeader(); err != nil {
 		return err
 	}
-	return p.txByteArray(data)
+	if err := p.txLength(len(data)); err != nil {
+		return err
+	}
+	n, err := p.peer.Write(data)
+	if err != nil {
+		return err
+	}
+	if n != len(data) {
+		return errors.New("short data write")
+	}
+	return nil
 }
 
 func (p *protocol) rxPb(pb proto.Message) error {
 	if err := p.rxMagicHeader(); err != nil {
 		return err
 	}
-	data, err := p.rxByteArray()
-	if err != nil {
-		return err
-	}
-	if err := proto.Unmarshal(data, pb); err != nil {
-		return err
-	}
-	return nil
-}
-
-func (p *protocol) rxByteArray() ([]byte, error) {
 	length, err := p.rxLength()
 	if err != nil {
-		return nil, err
+		return err
 	}
 
 	defer func() {
@@ -361,12 +286,15 @@ func (p *protocol) rxByteArray() ([]byte, error) {
 	data := make([]byte, length)
 	n, err := io.ReadFull(p.peer, data)
 	if err != nil {
-		return nil, err
+		return err
 	}
 	if n != length {
-		return nil, fmt.Errorf("short data read [%d != %d]", n, length)
+		return fmt.Errorf("short data read [%d != %d]", n, length)
 	}
-	return data, nil
+	if err := proto.Unmarshal(data, pb); err != nil {
+		return err
+	}
+	return nil
 }
 
 func (p *protocol) txMagicHeader() error {
@@ -380,47 +308,31 @@ func (p *protocol) txMagicHeader() error {
 	return nil
 }
 
-func (p *protocol) txByteArray(bytes []byte) error {
-	dataLen := len(bytes)
-	if err := p.txLength(dataLen); err != nil {
-		return err
-	}
-
-	n, err := p.peer.Write(bytes)
+func (p *protocol) txLength(length int) error {
+	out := make([]byte, 4)
+	binary.LittleEndian.PutUint32(out, uint32(length))
+	n, err := p.peer.Write(out)
 	if err != nil {
 		return err
 	}
-	if n != dataLen {
+	if n != 4 {
 		return errors.New("short length write")
 	}
 	return nil
 }
 
-func (p *protocol) txLength(len int) error {
-	return p.txInt32(int32(len))
-}
+func (p *protocol) txHeader(msgLen int) error {
+	if err := p.txMagicHeader(); err != nil {
+		return err
+	}
 
-func (p *protocol) txInt32(len int32) error {
-	out := new(bytes.Buffer)
-	if err := binary.Write(out, binary.LittleEndian, len); err != nil {
+	if err := p.txLength(msgLen); err != nil {
 		return err
 	}
-	n, err := p.peer.Write(out.Bytes())
-	if err != nil {
-		return err
-	}
-	if n != out.Len() {
-		return errors.New("short length write")
-	}
 	return nil
 }
 
 func (p *protocol) rxLength() (int, error) {
-	result, err := p.rxInt32()
-	return int(result), err
-}
-
-func (p *protocol) rxInt32() (int32, error) {
 	data := make([]byte, 4)
 	n, err := io.ReadFull(p.peer, data)
 	if err != nil {
@@ -435,7 +347,7 @@ func (p *protocol) rxInt32() (int32, error) {
 	if err != nil {
 		return -1, err
 	}
-	return length, nil
+	return int(length), nil
 }
 
 func (p *protocol) rxMagicHeader() error {
@@ -452,3 +364,19 @@ func (p *protocol) rxMagicHeader() error {
 	}
 	return nil
 }
+
+func (p *protocol) rxHeader() (int, error) {
+	if err := p.rxMagicHeader(); err != nil {
+		return 0, err
+	}
+	return p.rxLength()
+}
+
+func (p *protocol) rxMsgBody(length int) ([]byte, error) {
+	data := make([]byte, length)
+	_, err := io.ReadFull(p.peer, data)
+	if err != nil {
+		return nil, err
+	}
+	return data, err
+}

ziti-fabric-test/subcmd/loop3/scenario.go

@@ -19,11 +19,13 @@ package loop3
 import (
 	"gopkg.in/yaml.v2"
 	"io/ioutil"
+	"time"
 )
 
 type Scenario struct {
 	Workloads       []*Workload `yaml:"workloads"`
 	ConnectionDelay int32       `yaml:"connectionDelay"`
+	Metrics         *Metrics    `yaml:"metrics"`
 }
 
 type Workload struct {
@@ -34,12 +36,19 @@ type Workload struct {
 }
 
 type Test struct {
-	TxRequests      int32 `yaml:"txRequests"`
-	TxPacing        int32 `yaml:"txPacing"`
-	TxMaxJitter     int32 `yaml:"txMaxJitter"`
-	RxTimeout       int32 `yaml:"rxTimeout"`
-	PayloadMinBytes int32 `yaml:"payloadMinBytes"`
-	PayloadMaxBytes int32 `yaml:"payloadMaxBytes"`
+	TxRequests       int32 `yaml:"txRequests"`
+	TxPacing         int32 `yaml:"txPacing"`
+	TxMaxJitter      int32 `yaml:"txMaxJitter"`
+	RxTimeout        int32 `yaml:"rxTimeout"`
+	PayloadMinBytes  int32 `yaml:"payloadMinBytes"`
+	PayloadMaxBytes  int32 `yaml:"payloadMaxBytes"`
+	LatencyFrequency int32 `yaml:"latencyFrequency"`
+}
+
+type Metrics struct {
+	Service        string        `yaml:"service"`
+	ReportInterval time.Duration `yaml:"interval"`
+	ClientId       string        `yaml:"clientId"`
 }
 
 func LoadScenario(path string) (*Scenario, error) {

ziti-tunnel/cmd/ziti-tunnel/subcmd/root.go

@@ -34,6 +34,7 @@ import (
 const (
 	svcPollRateFlag = "svcPollRate"
 	resolverCfgFlag = "resolver"
+	dnsSvcIpRangeFlag = "dnsSvcIpRange"
 )
 
 func init() {
@@ -42,6 +43,7 @@ func init() {
 	root.PersistentFlags().Uint(svcPollRateFlag, 15, "Set poll rate for service updates (seconds)")
 	root.PersistentFlags().StringP(resolverCfgFlag, "r", "udp://127.0.0.1:53", "Resolver configuration")
 	root.PersistentFlags().StringVar(&logFormatter, "log-formatter", "", "Specify log formatter [json|pfxlog|text]")
+	root.PersistentFlags().StringP(dnsSvcIpRangeFlag, "d", "100.64.0.1/10", "cidr to use when assigning IPs to unresolvable intercept hostnames")
 
 	root.AddCommand(enrollment.NewEnrollCommand())
 }
@@ -96,13 +98,14 @@ func rootPostRun(cmd *cobra.Command, _ []string) {
 		entities.ServerConfigV1,
 	}
 	rootPrivateContext := ziti.NewContextWithConfig(zitiCfg)
-	if err != nil {
-		log.Fatalf("failed to initialize Ziti SDK: %v", err)
-	}
 
 	svcPollRate, _ := cmd.Flags().GetUint(svcPollRateFlag)
 	resolverConfig := cmd.Flag("resolver").Value.String()
 	resolver = dns.NewResolver(resolverConfig)
+	dnsIpRange, _ := cmd.Flags().GetString(dnsSvcIpRangeFlag)
+	err = intercept.SetDnsInterceptIpRange(dnsIpRange); if err != nil {
+		log.Fatalf("invalid dns service IP range %s: %v", dnsIpRange, err)
+	}
 
 	intercept.ServicePoller(rootPrivateContext, interceptor, resolver, time.Duration(svcPollRate)*time.Second)
 }

ziti/cmd/ziti/cmd/edge_controller/create_edge_router_policy.go

@@ -31,6 +31,7 @@ type createEdgeRouterPolicyOptions struct {
 	commonOptions
 	edgeRouterRoles []string
 	identityRoles   []string
+	semantic        string
 }
 
 // newCreateEdgeRouterPolicyCmd creates the 'edge controller create edge-router-policy' command
@@ -59,6 +60,7 @@ func newCreateEdgeRouterPolicyCmd(f cmdutil.Factory, out io.Writer, errOut io.Wr
 	cmd.Flags().SetInterspersed(true)
 	cmd.Flags().StringSliceVarP(&options.edgeRouterRoles, "edge-router-roles", "e", nil, "Edge router roles of the new edge router policy")
 	cmd.Flags().StringSliceVarP(&options.identityRoles, "identity-roles", "i", nil, "Identity roles of the new edge router policy")
+	cmd.Flags().StringVar(&options.semantic, "semantic", "", "Semantic dictating how multiple attributes should be interpreted. Valid values: AnyOf, AllOf")
 	options.AddCommonFlags(cmd)
 
 	return cmd
@@ -80,6 +82,10 @@ func runCreateEdgeRouterPolicy(o *createEdgeRouterPolicyOptions) error {
 	setJSONValue(entityData, o.Args[0], "name")
 	setJSONValue(entityData, edgeRouterRoles, "edgeRouterRoles")
 	setJSONValue(entityData, identityRoles, "identityRoles")
+	if o.semantic != "" {
+		setJSONValue(entityData, o.semantic, "semantic")
+	}
+
 	result, err := createEntityOfType("edge-router-policies", entityData.String(), &o.commonOptions)
 
 	if err != nil {

ziti/cmd/ziti/cmd/edge_controller/create_service_edge_router_policy.go

@@ -31,6 +31,7 @@ type createServiceEdgeRouterPolicyOptions struct {
 	commonOptions
 	edgeRouterRoles []string
 	serviceRoles    []string
+	semantic        string
 }
 
 // newCreateServiceEdgeRouterPolicyCmd creates the 'edge controller create service-edge-router-policy' command
@@ -59,6 +60,7 @@ func newCreateServiceEdgeRouterPolicyCmd(f cmdutil.Factory, out io.Writer, errOu
 	cmd.Flags().SetInterspersed(true)
 	cmd.Flags().StringSliceVarP(&options.edgeRouterRoles, "edge-router-roles", "e", nil, "Edge router roles of the new service edge router policy")
 	cmd.Flags().StringSliceVarP(&options.serviceRoles, "service-roles", "s", nil, "Identity roles of the new service edge router policy")
+	cmd.Flags().StringVar(&options.semantic, "semantic", "", "Semantic dictating how multiple attributes should be interpreted. Valid values: AnyOf, AllOf")
 	options.AddCommonFlags(cmd)
 
 	return cmd
@@ -79,6 +81,10 @@ func runCreateServiceEdgeRouterPolicy(o *createServiceEdgeRouterPolicyOptions) e
 	setJSONValue(entityData, o.Args[0], "name")
 	setJSONValue(entityData, edgeRouterRoles, "edgeRouterRoles")
 	setJSONValue(entityData, serviceRoles, "serviceRoles")
+	if o.semantic != "" {
+		setJSONValue(entityData, o.semantic, "semantic")
+	}
+
 	result, err := createEntityOfType("service-edge-router-policies", entityData.String(), &o.commonOptions)
 
 	if err != nil {

ziti/cmd/ziti/cmd/edge_controller/create_service_policy.go

@@ -34,6 +34,7 @@ type createServicePolicyOptions struct {
 	commonOptions
 	serviceRoles  []string
 	identityRoles []string
+	semantic      string
 }
 
 // newCreateServicePolicyCmd creates the 'edge controller create service-policy' command
@@ -62,6 +63,7 @@ func newCreateServicePolicyCmd(f cmdutil.Factory, out io.Writer, errOut io.Write
 	cmd.Flags().SetInterspersed(true)
 	cmd.Flags().StringSliceVarP(&options.serviceRoles, "service-roles", "s", nil, "Service roles of the new service policy")
 	cmd.Flags().StringSliceVarP(&options.identityRoles, "identity-roles", "i", nil, "Identity roles of the new service policy")
+	cmd.Flags().StringVar(&options.semantic, "semantic", "", "Semantic dictating how multiple attributes should be interpreted. Valid values: AnyOf, AllOf")
 	options.AddCommonFlags(cmd)
 
 	return cmd
@@ -89,6 +91,9 @@ func runCreateServicePolicy(o *createServicePolicyOptions) error {
 	setJSONValue(entityData, o.Args[1], "type")
 	setJSONValue(entityData, serviceRoles, "serviceRoles")
 	setJSONValue(entityData, identityRoles, "identityRoles")
+	if o.semantic != "" {
+		setJSONValue(entityData, o.semantic, "semantic")
+	}
 	result, err := createEntityOfType("service-policies", entityData.String(), &o.commonOptions)
 
 	if err != nil {

ziti/cmd/ziti/cmd/edge_controller/delete.go

@@ -96,7 +96,7 @@ func newDeleteCmdForEntityType(entityType string, command deleteCmdRunner, optio
 func runDeleteEntityOfType(o *commonOptions, entityType string) error {
 	var err error
 	ids := []string{o.Args[0]}
-	if entityType != "terminators" {
+	if entityType != "terminators" && entityType != "api-sessions" && entityType != "sessions" {
 		ids, err = mapNamesToIDs(entityType, o.Args[0])
 	}
 	if err != nil {