Merge pull request #350 from openziti/release-next

Release 0.19.8
2026-10-12 23:32:02 +00:00 · 2021-03-26 09:47:55 -04:00
parent 639471a3bf df0740b258
commit 981dbe05ea
30 changed files with 1079 additions and 174 deletions
@@ -1,3 +1,271 @@
+# Release 0.19.8
+
+## What's New
+
+* Transwarp beta_1
+* Converged Tunneler/Router (Beta)
+    * Updates to the host.v1 config type
+    * New host.v2 config type
+    * New health events
+
+## Fixes
+
+* Service Policies with no Posture Checks properly grant access even if an identity has another Service Policies with posture checks
+* Removing entities with no referencing entities via attributes no longer panics
+* API Sessions and Current Api Session now share the same modeling logic
+* API Sessions now have lastActivityAt and cachedLastActivityAt
+* Enrolling and unenrolling in MFA sets MFA posture data
+
+## Transwarp beta_1
+
+See the Transwarp beta_1 guide [here](doc/transwarp_b1/transwarp_b1.md).
+
+## Converged Tunneler/Router (Beta)
+
+ziti-router can now run with the tunneler embedded. It has the same capabilities as ziti-tunnel. As ziti-tunnel gains new features, the combined ziti-router/tunnel should maintain feature parity.
+
+### Beta Status
+
+This is a beta release. It should be relatively feature complete and bug-free for hosting services but intercept support is still nascent. Some features are likely to be changed based on user feedback.
+
+### Os Compatibilty
+
+Like ziti-tunnel, `tproxy` mode will only work on linux. `proxy` and `host` modes should work on all operating systems. 
+
+The converged tunneler/router does not support running in tun mode. This mode may be deprecated for ziti-tunnel as well at some point in the future if no advantages over tproxy mode are evident.
+
+### Supported configurations
+
+The following configurations are supported:
+
+* `ziti-tunneler-client.v1`
+* `ziti-tunneler-server.v1`
+* `host.v1` (updated)
+* `host.v2` (new)
+
+Support for `intercept.v1` will be added in a follow-up release.
+
+### Router Identities
+
+When an edge router is marked as being tunneler enabled, a matching identity will be created, of type Router, as well as an edge router policy. The edge router policy ensures that the identity always
+has access to the edge router. The identity allows the router to be included in service policies, to configure which services will intercepted/hosted.
+
+1. The identity will have the same id and name as the edge router
+1. If an identity with the same name as the router already exists, the router create/update will fail
+1. When the router name is changed, the identity name will be updated as well.
+1. The identity name and type cannot be changed directly. The type may not be changed at all and the name may only be changed by changing the name of the router.
+1. The identity may not be deleted except by deleting the router or disabling tunneler support for the identity.
+1. When the router is deleted, the accompanying identity and edge router policy will also be deleted
+1. If tunneler support is disabled in the router, the accompanying identity and edge router policy will also be deleted.
+1. The edge router policy will have the same id as the router and have a name of the form `edge-router-<edge-router-id>-system`, where `<edge-router-id>` is replaced by the id of the edge router.
+1. The edge router policy is considered a `system` entity, and cannot be updated and cannot deleted except by the system when the associated router is deleted.
+
+### Tunneler Prerequisites
+
+In order for a router instance to host a tunneler, it must meet the following criteria:
+
+1. It must be represented in the model by an edge router
+2. The edge router field `isTunnelerEnabled` must be set to true
+3. Edge functionality in the router must be enabled, which means the `edge:` config section must be present. NOTE: The edge listener does **not** need to be enabled.
+4. The tunnel listener must be enabled.
+
+### Making an Edge Router Tunneler enabled
+
+The ziti CLI can be used to enable/disable tunneler support on edge routers. When creating an edge router, the `-t` flag can be passed in to enable running the tunneler.
+
+```shell
+ziti edge create edge-router myEdgeRouter --tunneler-enabled
+```
+
+or
+
+```shell
+ziti edge create edge-router myEdgeRouter -t
+```
+
+An existing edge router can be marked as tunneler enabled as follows:
+
+```shell
+ziti edge update edge-router myEdgeRouter --tunneler-enabled
+```
+
+or
+
+```shell
+ziti edge update edge-router myEdgeRouter -t
+```
+
+An existing edge router can be marked as not supporting the tunneler as follows:
+
+```shell
+ziti edge update edge-router myEdgeRouter -t=false
+```
+
+or
+
+```shell
+ziti edge update edge-router myEdgeRouter --tunneler-enabled=false
+```
+
+### Tunnel listener configuration
+
+```yaml
+
+listeners:
+  - binding: tunnel
+    options:
+      mode: tproxy # mode to run in. Valid values [tproxy, host, proxy]. Default: tproxy
+      svcPollRate: 15s # How often to poll for service changes. Default: 15s
+      resolver: udp://127.0.0.1:53 # DNS resolve. Default: udp://127.0.0.1:53 for tproxy, blank for others
+      dnsSvcIpRange: 100.64.0.1/10 # cidr to use when assigning IPs to unresolvable intercept hostnames (default "100.64.0.1/10")
+      services: # services to intercept in proxy mode. Default: none
+        - echo:1977
+      lanIf: tun1 # if specified, INPUT rules for intercepted service addresses are assigned to this interface. Defaults to unspecified.
+```
+
+### Updates to health checks/health check events
+
+Health checks now have a new trigger option `change`. This will trigger the action whenever the health state changes from pass to fail or fail to pass.
+
+There's now a new action, `send event`, which will send a health event to the controller. This will be reported via the recently introduced service events.
+
+The `change` option is designed to be sued with `send event` as a way to send events only when the state changes, rather than sending every single pass/fail.
+
+An example configuration:
+
+```json
+
+{
+  "address": "localhost",
+  "port": 8171,
+  "protocol": "tcp",
+  "portChecks": [
+    {
+      "address": "localhost:8171",
+      "interval": "5s",
+      "timeout": "100ms",
+      "actions": [
+        {
+          "trigger": "change",
+          "action": "send event"
+        }
+      ]
+    }
+  ]
+}
+```
+
+What the service events look like:
+
+```json
+{
+  "namespace": "service.events",
+  "event_type": "service.health_check.failed",
+  "service_id": "vH3QndzRYt",
+  "count": 2,
+  "interval_start_utc": 1616703360,
+  "interval_length": 60
+}
+{
+  "namespace": "service.events",
+  "event_type": "service.health_check.failed",
+  "service_id": "vH3QndzRYt",
+  "count": 1,
+  "interval_start_utc": 1616703720,
+  "interval_length": 60
+}
+{
+  "namespace": "service.events",
+  "event_type": "service.health_check.passed",
+  "service_id": "vH3QndzRYt",
+  "count": 2,
+  "interval_start_utc": 1616703720,
+  "interval_length": 60
+}
+```
+
+### Changes to host.v1
+
+The host.v1 config type now support health checks. The configuration is the same for `ziti-tunneler-server-v1`.
+
+See here: https://github.com/openziti/edge/blob/v0.19.54/tunnel/entities/host.v1.json for the full schema
+
+### host.v2
+
+There is a new host configuration, `host.v2`, which should supercede `host.v1`. The primary difference from `host.v1` is that it supports multiple terminators. This means that when a service is hosted by a router, it can connect to multiple service instances if the service is horizontally scaled, or in a primary/failover setup.
+
+Each terminator definition is the same as a `host.v1` configuration, which one exception: `listOptions.connectTimeoutSeconds` is now `listenOptions.connectTimeout` and is specified as a duration (`5s`, `2500ms`, etc).
+
+Example:
+
+```json
+{
+  "terminators": [
+    {
+      "address": "localhost",
+      "port": 8171,
+      "protocol": "tcp",
+      "listenOptions": {
+        "cost": 50,
+        "precedence": "required"
+      },
+      "portChecks": [
+        {
+          "address": "localhost:8171",
+          "interval": "5s",
+          "timeout": "100ms",
+          "actions": [
+            {
+              "trigger": "change",
+              "action": "send event"
+            },
+            {
+              "trigger": "fail",
+              "action": "increase cost 25"
+            },
+            {
+              "trigger": "pass",
+              "action": "decrease cost 10"
+            }
+          ]
+        }
+      ]
+    },
+    {
+      "address": "localhost",
+      "port": 8172,
+      "protocol": "tcp",
+      "listenOptions": {
+        "cost": 51,
+        "precedence": "required"
+      },
+      "portChecks": [
+        {
+          "address": "localhost:8172",
+          "interval": "5s",
+          "timeout": "100ms",
+          "actions": [
+            {
+              "trigger": "change",
+              "action": "send event"
+            },
+            {
+              "trigger": "fail",
+              "action": "increase cost 25"
+            },
+            {
+              "trigger": "pass",
+              "action": "decrease cost 10"
+            }
+          ]
+        }
+      ]
+    }
+  ]
+}
+
+```
+
 # Release 0.19.7

 ## What's New
@@ -90,17 +358,19 @@ Example Output:

 ### Heartbeat Collection And Batching

-In previous versions heartbeats from REST API usage and discrete Edge Router connection would all cause writes
-for the same API Session as they were encountered. In situations where one or more REST API requests were issues and/or one or more
-Edge Router connections were held by a ZitI Application, multiple simultaneous heartbeats could occur for no apparent benefit
-and consume disk write I/O.
+In previous versions heartbeats from REST API usage and discrete Edge Router connection would all
+cause writes for the same API Session as they were encountered. In situations where one or more REST
+API requests were issues and/or one or more Edge Router connections were held by a ZitI Application,
+multiple simultaneous heartbeats could occur for no apparent benefit and consume disk write I/O.

-Heartbeats are now aggregated over a window of time in a cache and written to disk on an interval. The write interval defaults to 90s
-and the batch size (for write transactions) to 250. Additionally, all heartbeats are flush to disk when the controller is properly shut down.
+Heartbeats are now aggregated over a window of time in a cache and written to disk on an interval.
+The write interval defaults to 90s and the batch size (for write transactions) to 250. Additionally,
+all heartbeats are flush to disk when the controller is properly shut down.

 These settings can be defined in the `edge.api` section for the Ziti Controller configuration.

 Example:
+
 ```
 edge:
  api:
@@ -112,7 +382,6 @@ edge:
    ...
 ```

-
 ### Add Service Request Failures for Posture Checks

 When a Ziti Identity (endpoint) requests a service that is provided via a Service Policy with
@@ -0,0 +1,595 @@
+# Transwarp beta_1
+
+![transwarp](transwarp.png)
+
+`openziti/ziti@v0.19.8`
+
+## Goal: Long-haul, High-performance Data Plane for Ziti
+
+The primary goal of the Transwarp project is to create a long-haul, high-performance data plane protocol for the [Ziti fabric][fabric]. Current production deployments of the Ziti fabric currently rely primarily on standard TCP connections for the data plane links between routers. As has been previously discussed, TCP is designed with more generalized objectives, resulting in potentially less-than-optimal performance in a number of important conditions.
+
+TCP is designed to reduce its sending rate when confronted with packet loss in a way that creates even, balanced utilization for multiple connections sharing an underlay link. Transwarp is not primarily concerned with this kind of balance. In a typical Ziti deployment, Transwarp should be deployed when a network link can be mostly dedicated to the Ziti data plane. The Transwarp-based underlay is responsible for completely utilizing the available underlay capacity, and the overlay is responsible for balancing session resources at a higher layer.
+
+Transwarp is a set of Ziti framework components that integrate the high-performance `westworld3` protocol implemented in the [Dilithium project][dilithium].
+
+See the Dilithium [concepts guide][dilithium-concepts] for more details.
+
+## Configurability, Extensibility, Modern Architecture
+
+Transwarp is designed to be configurable and extensible in ways that traditional kernel-space networking stacks are not. This aligns with a number of other "programmable networking" concepts that are present in the Ziti fabric.
+
+The Dilithium components support programmatic extensibility allowing their core behavior to be influenced by application-specific extensions and requirements. This is outside of the scope of Transwarp but important to understand.
+
+The `westworld3` protocol supports a number of tunable parameters that deeply and directly affect its performance in different deployment scenarios. See the [tuning guide][tuning-guide] for more details on these parameters. See the sections below on `westworld3` _profiles_ for information on implementing those parameters in plain Dilithium (without Ziti) testing environments and also in the Ziti fabric.
+
+## Head's Up Deployment
+
+Transwarp _does not_ provide a one-size-fits-all profile that works in any deployment situation as TCP does. Transwarp will need to be intelligently deployed using a reasonable profile selection, so that it provides the appropriate posture to work well in a specific deployment scenario. The default _baseline profile_ has proven to be reasonably performant and effective in long-haul geographically dispersed deployments between cloud regions (in testing environments). It _does not work well_ when deployed on any kind of loopback interface. You can think of the baseline profile as your _starting point_ for developing a profile specific to your deployment.
+
+Incorrect deployment of Transwarp should not result in non-functioning communication links, but incorrect profile selection could result in under-performance, possibly below the level of TCP. In the case of loopback links, current `westworld3` profiles will _significantly_ underperform versus TCP.
+
+Think of Transwarp and `westworld3` as being similar to a race car, in that operating it successfully and well requires more awareness of the application than the typical fully-automated passenger car. TCP is like the latter... you can expect it to behave well in most situations, but you're not likely to get optimal performance from it in the cases where it matters. There are very limited "guard rails" to keep you from misconfiguring `westworld3` and creating performance issues, rather than solving them.
+
+Future work on the Transwarp stack will continue to evolve the "self-tuning" abilities of Transwarp, making it much more suitable for lights-out deployment. But like any powerful tool, when you get it properly dialed in for your deployment, it is possible to achieve throughputs that are _multiple times greater than TCP_.
+
+# Isolated Protocol Testing
+
+Before we look at Transwarp in the Ziti fabric, let's start with the `westworld3` protocol and the `dilithium` framework. This is the foundation of the Transwarp stack, and it is important to have an understanding of the protocol itself.
+
+## Dilithium Tunnel & Dilithium Loop
+
+The Dilithium project includes tooling for apples-to-apples protocol comparisons through an overlay proxy infrastructure, which is designed to operate like a simplified version of the Ziti fabric data plane. The proxy provides an _initiating_ side and a _terminating_ side, with local TCP loops on either side of the protocol under test. Using the `dilithium tunnel` as the basis of your testing environment allows for just the protocol between the terminator and the initiator to be swapped out for other protocols, minimizing variables in the testing.
+
+The Dilithium project also includes tooling for generating consistent loading on both sides of a `dilithium tunnel`. The `dilithium loop` facility is able to saturate a `dilithium tunnel` up to the capacity of a link under test, while (optionally) checking the veracity of the data reception. `dilithium loop` supports both uni-directional and bi-directional testing.
+
+The `dilithium tunnel` supports any TCP protocol on the initiating or terminating sides. There is no requirement to work exclusively with `dilithium loop`.
+
+## Host Requirements
+
+The current implementation of the `westworld3` protocol requires that the host operating system is tuned to support large socket buffer sizes. Development and testing environments include a set of `sysctl` configuration directives:
+
+```
+# adjust the socket buffer sizes
+net.core.rmem_max = 16777216
+net.core.wmem_max = 16777216
+net.core.rmem_default = 16777216
+net.core.wmem_default = 16777216
+net.ipv4.tcp_rmem = 4096 87380 16777216
+net.ipv4.tcp_wmem = 4096 65536 16777216
+net.ipv4.tcp_mem = 8388608 8388608 16777216
+net.ipv4.udp_mem = 8388608 8388608 16777216
+```
+
+> Please Note! Failure to properly configure these kernel parameters will result in SIGNIFICANT under-performance of `westworld3`.
+
+> Please Note! `westworld3` is a UDP-based protocol. Any firewall rules that you create for Transwarp-enabled connections will require UDP permissions.
+
+## Profiles
+
+The `westworld3` protocol includes support for _profiles_, which allow the behavior of the protocol to be tuned to match a specific deployment requirement. Here is the default profile that ships with `v0.3.3`:
+
+```
+*westworld3.Profile {
+	randomize_seq                   false
+	connection_setup_timeout_ms     5000
+	connection_inactive_timeout_ms  15000
+	send_keepalive                  true
+	close_wait_ms                   5000
+	close_check_ms                  500
+	tx_portal_start_sz              98304
+	tx_portal_min_sz                16384
+	tx_portal_max_sz                4194304
+	tx_portal_increase_thresh       224
+	tx_portal_increase_scale        1
+	tx_portal_dupack_thresh         64
+	tx_portal_dupack_capacity_scale 0.9
+	tx_portal_dupack_success_scale  0.75
+	tx_portal_retx_thresh           64
+	tx_portal_retx_capacity_scale   0.75
+	tx_portal_retx_success_scale    0.825
+	tx_portal_rx_sz_pressure_scale  2.8911
+	retx_start_ms                   200
+	retx_scale                      1.5
+	retx_scale_floor                1
+	retx_add_ms                     0
+	retx_evaluation_ms              2000
+	retx_evaluation_scale_incr      0.15
+	retx_evaluation_scale_decr      0.01
+	retx_batch_ms                   2
+	rtt_probe_ms                    50
+	rtt_probe_avg                   8
+	rx_portal_sz_pacing_thresh      0.5
+	max_segment_sz                  1450
+	pool_buffer_sz                  65536
+	rx_buffer_sz                    16777216
+	tx_buffer_sz                    16777216
+	tx_portal_tree_len              16384
+	retx_monitor_tree_len           65536
+	rx_portal_tree_len              16384
+	listener_peers_tree_len         1024
+	reads_queue_len                 1024
+	listener_rx_queue_len           1024
+	accept_queue_len                1024
+}
+```
+
+These values represent all of the tunable parameters that are exposed from the `westworld3` protocol. These can be provided to the `dilithium tunnel` invocations by providing the path to a YAML file containing override values for the default profile, like this:
+
+```
+profile_version: 1
+
+# Cable Upstream Profile
+#
+tx_portal_max_sz:                   655360
+tx_portal_increase_scale:           0.05
+tx_portal_dupack_thresh:            192
+```
+
+```
+$ dilithium tunnel -w cable_upstream.yml
+```
+
+See the [tuning guide][tuning-guide] for more details about all of the profile parameters and guidance about how to tune them.
+
+> Please Note! You _MUST_ include the `profile_version: 1` declaration in your profile YAML.
+
+The out-of-the-box default profile should perform reasonably well for the average long-haul connection.
+
+See the sections below on the _Westworld Analyzer_ for details about analyzing the operation of `westworld3`. There is no substitute for iterative tuning with real-world workloads over real-world underlay.
+
+## Metrics Instrument
+
+Westworld profiles include support for _instruments_. Instruments allow for development and operational analysis of different parts of the operation of the protocol stack. `westworld3` includes a `metrics` instrument that we'll make extensive use of for performance analysis. We can enable this instrument with a profile definition like this:
+
+```
+profile_version: 1
+
+instrument:
+  name:         metrics
+  path:         logs
+  snapshot_ms:  250
+  enabled:      true
+```
+
+The `path` value controls the filesystem path where metrics snapshots will be saved. The `snapshot_ms` value controls how frequently the metrics will be snapshotted.
+
+## Westworld Analyzer
+
+The Westworld Analyzer is the primary tool for understanding what is happening with the `westworld3` protocol and the internals that manage its state. In any situation where we're trying to understand why `westworld3` is performing a certain way, we're going to want to grab the output from the metrics instrument as described above.
+
+This section describes how to configure and operate the Westworld Analyzer for local analysis.
+
+### Run InfluxDB and Grafana with Docker
+
+The `dilithium/bin` folder includes a `run-influxdb.sh` script and a `run-grafana.sh` script. These are configured to launch an InfluxDB and Grafana instance to use with the Westworld Analyzer. Adjust the mount path in the `run-influxdb.sh` script if you want your persistent storage to go to a different folder.
+
+Once you have the InfluxDB container running, you'll want a local `influx` client to access the database. With that installed, you'll need to create a `dilithium` database instance to contain metrics from `westworld3`.
+
+```
+$ influx
+> create database dilithium
+```
+
+With the Grafana container running, connect to it through a web browser on port `3000`. The default username and password are `admin`/`admin`. With Grafana running, you'll need to add an InfluxDB datasource:
+
+![Add InfluxDB Datasource](analyzer-datasource.png)
+
+You'll want to update the name to `Dilithium`, the URL to `http://127.0.0.1:8086`, and the database to `dilithium`. The "Save & Test" button should return a successful result.
+
+Once you've got the datasource configured, you'll need to import the Westworld 3.1 Analyzer dashboard, which is found in the `dilithium/grafana` folder:
+
+![Import Westworld 3.1 Analyzer](analyzer-import.png)
+
+Make sure you've selected the `Dilithium` datasource for the dashboard as defined in the previous step.
+
+### Run Scenario, Snapshot Metrics
+
+With the metrics instrument enabled in your `westworld3` profile, the `dilithium tunnel` command should produce a log line like this:
+
+```
+[   0.001]    INFO dilithium/util.(*CtrlListener).run: [logs/westworld3.13389.sock] started
+```
+
+The `logs/westworld3.13389.sock` is a unix domain socket that can be used to control the metrics instrumentation with the `dilithium ctrl` commands. Use the `dilithium ctrl client` command to snapshot the metrics:
+
+```
+$ dilithium ctrl client logs/westworld3.13389.sock
+[   0.049]    INFO dilithium/cmd/dilithium/ctrl.client: received 'ok'
+[   0.049]    INFO main.main: finished
+```
+
+The above command will cause the onboard metrics instrument to write all observed snapshots out to a series of files, representing the metrics collected by the instrument:
+
+```
+[fedora@ip-10-0-0-217 ~]$ find logs/listener_0.0.0.0-6262_806204675/
+logs/listener_0.0.0.0-6262_806204675/
+logs/listener_0.0.0.0-6262_806204675/rx_keepalive_bytes.csv
+logs/listener_0.0.0.0-6262_806204675/tx_portal_sz.csv
+logs/listener_0.0.0.0-6262_806204675/metrics.id
+logs/listener_0.0.0.0-6262_806204675/rx_keepalive_msgs.csv
+logs/listener_0.0.0.0-6262_806204675/rx_ack_bytes.csv
+logs/listener_0.0.0.0-6262_806204675/tx_keepalive_msgs.csv
+logs/listener_0.0.0.0-6262_806204675/dup_rx_msgs.csv
+logs/listener_0.0.0.0-6262_806204675/dup_acks.csv
+logs/listener_0.0.0.0-6262_806204675/rx_ack_msgs.csv
+logs/listener_0.0.0.0-6262_806204675/tx_portal_rx_sz.csv
+logs/listener_0.0.0.0-6262_806204675/rx_msgs.csv
+logs/listener_0.0.0.0-6262_806204675/retx_scale.csv
+logs/listener_0.0.0.0-6262_806204675/tx_bytes.csv
+logs/listener_0.0.0.0-6262_806204675/tx_ack_msgs.csv
+logs/listener_0.0.0.0-6262_806204675/rx_bytes.csv
+logs/listener_0.0.0.0-6262_806204675/allocations.csv
+logs/listener_0.0.0.0-6262_806204675/retx_ms.csv
+logs/listener_0.0.0.0-6262_806204675/tx_portal_capacity.csv
+logs/listener_0.0.0.0-6262_806204675/errors.csv
+logs/listener_0.0.0.0-6262_806204675/dup_rx_bytes.csv
+logs/listener_0.0.0.0-6262_806204675/rx_portal_sz.csv
+logs/listener_0.0.0.0-6262_806204675/tx_keepalive_bytes.csv
+logs/listener_0.0.0.0-6262_806204675/retx_bytes.csv
+logs/listener_0.0.0.0-6262_806204675/retx_msgs.csv
+logs/listener_0.0.0.0-6262_806204675/tx_msgs.csv
+logs/listener_0.0.0.0-6262_806204675/tx_ack_bytes.csv
+```
+
+You'll want to collect these files for processing by the Westworld Analyzer.
+
+### Import Metrics
+
+The `dilithium influx` commands are used to load the metrics data produced by the metrics instrument into the Westworld Analyzer. With the above metrics data collected, it can be imported into the Westworld Analyzer with the `dilithium influx load` command, like this:
+
+```
+$ dilithium influx load /home/michael/.fablab/instances/transwarp/forensics/1616435348256
+[   0.001]    INFO dilithium/cmd/dilithium/influx.loadWestworld31Metrics: wrote [265] points for westworld3.1 peer [listenerConn_0.0.0.0-6262_13.211.35.123-60243_412359814] dataset [tx_bytes]
+[   0.002]    INFO dilithium/cmd/dilithium/influx.loadWestworld31Metrics: wrote [265] points for westworld3.1 peer [listenerConn_0.0.0.0-6262_13.211.35.123-60243_412359814] dataset [tx_msgs]
+[   0.002]    INFO dilithium/cmd/dilithium/influx.loadWestworld31Metrics: wrote [265] points for westworld3.1 peer [listenerConn_0.0.0.0-6262_13.211.35.123-60243_412359814] dataset [retx_bytes]
+[   0.003]    INFO dilithium/cmd/dilithium/influx.loadWestworld31Metrics: wrote [265] points for westworld3.1 peer [listenerConn_0.0.0.0-6262_13.211.35.123-60243_412359814] dataset [retx_msgs]
+[   0.005]    INFO dilithium/cmd/dilithium/influx.loadWestworld31Metrics: wrote [265] points for 
+...
+[   0.124]    INFO dilithium/cmd/dilithium/influx.loadWestworld31Metrics: wrote [266] points for westworld3.1 peer [dialerConn_[--]-60243_54.167.243.24-6262_623195017] dataset [dup_rx_msgs]
+[   0.125]    INFO dilithium/cmd/dilithium/influx.loadWestworld31Metrics: wrote [266] points for westworld3.1 peer [dialerConn_[--]-60243_54.167.243.24-6262_623195017] dataset [allocations]
+[   0.125]    INFO dilithium/cmd/dilithium/influx.loadWestworld31Metrics: wrote [266] points for westworld3.1 peer [dialerConn_[--]-60243_54.167.243.24-6262_623195017] dataset [errors]
+[   0.142]    INFO main.main: finished
+```
+
+`dilithium influx load` is designed to take the root of the filesystem tree where the directory structure created by the metrics intrument is located. It will automatically traverse the tree and load any metrics it finds into the analyzer database.
+
+### Analyze
+
+With the dataset loaded into the Westworld Analyzer, it can be explored using the _Westworld 3.1 Analyzer_ dashboard you previously imported. You'll want to use the time range controls to find your run telemetry and zoom in on relevant portions of the dataset:
+
+![Westworld3.1 Analyzer](analyzer.png)
+
+Notice the "peer" dropdown in the upper left corner of the dashboard. This will allow you to select the metrics from the peer components present in the dataset. Typically there are separate metrics from the `dialerConn` and `listenerConn` connections, and also a separate set of metrics from the active `listener` that is persistently available in the `westworld3` application.
+
+Every connection has at least 1 `dialerConn` peer and 1 `listenerConn` peer. An application embedding `westworld3` will have only 1 global `listener` peer.
+
+### Clean Metrics
+
+The `dilithium influx clean` command will remove all metrics snapshots from your analyzer database. Because differentiating multiple peers and time ranges can get cumbersome, it's often best to run `dilithium influx clean` to start with a clean slate, and then load in an appropriate dataset for investigation. It's easiest to keep the datasets on disk as files, and only load them into the analyzer for analysis.
+
+## Complete Dilithium Example
+
+The following illustrates a complete example using `dilithium tunnel` and `fablab` (outside of scope for this document) to execute a test and analyze the performance. There is no requirement to use `fablab`, and these scenarios can be re-created manually or through any other automation stack.
+
+### Scenario
+
+In the testing scenario there are two Linux hosts each in a VPC in a different AWS region, separated by significant geography. We're going to compare the performance of `westworld3` and `tcp` over the `dilithium tunnel`.
+
+To keep things simple, we're going to run a file transfer over the tunnel using `scp`, but any TCP-based application protocol should work just fine.
+
+### Launch Tunnel Server and Client
+
+First, we'll need to launch the `dilithium tunnel server` and the `dilithium tunnel client`:
+
+```
+[fedora@ip-10-0-0-217 ~]$ fablab/bin/dilithium tunnel server 0.0.0.0:6262 127.0.0.1:22 -w fablab/cfg/dilithium/westworld3.1/development.yml 
+[   0.000]    INFO dilithium/protocol/westworld3.NewMetricsInstrument: config {
+	path        logs
+	snapshot_ms 250
+	enabled     true
+}
+
+[   0.001]    INFO dilithium/cmd/dilithium/tunnel.tunnelServer: created tunnel listener at [0.0.0.0:6262]
+[   0.001]    INFO dilithium/protocol/westworld3.(*listener).run: started
+[   0.001]    INFO dilithium/util.(*CtrlListener).run: [logs/westworld3.13877.sock] started
+[   0.001]    INFO dilithium/protocol/westworld3.(*metricsInstrumentInstance).snapshotter: started
+```
+
+Use `dilithium tunnel server --help` for details. `0.0.0.0:6262` is the address that the server will listen on for incoming connections for the protocol under test. `127.0.0.1:22` is the "service address" where the tunneled connection will terminate (an `ssh` endpoint on port `22` in our case).
+
+```
+[fedora@ip-10-0-0-222 ~]$ fablab/bin/dilithium tunnel client 54.167.243.24:6262 127.0.0.1:1122 -w fablab/cfg/dilithium/westworld3.1/development.yml 
+[   0.001]    INFO dilithium/cmd/dilithium/tunnel.tunnelClient: created initiator listener at [127.0.0.1:1122]
+```
+
+Use `dilithium tunnel client --help` for details. `54.167.243.24:6262` corresponds to the public address of the `server` we created above. `127.0.0.1:1122` is the local listening address where connections to be tunneled will be accepted.
+
+So in the above example, we'll point our `scp` client at `127.0.0.1:1122` on the host running the `dilithium tunnel client`, and that connection will be proxied across the `westworld3` connection between the `server` and the `client`, and will be terminated at `127.0.0.1:22` on the host running the server.
+
+### Run Workload
+
+The scenario workload is executed from the host running the `dilithium tunnel client`:
+
+```
+[fedora@ip-10-0-0-222 ~]$ scp -P 1122 junk 127.0.0.1:
+junk                                                                                                                         100%  511MB   9.1MB/s   00:56
+```
+
+### Collect Metrics
+
+Snapshot `server` metrics:
+
+```
+[fedora@ip-10-0-0-217 ~]$ fablab/bin/dilithium ctrl client logs/westworld3.13877.sock 
+[   0.104]    INFO dilithium/cmd/dilithium/ctrl.client: received 'ok'
+[   0.104]    INFO main.main: finished
+```
+
+Snapshot `client` metrics:
+
+```
+[fedora@ip-10-0-0-222 ~]$ fablab/bin/dilithium ctrl client logs/westworld3.25891.sock 
+[   0.017]    INFO dilithium/cmd/dilithium/ctrl.client: received 'ok'
+[   0.017]    INFO main.main: finished
+```
+
+Retrieve metrics data from both hosts using whatever mechanism makes the most sense for your environment. In this example, I'm using `fablab` to easily pull down the entire metrics tree from both hosts with a single command:
+
+```
+$ fablab exec logs
+[   0.000]    INFO fablab/zitilib/actions.(*logs).forHost: => [/home/michael/.fablab/instances/transwarp/forensics/1616440013148/logs/local/local]
+[   3.452]    INFO fablab/kernel/fablib.RetrieveRemoteFiles: logs/listenerConn_0.0.0.0-6262_13.211.35.123-53553_388949287/rx_keepalive_bytes.csv => 5.9 kB
+[   3.718]    INFO fablab/kernel/fablib.RetrieveRemoteFiles: logs/listenerConn_0.0.0.0-6262_13.211.35.123-53553_388949287/tx_portal_sz.csv => 6.3 kB
+[   3.996]    INFO fablab/kernel/fablib.RetrieveRemoteFiles: logs/listenerConn_0.0.0.0-6262_13.211.35.123-53553_388949287/metrics.id => 26 B
+...
+[  14.628]    INFO fablab/kernel/fablib.RetrieveRemoteFiles: logs/listener_0.0.0.0-6262_142707432/metrics.id => 26 B
+[  14.938]    INFO fablab/kernel/fablib.RetrieveRemoteFiles: logs/listener_0.0.0.0-6262_142707432/rx_keepalive_msgs.csv => 7.7 kB
+[  15.211]    INFO fablab/kernel/fablib.RetrieveRemoteFiles: logs/listener_0.0.0.0-6262_142707432/rx_ack_bytes.csv => 7.7 kB
+[  15.507]    INFO fablab/kernel/fablib.RetrieveRemoteFiles: logs/listener_0.0.0.0-6262_142707432/tx_keepalive_msgs.csv => 7.7 kB
+...
+[  23.288]    INFO fablab/zitilib/actions.(*logs).forHost: => [/home/michael/.fablab/instances/transwarp/forensics/1616440013148/logs/remote/remote]
+[  51.316]    INFO fablab/kernel/fablib.RetrieveRemoteFiles: logs/dialerConn_[--]-53553_54.167.243.24-6262_389501639/rx_keepalive_bytes.csv => 5.9 kB
+[  53.770]    INFO fablab/kernel/fablib.RetrieveRemoteFiles: logs/dialerConn_[--]-53553_54.167.243.24-6262_389501639/tx_portal_sz.csv => 7.3 kB
+[  56.217]    INFO fablab/kernel/fablib.RetrieveRemoteFiles: logs/dialerConn_[--]-53553_54.167.243.24-6262_389501639/metrics.id => 26 B
+...
+```
+
+We've pulled down peer data for `listenerConn_0.0.0.0-6262_13.211.35.123-53553_388949287`, `listener_0.0.0.0-6262_142707432`, and `dialerConn_[--]-53553_54.167.243.24-6262_389501639`, and those trees of files are located at `/home/michael/.fablab/instances/transwarp/forensics/1616440013148/logs`.
+
+If you're working with Transwarp and `westworld3` regularly, you'll want to invest in tooling to easily retrieve metrics snapshot trees for analysis with the Westworld Analyzer. Anytime we need to diagnose what's going on with Transwarp or `westworld3`, getting a metrics dataset will be the first requirement.
+
+Next, we'll make sure the analyzer is cleaned of previous analysis data:
+
+```
+$ dilithium influx clean
+[   0.003]    INFO dilithium/cmd/dilithium/influx.influxClean: dropped series [dup_acks]
+[   0.004]    INFO dilithium/cmd/dilithium/influx.influxClean: dropped series [errors]
+[   0.005]    INFO dilithium/cmd/dilithium/influx.influxClean: dropped series [retx_scale]
+...
+[   0.075]    INFO dilithium/cmd/dilithium/influx.influxClean: dropped series [dup_rx_bytes]
+[   0.075]    INFO dilithium/cmd/dilithium/influx.influxClean: dropped series [rx_portal_sz]
+[   0.076]    INFO dilithium/cmd/dilithium/influx.influxClean: dropped series [tx_portal_rx_sz]
+[   0.076]    INFO main.main: finished
+```
+
+Then we'll import the new dataset:
+
+```
+$ dilithium influx load /home/michael/.fablab/instances/transwarp/forensics/1616440013148/logs
+[   0.001]    INFO dilithium/cmd/dilithium/influx.loadWestworld31Metrics: wrote [352] points for westworld3.1 peer [listener_0.0.0.0-6262_142707432] dataset [tx_bytes]
+[   0.002]    INFO dilithium/cmd/dilithium/influx.loadWestworld31Metrics: wrote [352] points for westworld3.1 peer [listener_0.0.0.0-6262_142707432] dataset [tx_msgs]
+...
+[   0.091]    INFO dilithium/cmd/dilithium/influx.loadWestworld31Metrics: wrote [267] points for westworld3.1 peer [listenerConn_0.0.0.0-6262_13.211.35.123-53553_388949287] dataset [dup_rx_msgs]
+[   0.091]    INFO dilithium/cmd/dilithium/influx.loadWestworld31Metrics: wrote [267] points for westworld3.1 peer [listenerConn_0.0.0.0-6262_13.211.35.123-53553_388949287] dataset [allocations]
+[   0.092]    INFO dilithium/cmd/dilithium/influx.loadWestworld31Metrics: wrote [267] points for westworld3.1 peer [listenerConn_0.0.0.0-6262_13.211.35.123-53553_388949287] dataset [errors]
+[   0.111]    INFO main.main: finished
+```
+
+If we load up the Westworld 3.1 Analyzer in Grafana, we should see something like this:
+
+![Last 5](analyzer-lastfive.png)
+
+By default the dashboard shows the last 5 minutes, so we'll want to zoom out to the last 30 minutes (or whatever is necessary to find your dataset in time):
+
+![Last 30](analyzer-lastthirty.png)
+
+From there, you can zoom in on the dataset in detail:
+
+![Zoom](analyzer-selected.png)
+
+### Comparing Against TCP
+
+We can run the scenerio again, swapping in the `tcp` protocol instead of the `westworld3` protocol by invoking the `dilithium tunnel server` and `client` like this:
+
+```
+[fedora@ip-10-0-0-217 ~]$ fablab/bin/dilithium tunnel server 0.0.0.0:6262 127.0.0.1:22 -p tcp
+```
+
+```
+[fedora@ip-10-0-0-222 ~]$ fablab/bin/dilithium tunnel client 54.167.243.24:6262 127.0.0.1:1122 -p tcp
+
+```
+
+The `-p <protocol>` option allows for specifying a different underlay protocol to be used when proxying.
+
+# Enabling Transwarp in Ziti
+
+Transwarp is the integration of `westworld3` into Ziti's `transport` framework, allowing it to be used with any Ziti channel. Even though you _could_ use it for a Ziti control channel, it is only recommended that it be used for data plane links.
+
+## Transwarp vs TranswarpTLS
+
+The `transport` integration means that Ziti now has support for the `transwarp:` and `transwarptls:` transports. These `transwarp:` is analagous to `tcp:` and `transwarptls:` is conceptually similar to `tls:`. `transwarptls:` wraps the `westworld3` protocol in a TLS wrapper, providing the same privacy and authentication mechanisms as `tls:`.
+
+## Link Listener
+
+To enable Transwarp in the Ziti fabric data plane, simply adjust any link listeners (in target router configurations) to use the `transwarp:` or `transwarptls:` protocols:
+
+```
+link:
+  listeners:
+    - binding:          transport
+      bind:             transwarptls:127.0.0.1:6002
+      advertise:        transwarptls:127.0.0.1:6002
+```
+
+## Profile Specification
+
+The underlying `westworld3` protocol profile can be specified in the Ziti router configuration like this:
+
+```
+transport:
+  westworld3:
+    profile_version:              1
+    tx_portal_min_sz:             16384
+    tx_portal_max_sz:             1073741824
+    instrument:
+      name:                       metrics
+      path:                       /tmp/westworld3
+      snapshot_ms:                250
+      enabled:                    true
+```
+
+This allows for configuring the `westworld3` profile values as described above in the `dilithium`-only example.
+
+Without specifying a profile the default _baseline profile_ will be used.
+
+## Debugging Transwarp in Ziti
+
+The metrics instrument (and the other `dilithium` instruments) can be used with the downstream Westworld 3.1 Analyzer in the same way as the `dilithium`-only example. Enable the metrics instrument in the router's profile, and then use the `dilithium ctrl client` tool to snapshot the instrument, producing the dataset for the analyzer.
+
+# A Complete Ziti Example
+
+This example will be configured similarly to the pure Dilithium example above. There are two routers, each in an AWS VPC in different regions separated by geography. One of the routers provides a link listener, configured to use `transwarptls:`.
+
+Here's the relevant logging output from the `ziti-router` process:
+
+```
+[   0.002]    INFO foundation/transport/transwarptls.Listen: westworld3 profile = [
+*westworld3.Profile {
+	randomize_seq                   false
+	connection_setup_timeout_ms     5000
+	connection_inactive_timeout_ms  15000
+	send_keepalive                  true
+	close_wait_ms                   5000
+	close_check_ms                  500
+	tx_portal_start_sz              98304
+	tx_portal_min_sz                16384
+	tx_portal_max_sz                4194304
+	tx_portal_increase_thresh       224
+	tx_portal_increase_scale        1
+	tx_portal_dupack_thresh         64
+	tx_portal_dupack_capacity_scale 0.9
+	tx_portal_dupack_success_scale  0.75
+	tx_portal_retx_thresh           64
+	tx_portal_retx_capacity_scale   0.75
+	tx_portal_retx_success_scale    0.825
+	tx_portal_rx_sz_pressure_scale  2.8911
+	retx_start_ms                   200
+	retx_scale                      1.5
+	retx_scale_floor                1
+	retx_add_ms                     0
+	retx_evaluation_ms              2000
+	retx_evaluation_scale_incr      0.15
+	retx_evaluation_scale_decr      0.01
+	retx_batch_ms                   2
+	rtt_probe_ms                    50
+	rtt_probe_avg                   8
+	rx_portal_sz_pacing_thresh      0.5
+	max_segment_sz                  1450
+	pool_buffer_sz                  65536
+	rx_buffer_sz                    16777216
+	tx_buffer_sz                    16777216
+	tx_portal_tree_len              16384
+	retx_monitor_tree_len           65536
+	rx_portal_tree_len              16384
+	listener_peers_tree_len         1024
+	reads_queue_len                 1024
+	listener_rx_queue_len           1024
+	accept_queue_len                1024
+}
+
+]
+[   0.002]    INFO fabric/router.(*Router).startXlinkListeners: started Xlink listener with binding [transport] advertising [transwarptls:54.167.243.24:6000]
+```
+
+Here's the relevant log output when starting the second `ziti-router`, configured without a link listener. This output represents it dialing the `transwarptls:` link connection to the first router:
+
+```
+[   0.630]    INFO fabric/router/handler_ctrl.(*dialHandler).handle: received link connect request
+[   0.630]    INFO fabric/router/xlink_transport.(*dialer).Dial: dialing link with split payload/ack channels [l/ZoLq]
+[   0.631]    INFO fabric/router/xlink_transport.(*dialer).Dial: dialing payload channel for link [l/ZoLq]
+[   0.631]    INFO foundation/transport/transwarptls.Dial: westworld3 profile = [
+*westworld3.Profile {
+	...
+}
+
+]
+[   0.631]    INFO dilithium/protocol/westworld3.(*dialerConn).hello: starting hello process
+[   0.631]    INFO dilithium/protocol/westworld3.(*rxPortal).run: started
+[   0.830]    INFO dilithium/protocol/westworld3.(*dialerConn).hello: completed hello process
+[   0.830]    INFO dilithium/protocol/westworld3.(*dialerConn).rxer: started
+[   0.831]    INFO dilithium/protocol/westworld3.(*txPortal).keepaliveSender: started
+[   0.831]    INFO dilithium/protocol/westworld3.(*closer).run: started
+[   0.831]    INFO dilithium/protocol/westworld3.(*retxMonitor).run: started
+[   1.264]    INFO fabric/router/xlink_transport.(*dialer).Dial: dialing ack channel for link [l/ZoLq]
+[   1.264]    INFO foundation/transport/transwarptls.Dial: westworld3 profile = [
+*westworld3.Profile {
+	...
+}
+
+]
+[   1.265]    INFO dilithium/protocol/westworld3.(*dialerConn).hello: starting hello process
+[   1.265]    INFO dilithium/protocol/westworld3.(*rxPortal).run: started
+[   1.464]    INFO dilithium/protocol/westworld3.(*dialerConn).hello: completed hello process
+[   1.464]    INFO dilithium/protocol/westworld3.(*dialerConn).rxer: started
+[   1.464]    INFO dilithium/protocol/westworld3.(*txPortal).keepaliveSender: started
+[   1.464]    INFO dilithium/protocol/westworld3.(*closer).run: started
+[   1.464]    INFO dilithium/protocol/westworld3.(*retxMonitor).run: started
+[   1.896]    INFO fabric/router.(*xlinkAccepter).Accept: accepted new link [l/ZoLq]
+[   1.896]    INFO fabric/router/handler_ctrl.(*dialHandler).handle: link [l/ZoLq] established
+```
+
+Ziti links currently require 2 underlay connections, with one of them dedicated to flow control processing. This is why there are 2 `westworld3` connections established above.
+
+> There is currently an open issue in the `openziti/fabric` project to allow the ACK channel to operate on a protocol other than the data channel. Running the ACK channel over TCP and the data channel over `westworld3` will make analyzing the metrics simpler in Ziti-enabled cases. Coming soon.
+
+After running our workload over the Ziti fabric with a Transwarp data plane, we can retrieve the `westworld3` analysis dataset in the same way we did in the plain `dilithium` scenario above. Both routers emit log messages advertising their metrics instrument control socket:
+
+```
+[   0.003]    INFO dilithium/util.(*CtrlListener).run: [logs/westworld3.17856.sock] started
+```
+
+```
+[   0.637]    INFO dilithium/util.(*CtrlListener).run: [logs/westworld3.30033.sock] started
+```
+
+We can use those sockets to snapshot the metrics:
+
+```
+[fedora@ip-10-0-0-217 ~]$ fablab/bin/dilithium ctrl client logs/westworld3.17856.sock 
+[   0.070]    INFO dilithium/cmd/dilithium/ctrl.client: received 'ok'
+[   0.071]    INFO main.main: finished
+```
+
+```
+[fedora@ip-10-0-0-222 ~]$ fablab/bin/dilithium ctrl client logs/westworld3.30033.sock 
+[   0.053]    INFO dilithium/cmd/dilithium/ctrl.client: received 'ok'
+[   0.053]    INFO main.main: finished
+```
+
+Retrieve the metrics dataset from the routers and ingest them into the Westworld 3.1 Analyzer using the approach described in the plain Dilithium example. The analyzer does not care if the metrics dataset came from a `dilithium tunnel` or from Ziti.
+
+![Transwarp Analyzer](analyzer-transwarp.png)
+
+When loading the dataset into the analyzer, you'll see that there is 1 `listener` peer, and 2 each of `listenerConn` and `dialerConn` for the above scenario. In situations where more links are being used, there will potentially be more peers and a larger dataset.
+
+# This is a BETA
+
+This is the first Transwarp beta release (`beta_1`). Building a protocol like `westworld3` and integrating it into an overlay like Ziti requires a great deal of testing and generation of experience in different environments with different profiles. Anytime we're investigating an issue with Transwarp or with `westworld3`, it's going to be very important that we have a clear metrics dataset and a concrete description of the profile that was used to generate the dataset.
+
+Happy Transwarp-ing.
+
+[dilithium]: https://github.com/openziti/dilithium
+[dilithium-concepts]: https://github.com/openziti/dilithium/blob/main/docs/concepts.md
+[tuning-guide]: https://github.com/openziti/dilithium/blob/main/docs/tuning.md
+[fabric]: https://github.com/openziti/fabric
@@ -72,8 +72,6 @@ link:
  dialers:
    - binding:          transport

-
-
 listeners:
  # basic ssh proxy
  - binding:            proxy
@@ -116,27 +114,4 @@ dialers:
 # xctrl_example
 #
 #example:
-#  delay:                1
-
-transport:
-  # Mumbai_2
-  westworld2:
-    tx_portal_start_sz:           131072
-    tx_portal_min_sz:             16384
-    tx_portal_max_sz:             1073741824
-    tx_portal_increase_ct:        2
-    tx_portal_increase_frac:      1.5
-    tx_portal_dup_ack_ct:         256
-    tx_portal_dup_ack_frac:       0.9995
-    tx_portal_retx_ct:            128
-    tx_portal_retx_frac:          0.9995
-    retx_start_ms:                200
-    retx_add_ms:                  25
-    rtt_probe_ms:                 50
-    rtt_probe_avg_ct:             8
-    max_segment_sz:               1450
-    rx_buffer_sz:                 4194304
-    tx_buffer_sz:                 4194304
-    #instrument:
-      #name:                       metrics
-      #prefix:                     /tmp/logs
+#  delay:                1
@@ -17,9 +17,8 @@ link:
    - binding:          transport
      bind:             tls:127.0.0.1:6002
      advertise:        tls:127.0.0.1:6002
-      # Enable transwarp for the mesh data plane
-      #bind:             transwarp:127.0.0.1:6002
-      #advertise:        transwarp:127.0.0.1:6002
+      #bind:             transwarptls:127.0.0.1:6002
+      #advertise:        transwarptls:127.0.0.1:6002
      options:
        outQueueSize:   16
  dialers:
@@ -28,24 +27,12 @@ link:
        outQueueSize:   32

 transport:
-  # Mumbai_2
-  westworld2:
-    tx_portal_start_sz:           131072
-    tx_portal_min_sz:             16384
-    tx_portal_max_sz:             1073741824
-    tx_portal_increase_ct:        2
-    tx_portal_increase_frac:      1.5
-    tx_portal_dup_ack_ct:         256
-    tx_portal_dup_ack_frac:       0.9995
-    tx_portal_retx_ct:            128
-    tx_portal_retx_frac:          0.9995
-    retx_start_ms:                200
-    retx_add_ms:                  25
-    rtt_probe_ms:                 50
-    rtt_probe_avg_ct:             8
-    max_segment_sz:               1450
-    rx_buffer_sz:                 4194304
-    tx_buffer_sz:                 4194304
+  westworld3:
+    profile_version:              1
+    #tx_portal_min_sz:             16384
+    #tx_portal_max_sz:             1073741824
    #instrument:
-      #name:                       metrics
-      #prefix:                     /tmp/logs
+    #  name:                       metrics
+    #  path:                       /tmp/westworld3
+    #  snapshot_ms:                250
+    #  enabled:                    true
@@ -52,10 +52,10 @@ metrics:
    #url:                http://localhost:8086
    #database:           ziti

-# events:
+#events:
 #  jsonLogger:
 #    subscriptions:
-#       - type: fabric.terminators
+#      - type: fabric.terminators
 #      - type: metrics
 #        sourceFilter: .*
 #        metricFilter: .*egress.*m1_rate*
@@ -66,6 +66,7 @@ metrics:
 #        include:
 #          - created
 #      - type: fabric.usage
+#      - type: services
 #    handler:
 #      type: file
 #      format: json
@@ -80,6 +80,9 @@ dialers:
  - binding: transport

 listeners:
+  - binding: tunnel
+    options:
+      mode: host
  - binding: edge
    address: tls:0.0.0.0:${ZITI_EDGE_PORT}
    options:
@@ -4,7 +4,7 @@ go 1.16

 //replace github.com/openziti/foundation => ../foundation

-//replace github.com/michaelquigley/dilithium => ../../q/research/dilithium
+//replace github.com/openziti/dilithium => ../dilithium

 //replace github.com/openziti/fabric => ../fabric

@@ -27,10 +27,10 @@ require (
 	github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d
 	github.com/keybase/go-ps v0.0.0-20190827175125-91aafc93ba19
 	github.com/michaelquigley/pfxlog v0.3.7
-	github.com/openziti/edge v0.19.42
-	github.com/openziti/fabric v0.16.34
-	github.com/openziti/foundation v0.15.38
-	github.com/openziti/sdk-golang v0.15.25
+	github.com/openziti/edge v0.19.55
+	github.com/openziti/fabric v0.16.44
+	github.com/openziti/foundation v0.15.44
+	github.com/openziti/sdk-golang v0.15.31
 	github.com/pborman/uuid v1.2.0
 	github.com/pkg/errors v0.9.1
 	github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0
@@ -44,7 +44,7 @@ require (
 	github.com/stretchr/testify v1.7.0
 	github.com/urfave/negroni v1.0.0
 	go.etcd.io/bbolt v1.3.5-0.20200615073812-232d8fc87f50
-	golang.org/x/net v0.0.0-20210226101413-39120d07d75e
+	golang.org/x/net v0.0.0-20210226172049-e18ecbb05110
 	google.golang.org/grpc v1.27.1
 	gopkg.in/AlecAivazis/survey.v1 v1.8.7
 	gopkg.in/resty.v1 v1.12.0
@@ -30,8 +30,8 @@ dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBr
 dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4=
 dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU=
 git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg=
-github.com/AppsFlyer/go-sundheit v0.2.0 h1:FArqX+HbqZ6U32RC3giEAWRUpkggqxHj91KIvxNgwjU=
-github.com/AppsFlyer/go-sundheit v0.2.0/go.mod h1:rCRkVTMQo7/krF7xQ9X0XEF1an68viFR6/Gy02q+4ds=
+github.com/AppsFlyer/go-sundheit v0.3.1 h1:Zqnr3wV3WQmXonc234k9XZAoV2KHUHw3osR5k2iHQZE=
+github.com/AppsFlyer/go-sundheit v0.3.1/go.mod h1:iZ8zWMS7idcvmqewf5mEymWWgoOiG/0WD4+aeh+heX4=
 github.com/Azure/azure-sdk-for-go v32.4.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
 github.com/Azure/go-autorest/autorest v0.1.0/go.mod h1:AKyIcETwSUFxIcs/Wnq/C+kwCtlEYGUVd7FPNb2slmg=
 github.com/Azure/go-autorest/autorest v0.5.0/go.mod h1:9HLKlQjVBH6U3oDfsXOeVc56THsLPw1L03yban4xThw=
@@ -78,7 +78,6 @@ github.com/antlr/antlr4 v0.0.0-20210114010855-d34d2e1c271a h1:gXj02obcEEw9NVyvbn
 github.com/antlr/antlr4 v0.0.0-20210114010855-d34d2e1c271a/go.mod h1:T7PbCXFs94rrTttyxjbyT5+/1V8T2TYDejxUfHJjw1Y=
 github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
-github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
@@ -116,11 +115,9 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cloudflare/cloudflare-go v0.13.2/go.mod h1:27kfc1apuifUmJhp069y0+hwlKDg4bd8LWlu7oKeZvM=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
-github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-iptables v0.5.0 h1:mw6SAibtHKZcNzAsOxjoHIG0gy5YFHhypWSSNc6EjbQ=
 github.com/coreos/go-iptables v0.5.0/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU=
-github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
@@ -133,7 +130,7 @@ github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/deepmap/oapi-codegen v1.3.6/go.mod h1:aBozjEveG+33xPiP55Iw/XbVkhtZHEGLq3nxlX0+hfU=
+github.com/deepmap/oapi-codegen v1.3.13/go.mod h1:WAmG5dWY8/PYHt4vKxlt90NsbHMAOCiteYKZMiIRfOo=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/dgoogauth v0.0.0-20190221195224-5a805980a5f3 h1:AqeKSZIG/NIC75MNQlPy/LM3LxfpLwahICJBHwSMFNc=
@@ -169,7 +166,7 @@ github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWo
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa h1:RDBNVkRviHZtvDvId8XSGPu3rmpmSe+wKRcEWNgsfWU=
 github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa/go.mod h1:KnogPXtdwXqoenmZCw6S+25EAm2MkxbG0deNDu4cbSA=
-github.com/getkin/kin-openapi v0.2.0/go.mod h1:V1z9xl9oF5Wt7v32ne4FmiF1alpS4dM6mNzoywPOXlk=
+github.com/getkin/kin-openapi v0.13.0/go.mod h1:WGRs2ZMM1Q8LR1QBEwUxC6RJEfaBcD0s+pcEVXFuAjw=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
 github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
@@ -236,8 +233,8 @@ github.com/go-openapi/runtime v0.19.4/go.mod h1:X277bwSUBxVlCYR3r7xgZZGKVvBd/29g
 github.com/go-openapi/runtime v0.19.15/go.mod h1:dhGWCTKRXlAfGnQG0ONViOZpjfg0m2gUt9nTQPQZuoo=
 github.com/go-openapi/runtime v0.19.16/go.mod h1:5P9104EJgYcizotuXhEuUrzVc+j1RiSjahULvYmlv98=
 github.com/go-openapi/runtime v0.19.24/go.mod h1:Lm9YGCeecBnUUkFTxPC4s1+lwrkJ0pthx8YvyjCfkgk=
-github.com/go-openapi/runtime v0.19.26 h1:K/6PoVNj5WJXUnMk+VEbELeXjtBkCS1UxTDa04tdXE0=
-github.com/go-openapi/runtime v0.19.26/go.mod h1:BvrQtn6iVb2QmiVXRsFAm6ZCAZBpbVKFfN6QWCp582M=
+github.com/go-openapi/runtime v0.19.27 h1:4zrQCJoP7rqNCUaApDv1MdPkaa5TuPfO05Lq5WLhX9I=
+github.com/go-openapi/runtime v0.19.27/go.mod h1:BvrQtn6iVb2QmiVXRsFAm6ZCAZBpbVKFfN6QWCp582M=
 github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI=
 github.com/go-openapi/spec v0.18.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI=
 github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY=
@@ -317,7 +314,6 @@ github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4er
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
@@ -385,7 +381,6 @@ github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2z
 github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
 github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
 github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
-github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
@@ -425,7 +420,7 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:
 github.com/iij/doapi v0.0.0-20190504054126-0bbf12d6d7df/go.mod h1:QMZY7/J/KSQEhKWFeDesPjMj+wCHReeknARU3wqlyN4=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
-github.com/influxdata/influxdb-client-go v1.3.0/go.mod h1:S+oZsPivqbcP1S9ur+T+QqXvrYS3NCZeMQtBoH4D1dw=
+github.com/influxdata/influxdb-client-go/v2 v2.2.2/go.mod h1:fa/d1lAdUHxuc1jedx30ZfNG573oQTQmUni3N6pcW+0=
 github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d h1:/WZQPMZNsjZ7IlCpsLGdQBINg5bxKQ1K1sh6awxLtkA=
 github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo=
 github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo=
@@ -490,13 +485,12 @@ github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvf
 github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k=
 github.com/linode/linodego v0.21.0/go.mod h1:UTpq1JUZD0CZsJ8rt+0CRkqbzrp1MbGakVPt2DXY5Mk=
 github.com/liquidweb/liquidweb-go v1.6.1/go.mod h1:UDcVnAMDkZxpw4Y7NOHkqoeiGacVLEIG/i5J9cyixzQ=
-github.com/lucas-clemente/quic-go v0.17.1/go.mod h1:I0+fcNTdb9eS1ZcjQZbDVPGchJ86chcIxPALn9lEJqE=
+github.com/lucas-clemente/quic-go v0.18.0/go.mod h1:yXttHsSNxQi8AWijC/vLP+OJczXqzHSOcJrM5ITUlCg=
 github.com/lucas-clemente/quic-go v0.18.1 h1:DMR7guC0NtVS8zNZR3IO7NARZvZygkSC56GGtC6cyys=
 github.com/lucas-clemente/quic-go v0.18.1/go.mod h1:yXttHsSNxQi8AWijC/vLP+OJczXqzHSOcJrM5ITUlCg=
 github.com/lucsky/cuid v1.0.2 h1:z4XlExeoderxoPj2/dxKOyPxe9RCOu7yNq9/XWxIUMQ=
 github.com/lucsky/cuid v1.0.2/go.mod h1:QaaJqckboimOmhRSJXSx/+IT+VTfxfPGSo/6mfgUfmE=
 github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI=
-github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
 github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
@@ -509,9 +503,7 @@ github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0
 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE=
 github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0=
-github.com/marten-seemann/qpack v0.1.0/go.mod h1:LFt1NU/Ptjip0C2CPkhimBz5CGE3WGDAUWqna+CNTrI=
 github.com/marten-seemann/qpack v0.2.0/go.mod h1:F7Gl5L1jIgN1D11ucXefiuJS9UMVP2opoCp2jDKb7wc=
-github.com/marten-seemann/qtls v0.9.1/go.mod h1:T1MmAdDPyISzxlK6kjRr0pcZFBVd1OZbBb/j3cvzHhk=
 github.com/marten-seemann/qtls v0.10.0 h1:ECsuYUKalRL240rRD4Ri33ISb7kAQ3qGDlrrl55b2pc=
 github.com/marten-seemann/qtls v0.10.0/go.mod h1:UvMd1oaYDACI99/oZUYLzMCkBXQVT0aGm99sJhbT8hs=
 github.com/marten-seemann/qtls-go1-15 v0.1.0 h1:i/YPXVxz8q9umso/5y474CNcHmTpA+5DH+mFPjx6PZg=
@@ -521,8 +513,8 @@ github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaO
 github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
 github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
-github.com/mattn/go-colorable v0.1.6 h1:6Su7aK7lXmJ/U79bYtBjLNaha4Fs1Rg9plHpcH+vvnE=
-github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
+github.com/mattn/go-colorable v0.1.7 h1:bQGKb3vps/j0E9GfJQ03JyhRuxsvdAanXlT9BTw3mdw=
+github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
@@ -549,18 +541,17 @@ github.com/mdlayher/netlink v1.2.2-0.20210123213345-5cc92139ae3e/go.mod h1:bacnN
 github.com/mdlayher/netlink v1.3.0/go.mod h1:xK/BssKuwcRXHrtN04UBkwQ6dY9VviGGuriDdoPSWys=
 github.com/mdlayher/netlink v1.4.0 h1:n3ARR+Fm0dDv37dj5wSWZXDKcy+U0zwcXS3zKMnSiT0=
 github.com/mdlayher/netlink v1.4.0/go.mod h1:dRJi5IABcZpBD2A3D0Mv/AiX8I9uDEu5oGkAVrekmf8=
-github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4=
 github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
-github.com/michaelquigley/dilithium v0.2.0 h1:JNVb9ckXDTAxYVpPHPzkBuXObh0TYymuZgAhZclcvAo=
-github.com/michaelquigley/dilithium v0.2.0/go.mod h1:MIqCw2+ObUwmS7G+8N8hWiwNMjItUEBzeTlipmiZqOw=
-github.com/michaelquigley/pfxlog v0.0.0-20190813191113-2be43bd0dccc/go.mod h1:Z0K0l/NScHvLeGoy4SZTps+ZxGv85ql4AvZqTqOxBY8=
+github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQE9x6ikvDFZS2mDVS3drnohI=
+github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
+github.com/michaelquigley/pfxlog v0.3.1/go.mod h1:pwJmJ9nN787xk9U+0z8/5nP2GMhU6GGCTfgBWGpiXaQ=
 github.com/michaelquigley/pfxlog v0.3.7 h1:itV+da+Kek0GwApI2vOcGhLdMk6eCNfjXoR6/Pezo+k=
 github.com/michaelquigley/pfxlog v0.3.7/go.mod h1:s4uqqB54PDLSU6JvpizMBJ2VU4cpG3bXsXJvFHqHDLU=
 github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/miekg/dns v1.1.31/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
-github.com/miekg/dns v1.1.40 h1:pyyPFfGMnciYUk/mXpKkVmeMQjfXqt3FAJ2hy7tPiLA=
-github.com/miekg/dns v1.1.40/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
+github.com/miekg/dns v1.1.41 h1:WMszZWJG0XmzbK9FEmzH2TVcqYzFesusSIB41b8KHxY=
+github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI=
 github.com/miekg/pkcs11 v1.0.3 h1:iMwmD7I5225wv84WxIG/bmxz9AXjWvTWIbM/TYHvWtw=
 github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs=
 github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
@@ -606,13 +597,11 @@ github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn
 github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
 github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA=
 github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
-github.com/onsi/gomega v1.8.1/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA=
 github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
 github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
@@ -620,14 +609,16 @@ github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTm
 github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw=
 github.com/openziti/bbolt v1.3.6-0.20210317142109-547da822475e h1:ST9+54UtCr96S5+Wa7LQtpGS6/xK7Z+KQXvOAtYgJlA=
 github.com/openziti/bbolt v1.3.6-0.20210317142109-547da822475e/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
-github.com/openziti/edge v0.19.42 h1:N/AJpNlWWuTe9T3Xm+yKvZtn3s5AuPF3yiwB4hc78IE=
-github.com/openziti/edge v0.19.42/go.mod h1:f4RGk64cNx+SfENoz/SQb9vqj4zhGfJETVCeDWACOTU=
-github.com/openziti/fabric v0.16.34 h1:WmznP2xp6KkevPIPis9EgRM9F3nHz4F4XAAeZTe2jbc=
-github.com/openziti/fabric v0.16.34/go.mod h1:VJeX6WbHMVsUiYuiAVRnChAGTM4fN2DL01M7ibD09a4=
-github.com/openziti/foundation v0.15.38 h1:Cy6LkIhROI0k2tHbuEDWUu4aouNZrv4DZ5fIXOOkIBA=
-github.com/openziti/foundation v0.15.38/go.mod h1:DsIwSMXxjcbnJGRPeOfGqNSN44JJGG3S8gVq0JlCCK8=
-github.com/openziti/sdk-golang v0.15.25 h1:BNIO/A2CTZVS8SzF8Qj206lFSoTZLXzvHCr9H+baf1s=
-github.com/openziti/sdk-golang v0.15.25/go.mod h1:xj9QMaqLyEyxFhB8m3ybLLwdUNkOTZevubvSJWPmB+s=
+github.com/openziti/dilithium v0.3.3 h1:PLgQ6PMNLSTzCFbX/h98cmudgz/cU6TmjdSv5NAPD8k=
+github.com/openziti/dilithium v0.3.3/go.mod h1:vsCjI2AU/hon9e+dLhUFbCNGesJDj2ASgkySOcpmvjo=
+github.com/openziti/edge v0.19.55 h1:FZntwcmgTw2GcX6Hib+l4UkHSsCe1mEUPJVDpQpi4cg=
+github.com/openziti/edge v0.19.55/go.mod h1:Ga0sP83HV0dtMAAZG5WcFZvqqdyPD2vpj4AV2b2BD+c=
+github.com/openziti/fabric v0.16.44 h1:pMEXl8uSdgd/Dj7d4F0qK5TnlLuse5uCpZs3Ah6fPbM=
+github.com/openziti/fabric v0.16.44/go.mod h1:soU2ImwCdNOnLDM5JUqpk5wAvrN9LaslZL02oThJQUM=
+github.com/openziti/foundation v0.15.44 h1:+XMOMaog+ABBvxgmDhk70H0buhw2RtpeDClxk+TJ3qQ=
+github.com/openziti/foundation v0.15.44/go.mod h1:kYCUttsGO/GjoD7BSNhVOeyYDXHYMAfz59YeLoEyhHY=
+github.com/openziti/sdk-golang v0.15.31 h1:vo+/wRbNtYBvkxHrAo9Is32SAJQjlhIS+9gHoajB1g0=
+github.com/openziti/sdk-golang v0.15.31/go.mod h1:jrvkpxP7U1ejSKEj4J9/UTNWlt6nJaofUGKkTx7afvM=
 github.com/oracle/oci-go-sdk v24.2.0+incompatible/go.mod h1:VQb79nF8Z2cwLkLS35ukwStZIg5F66tcBccjip/j888=
 github.com/orcaman/concurrent-map v0.0.0-20190826125027-8c72a8bb44f6/go.mod h1:Lu3tH6HLW3feq74c2GC+jIMS/K2CFcDWnWD9XkenwhI=
 github.com/orcaman/concurrent-map v0.0.0-20210106121528-16402b402231 h1:fa50YL1pzKW+1SsBnJDOHppJN9stOEwS+CRWyUtyYGU=
@@ -720,7 +711,6 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx
 github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
-github.com/sirupsen/logrus v1.5.0/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo=
 github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
 github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
 github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
@@ -743,7 +733,6 @@ github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B
 github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
 github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
-github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
 github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI=
 github.com/spf13/cobra v1.1.3 h1:xghbfqPkxzxP3C/f3n5DdpAbdKLj4ZE4BWQI362l53M=
 github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo=
@@ -752,7 +741,6 @@ github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb6
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
-github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
 github.com/spf13/viper v1.7.0 h1:xVKxvI7ouOI5I+U9s2eeiUfMaWBVoXA3AWskkrqK0VM=
 github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -777,7 +765,6 @@ github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhV
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/transip/gotransip/v6 v6.2.0/go.mod h1:pQZ36hWWRahCUXkFWlx9Hs711gLd8J4qdgLdRzmtY+g=
 github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g=
-github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
 github.com/urfave/cli v1.22.4/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
 github.com/urfave/negroni v1.0.0 h1:kIimOitoypq34K7TG7DUaJ9kq/N4Ofuwi1sjz0KipXc=
 github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4=
@@ -797,7 +784,6 @@ github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:
 github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74=
 github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
-github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
 go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
 go.mongodb.org/mongo-driver v1.3.0/go.mod h1:MSWZXKOynuguX+JSvwP8i+58jYCXxbia8HS3gZBapIE=
@@ -813,9 +799,7 @@ go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
 go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
-go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
-go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8=
 go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
@@ -834,6 +818,7 @@ golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACk
 golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190418165655-df01cb2cc480/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
 golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
+golang.org/x/crypto v0.0.0-20190422183909-d864b10871cd/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
@@ -845,10 +830,9 @@ golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20191202143827-86a70503ff7e/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899 h1:DZhuSZLsGlFL4CmhA8BcRA0mnthyA/nZ00AqCUo7vHg=
-golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a h1:vclmkQCjlDX5OydZ9wv8rBCcS0QyQY66Mpf/7BZbInM=
+golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -892,14 +876,12 @@ golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73r
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190228165749-92fc7df08ae7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190320064053-1272bf9dcd53/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -926,8 +908,8 @@ golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwY
 golang.org/x/net v0.0.0-20201216054612-986b41b23924/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20210226101413-39120d07d75e h1:jIQURUJ9mlLvYwTBtRHm9h58rYhSonLvRvgAnP8Nr7I=
-golang.org/x/net v0.0.0-20210226101413-39120d07d75e/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -943,8 +925,8 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 h1:qwRHBd0NqMbJxfbotnDhm2ByMI1Shq4Y6oRJo21SGJA=
-golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180606202747-9527bec2660b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180622082034-63fc586f45fe/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -996,8 +978,7 @@ golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201118182958-a01c418693c7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -1006,8 +987,9 @@ golang.org/x/sys v0.0.0-20201218084310-7d0127a74742/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210110051926-789bb1bd4061/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210123111255-9b0068b26619/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210216163648-f7da38b97c65 h1:pTMjDVnP5eVRRlWO76rEWJ8JoC6Lf1CmyjPZXRiy2Sw=
 golang.org/x/sys v0.0.0-20210216163648-f7da38b97c65/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210303074136-134d130e1a04 h1:cEhElsAv9LUt9ZUUocxzWe05oFLVd+AA2nstydTeI8g=
+golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -1123,7 +1105,6 @@ google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.19.1/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
-google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
 google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
 google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
@@ -1170,7 +1151,6 @@ gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bl
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
@@ -24,6 +24,7 @@ import (
 	"github.com/openziti/foundation/transport/tcp"
 	"github.com/openziti/foundation/transport/tls"
 	"github.com/openziti/foundation/transport/transwarp"
+	"github.com/openziti/foundation/transport/transwarptls"
 	"github.com/openziti/foundation/transport/wss"
 	"github.com/openziti/ziti/common/version"
 	"github.com/openziti/ziti/ziti-controller/subcmd"
@@ -38,6 +39,7 @@ func init() {
 	transport.AddAddressParser(tls.AddressParser{})
 	transport.AddAddressParser(tcp.AddressParser{})
 	transport.AddAddressParser(transwarp.AddressParser{})
+	transport.AddAddressParser(transwarptls.AddressParser{})
 	transport.AddAddressParser(wss.AddressParser{})

 	build.InitBuildInfo(version.GetCmdBuildInfo())
@@ -25,6 +25,9 @@ import (
 	"github.com/openziti/ziti/common/version"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
+	"os"
+	"os/signal"
+	"syscall"
 )

 func init() {
@@ -54,28 +57,39 @@ func run(cmd *cobra.Command, args []string) {
 			}
 		}

-		var c *controller.Controller
-		if c, err = controller.NewController(config, version.GetCmdBuildInfo()); err != nil {
+		var fabricController *controller.Controller
+		if fabricController, err = controller.NewController(config, version.GetCmdBuildInfo()); err != nil {
 			fmt.Printf("unable to create fabric controller %+v\n", err)
 			panic(err)
 		}

-		ec, err := server.NewController(config)
+		edgeController, err := server.NewController(config)

 		if err != nil {
 			panic(err)
 		}

-		ec.SetHostController(c)
-		ec.Initialize()
-
-		go ec.RunAndWait()
-
-		if err = c.Run(); err != nil {
+		edgeController.SetHostController(fabricController)
+		edgeController.Initialize()
+		go waitForShutdown(fabricController, edgeController)
+		go edgeController.Run()
+		if err := fabricController.Run(); err != nil {
 			panic(err)
 		}

+
 	} else {
 		panic(err)
 	}
 }
+
+func waitForShutdown(fabricController *controller.Controller, edgeController *server.Controller) {
+	ch := make(chan os.Signal, 1)
+	signal.Notify(ch, os.Interrupt, syscall.SIGTERM)
+
+	<-ch
+
+	pfxlog.Logger().Info("shutting down ziti-controller")
+	edgeController.Shutdown()
+	fabricController.Shutdown()
+}
@@ -23,6 +23,7 @@ import (
 	"github.com/openziti/foundation/transport/tcp"
 	"github.com/openziti/foundation/transport/tls"
 	"github.com/openziti/foundation/transport/transwarp"
+	"github.com/openziti/foundation/transport/transwarptls"
 	"github.com/openziti/foundation/transport/wss"
 	"github.com/openziti/ziti/ziti-fabric-gw/subcmd"
 	"github.com/sirupsen/logrus"
@@ -36,6 +37,7 @@ func init() {
 	transport.AddAddressParser(tls.AddressParser{})
 	transport.AddAddressParser(tcp.AddressParser{})
 	transport.AddAddressParser(transwarp.AddressParser{})
+	transport.AddAddressParser(transwarptls.AddressParser{})
 	transport.AddAddressParser(wss.AddressParser{})
 }

@@ -23,6 +23,7 @@ import (
 	"github.com/openziti/foundation/transport/tcp"
 	"github.com/openziti/foundation/transport/tls"
 	"github.com/openziti/foundation/transport/transwarp"
+	"github.com/openziti/foundation/transport/transwarptls"
 	"github.com/openziti/foundation/transport/wss"
 	"github.com/openziti/foundation/util/info"
 	"github.com/openziti/ziti/ziti-fabric-test/subcmd"
@@ -41,6 +42,7 @@ func init() {
 	transport.AddAddressParser(tls.AddressParser{})
 	transport.AddAddressParser(tcp.AddressParser{})
 	transport.AddAddressParser(transwarp.AddressParser{})
+	transport.AddAddressParser(transwarptls.AddressParser{})
 	transport.AddAddressParser(wss.AddressParser{})
 }

@@ -70,7 +70,8 @@ func newDialerCmd() *dialerCmd {
 func (cmd *dialerCmd) run(_ *cobra.Command, args []string) {
 	log := pfxlog.Logger()

-	if err := agent.Listen(agent.Options{}); err != nil {
+	shutdownClean := false
+	if err := agent.Listen(agent.Options{ShutdownCleanup: &shutdownClean}); err != nil {
 		pfxlog.Logger().WithError(err).Error("unable to start CLI agent")
 	}

@@ -68,7 +68,8 @@ func (cmd *listenerCmd) run(_ *cobra.Command, args []string) {
 	log := pfxlog.Logger()

 	var err error
-	if err = agent.Listen(agent.Options{}); err != nil {
+	shutdownClean := false
+	if err = agent.Listen(agent.Options{ShutdownCleanup: &shutdownClean}); err != nil {
 		pfxlog.Logger().WithError(err).Error("unable to start CLI agent")
 	}

@@ -23,6 +23,7 @@ import (
 	"github.com/openziti/foundation/transport/tcp"
 	"github.com/openziti/foundation/transport/tls"
 	"github.com/openziti/foundation/transport/transwarp"
+	"github.com/openziti/foundation/transport/transwarptls"
 	"github.com/openziti/foundation/transport/wss"
 	"github.com/openziti/ziti/ziti-fabric/subcmd"
 	_ "github.com/openziti/ziti/ziti-fabric/subcmd/trace"
@@ -37,6 +38,7 @@ func init() {
 	transport.AddAddressParser(tls.AddressParser{})
 	transport.AddAddressParser(tcp.AddressParser{})
 	transport.AddAddressParser(transwarp.AddressParser{})
+	transport.AddAddressParser(transwarptls.AddressParser{})
 	transport.AddAddressParser(wss.AddressParser{})
 }

@@ -24,6 +24,7 @@ import (
 	"github.com/openziti/foundation/transport/tcp"
 	"github.com/openziti/foundation/transport/tls"
 	"github.com/openziti/foundation/transport/transwarp"
+	"github.com/openziti/foundation/transport/transwarptls"
 	"github.com/openziti/foundation/transport/ws"
 	"github.com/openziti/foundation/transport/wss"
 	"github.com/openziti/ziti/common/version"
@@ -39,6 +40,7 @@ func init() {
 	transport.AddAddressParser(tls.AddressParser{})
 	transport.AddAddressParser(tcp.AddressParser{})
 	transport.AddAddressParser(transwarp.AddressParser{})
+	transport.AddAddressParser(transwarptls.AddressParser{})
 	transport.AddAddressParser(ws.AddressParser{})
 	transport.AddAddressParser(wss.AddressParser{})

@@ -18,8 +18,11 @@ package subcmd

 import (
 	"github.com/michaelquigley/pfxlog"
+	"github.com/openziti/edge/edge_common"
+	"github.com/openziti/edge/router/fabric"
 	"github.com/openziti/edge/router/xgress_edge"
 	"github.com/openziti/edge/router/xgress_edge_transport"
+	"github.com/openziti/edge/router/xgress_edge_tunnel"
 	"github.com/openziti/fabric/router"
 	"github.com/openziti/fabric/router/xgress"
 	"github.com/openziti/foundation/agent"
@@ -28,6 +31,9 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 	"io"
+	"os"
+	"os/signal"
+	"syscall"
 )

 func init() {
@@ -42,6 +48,7 @@ var runCmd = &cobra.Command{
 }

 func run(cmd *cobra.Command, args []string) {
+
 	logrus.WithField("version", version.GetVersion()).
 		WithField("go-version", version.GetGoVersion()).
 		WithField("os", version.GetOS()).
@@ -69,15 +76,25 @@ func run(cmd *cobra.Command, args []string) {

 		config.SetFlags(getFlags(cmd))

-		xgressEdgeFactory := xgress_edge.NewFactory(config, version.GetCmdBuildInfo())
-		xgress.GlobalRegistry().Register("edge", xgressEdgeFactory)
+		stateManager := fabric.NewStateManager()
+
+		xgressEdgeFactory := xgress_edge.NewFactory(config, version.GetCmdBuildInfo(), stateManager)
+		xgress.GlobalRegistry().Register(edge_common.EdgeBinding, xgressEdgeFactory)
 		if err := r.RegisterXctrl(xgressEdgeFactory); err != nil {
 			logrus.Panicf("error registering edge in framework (%v)", err)
 		}

-		xgressEdgeTransportFactory := xgress_edge_transport.NewFactory(config.Id, r)
+		xgressEdgeTransportFactory := xgress_edge_transport.NewFactory(r)
 		xgress.GlobalRegistry().Register(xgress_edge_transport.BindingName, xgressEdgeTransportFactory)

+		xgressEdgeTunnelFactory := xgress_edge_tunnel.NewFactory(config, stateManager)
+		xgress.GlobalRegistry().Register(edge_common.TunnelBinding, xgressEdgeTunnelFactory)
+		if err := r.RegisterXctrl(xgressEdgeTunnelFactory); err != nil {
+			logrus.Panicf("error registering edge tunnel in framework (%v)", err)
+		}
+
+		go waitForShutdown(r)
+
 		if err := r.Run(); err != nil {
 			logrus.WithError(err).Fatal("error starting")
 		}
@@ -94,3 +111,17 @@ func getFlags(cmd *cobra.Command) map[string]*pflag.Flag {
 	})
 	return ret
 }
+
+func waitForShutdown(r *router.Router) {
+
+	ch := make(chan os.Signal, 1)
+	signal.Notify(ch, os.Interrupt, syscall.SIGTERM)
+
+	<-ch
+
+	pfxlog.Logger().Info("shutting down ziti-router")
+
+	if err := r.Shutdown(); err != nil {
+		pfxlog.Logger().Info("error encountered during shutdown: %v", err)
+	}
+}
@@ -17,14 +17,12 @@
 package subcmd

 import (
-	"fmt"
-	"github.com/openziti/edge/tunnel/intercept"
 	"github.com/openziti/edge/tunnel/intercept/proxy"
+	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 	"math"
 	"net"
 	"strconv"
-	"strings"
 )

 var runProxyCmd = &cobra.Command{
@@ -41,36 +39,6 @@ func init() {
 }

 func runProxy(_ *cobra.Command, args []string) error {
-	services := make(map[string]*proxy.Service, len(args))
-
-	for _, arg := range args {
-		parts := strings.Split(arg, ":")
-		if len(parts) < 2 || len(parts) > 3 {
-			return fmt.Errorf("invalid argument '%s'", arg)
-		}
-
-		port, err := strconv.Atoi(parts[1])
-		if err != nil {
-			return fmt.Errorf("invalid port specified in '%s'", arg)
-		}
-
-		service := &proxy.Service{
-			Name:     parts[0],
-			Port:     port,
-			Protocol: intercept.TCP,
-		}
-
-		if len(parts) == 3 {
-			protocol := parts[2]
-			if protocol == "udp" {
-				service.Protocol = intercept.UDP
-			} else if protocol != "tcp" {
-				return fmt.Errorf("invalid protocol specified in '%s', must be tcp or udp", arg)
-			}
-		}
-		services[parts[0]] = service
-	}
-
 	// Fiddle with the poll rate and resolver settings if the user didn't wan't anything special.
 	if !root.Flag(svcPollRateFlag).Changed {
 		_ = root.PersistentFlags().Set(svcPollRateFlag, strconv.FormatUint(math.MaxUint32, 10))
@@ -79,9 +47,8 @@ func runProxy(_ *cobra.Command, args []string) error {
 		_ = root.PersistentFlags().Set(resolverCfgFlag, "")
 	}
 	var err error
-	interceptor, err = proxy.New(net.IPv4zero, services)
-	if err != nil {
-		return fmt.Errorf("failed to initialize proxy interceptor: %v", err)
+	if interceptor, err = proxy.New(net.IPv4zero, args); err != nil {
+		return errors.Wrap(err, "failed to initialize proxy interceptor")
 	}
 	return nil
 }
@@ -30,8 +30,9 @@ import (

 type createEdgeRouterOptions struct {
 	edgeOptions
-	roleAttributes []string
-	jwtOutputFile  string
+	isTunnelerEnabled bool
+	roleAttributes    []string
+	jwtOutputFile     string
 }

 func newCreateEdgeRouterCmd(f cmdutil.Factory, out io.Writer, errOut io.Writer) *cobra.Command {
@@ -59,6 +60,7 @@ func newCreateEdgeRouterCmd(f cmdutil.Factory, out io.Writer, errOut io.Writer)
 	// allow interspersing positional args and flags
 	cmd.Flags().SetInterspersed(true)
 	cmd.Flags().StringSliceVarP(&options.roleAttributes, "role-attributes", "a", nil, "Role attributes of the new edge router")
+	cmd.Flags().BoolVarP(&options.isTunnelerEnabled, "tunneler-enabled", "t", false, "Can this edge router be used as a tunneler")
 	cmd.Flags().StringVarP(&options.jwtOutputFile, "jwt-output-file", "o", "", "File to which to output the JWT used for enrolling the edge router")
 	options.AddCommonFlags(cmd)

@@ -69,6 +71,7 @@ func newCreateEdgeRouterCmd(f cmdutil.Factory, out io.Writer, errOut io.Writer)
 func runCreateEdgeRouter(o *createEdgeRouterOptions) error {
 	routerData := gabs.New()
 	setJSONValue(routerData, o.Args[0], "name")
+	setJSONValue(routerData, o.isTunnelerEnabled, "isTunnelerEnabled")
 	setJSONValue(routerData, o.roleAttributes, "roleAttributes")

 	result, err := createEntityOfType("edge-routers", routerData.String(), &o.edgeOptions)
@@ -38,6 +38,7 @@ func newCreatePostureCheckCmd(f cmdutil.Factory, out io.Writer, errOut io.Writer
 	cmd.AddCommand(newCreatePostureCheckDomainCmd(f, out, errOut))
 	cmd.AddCommand(newCreatePostureCheckProcessCmd(f, out, errOut))
 	cmd.AddCommand(newCreatePostureCheckOsCmd(f, out, errOut))
+	cmd.AddCommand(newCreatePostureCheckMfaCmd(f, out, errOut))

 	return cmd
 }
@@ -225,6 +226,7 @@ const (
 	PostureCheckTypeProcess = "PROCESS"
 	PostureCheckTypeMAC     = "MAC"
 	PostureCheckTypeOS      = "OS"
+	PostureCheckTypeMFA 	= "MFA"
 )

 // Returns the normalized Edge API value or empty string
@@ -538,3 +540,62 @@ func parseOsSpec(osSpecStr string) (*osSpec, error) {
 		Versions: cleanVersions,
 	}, nil
 }
+
+
+func newCreatePostureCheckMfaCmd(f cmdutil.Factory, out io.Writer, errOut io.Writer) *cobra.Command {
+	options := createPostureCheckOptions{
+		edgeOptions: edgeOptions{
+			CommonOptions: common.CommonOptions{
+				Factory: f,
+				Out:     out,
+				Err:     errOut,
+			},
+		},
+		tags: make(map[string]string),
+	}
+
+	cmd := &cobra.Command{
+		Use:   "mfa <name>",
+		Short: "creates a posture check indicating client-side mfa is required",
+		Args: func(cmd *cobra.Command, args []string) error {
+			if err := cobra.MinimumNArgs(1)(cmd, args); err != nil {
+				return err
+			}
+
+			return nil
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			options.Cmd = cmd
+			options.Args = args
+
+			err := runCreatePostureCheckMfa(&options)
+			cmdhelper.CheckErr(err)
+		},
+		SuggestFor: []string{},
+	}
+
+	// allow interspersing positional args and flags
+	cmd.Flags().SetInterspersed(true)
+	options.AddCommonFlags(cmd)
+	options.addPostureFlags(cmd)
+
+	return cmd
+}
+func runCreatePostureCheckMfa(o *createPostureCheckOptions) error {
+	entityData := gabs.New()
+	setPostureCheckEntityValues(entityData, o, PostureCheckTypeMFA)
+
+	result, err := createEntityOfType("posture-checks", entityData.String(), &o.edgeOptions)
+
+	if err != nil {
+		panic(err)
+	}
+
+	checkId := result.S("data", "id").Data()
+
+	if _, err = fmt.Fprintf(o.Out, "%v\n", checkId); err != nil {
+		panic(err)
+	}
+
+	return nil
+}
@@ -88,7 +88,7 @@ func runCreateServicePolicy(o *createServicePolicyOptions) error {
 		return err
 	}

-	postureCheckRoles, err := convertNamesToIds(o.postureCheckRoles, "postureChecks", o.edgeOptions)
+	postureCheckRoles, err := convertNamesToIds(o.postureCheckRoles, "posture-checks", o.edgeOptions)
 	if err != nil {
 		return err
 	}
@@ -31,8 +31,9 @@ import (

 type updateEdgeRouterOptions struct {
 	edgeOptions
-	name           string
-	roleAttributes []string
+	name              string
+	isTunnelerEnabled bool
+	roleAttributes    []string
 }

 func newUpdateEdgeRouterCmd(f cmdutil.Factory, out io.Writer, errOut io.Writer) *cobra.Command {
@@ -59,6 +60,7 @@ func newUpdateEdgeRouterCmd(f cmdutil.Factory, out io.Writer, errOut io.Writer)
 	// allow interspersing positional args and flags
 	cmd.Flags().SetInterspersed(true)
 	cmd.Flags().StringVarP(&options.name, "name", "n", "", "Set the name of the edge router")
+	cmd.Flags().BoolVarP(&options.isTunnelerEnabled, "tunneler-enabled", "t", false, "Can this edge router be used as a tunneler")
 	cmd.Flags().StringSliceVarP(&options.roleAttributes, "role-attributes", "a", nil,
 		"Set role attributes of the edge router. Use --role-attributes '' to set an empty list")
 	options.AddCommonFlags(cmd)
@@ -80,6 +82,11 @@ func runUpdateEdgeRouter(o *updateEdgeRouterOptions) error {
 		change = true
 	}

+	if o.Cmd.Flags().Changed("tunneler-enabled") {
+		setJSONValue(entityData, o.isTunnelerEnabled, "isTunnelerEnabled")
+		change = true
+	}
+
 	if o.Cmd.Flags().Changed("role-attributes") {
 		setJSONValue(entityData, o.roleAttributes, "roleAttributes")
 		change = true