From bc0e8c0f26dead690de364410aa658fa6975640c Mon Sep 17 00:00:00 2001 From: Andrew M Date: Mon, 3 Aug 2020 11:24:24 -0400 Subject: [PATCH] update deps for openziti/fabric#109 - add documentation --- doc/release-notes.md | 50 +++++++++++++++++++++++++++++++++++++++++++- etc/001.yml | 16 ++++++++++++++ go.mod | 2 +- go.sum | 2 ++ 4 files changed, 68 insertions(+), 2 deletions(-) diff --git a/doc/release-notes.md b/doc/release-notes.md index d5b0e758b..13f1fad8b 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -3,7 +3,9 @@ # What's New: * End-To-End Encryption Enhancements - * [e2e Service Configuration & Router Termination](https://github.com/openziti/edge/issues/173) + * [e2e Service Configuration & Router Termination](https://github.com/openziti/edge/issues/173) +* Router Scaling Issues + * [Add worker pools for link and xgress dials](https://github.com/openziti/fabric/issues/109) * Bug Fixes: * [#152](https://github.com/openziti/ziti/issues/152) - Fix ziti-router enroll exit code on failure @@ -52,7 +54,53 @@ POST /services "encryptionRequired": true } ``` +## Router Scaling Issues +When scaling Ziti Routers it was possible that numerous requests to +complete xgress routes or establish links between routers could block +the control plane of a router. This could cause timeouts of other +control messages and delay the establishment of new service routes and +links. This would be especially noticeable when starting multiple +routers at the same time or when a Ziti Controller was restarted with +multiple routers already connected. + +To alleviate control channel congestion, worker queues and pools +have been added to xgress and link dial processing. New options are +exposed in the `forwarder` section of router configuration files to +control the queue and worker pool. + +The new settings are: + +* `xgressDialQueueLength` +* `xgressDialWorkerCount` +* `linkDialQueueLength` +* `linkDialWorkerCount` + +...and are explained in the following example. + +##### Example Router Configuration Section: +``` +forwarder: + # How frequently does the forwarder probe the link latency. This will ultimately determine the resolution of the + # responsiveness available to smart routing. This resolution comes at the expense of bandwidth utilization for the + # probes, control plane utilization, and CPU utilization processing the results. + # + latencyProbeInterval: 1000 + # How many xgress dials can be queued for processing by the xgress dial workers. An xgress dial occurs + # for services that have a terminator egress specified with an xgress binding (e.g. transport) + # (minimum 1, max 10000, default 1000) + xgressDialQueueLength: 1000 + # The number of xgress dial workers used to process the xgress dial queue. + # (minimum 1, max 10000, default 10) + xgressDialWorkerCount: 10 + # How many link dials can be queued for processing by the link dial workers. An link dial occurs + # when a router is notified of a new router by the controller. + # (minimum 1, max 10000, default 1000) + linkDialQueueLength: 1000 + # The number of link dial workers used to process the link dial queue. + # (minimum 1, max 10000, default 10) + linkDialWorkerCount: 10 +``` # Release 0.15.2 diff --git a/etc/001.yml b/etc/001.yml index 0f26dde47..ad728caad 100644 --- a/etc/001.yml +++ b/etc/001.yml @@ -14,6 +14,20 @@ forwarder: # probes, control plane utilization, and CPU utilization processing the results. # latencyProbeInterval: 1000 + # How many xgress dials can be queued for processing by the xgress dial workers. An xgress dial occurs + # for services that have a terminator egress specified with an xgress binding (e.g. transport) + # (minimum 1, max 10000, default 1000) + xgressDialQueueLength: 1000 + # The number of xgress dial workers used to process the xgress dial queue. + # (minimum 1, max 10000, default 10) + xgressDialWorkerCount: 10 + # How many link dials can be queued for processing by the link dial workers. An link dial occurs + # when a router is notified of a new router by the controller. + # (minimum 1, max 10000, default 1000) + linkDialQueueLength: 1000 + # The number of link dial workers used to process the link dial queue. + # (minimum 1, max 10000, default 10) + linkDialWorkerCount: 10 #trace: # path: 001.trace @@ -31,6 +45,8 @@ link: dialers: - binding: transport + + listeners: # basic ssh proxy - binding: proxy diff --git a/go.mod b/go.mod index 6e1ff976c..5d15ec24f 100644 --- a/go.mod +++ b/go.mod @@ -26,7 +26,7 @@ require ( github.com/keybase/go-ps v0.0.0-20190827175125-91aafc93ba19 github.com/michaelquigley/pfxlog v0.0.0-20190813191113-2be43bd0dccc github.com/openziti/edge v0.15.30 - github.com/openziti/fabric v0.11.58 + github.com/openziti/fabric v0.11.59 github.com/openziti/foundation v0.11.16 github.com/openziti/sdk-golang v0.13.27 github.com/pborman/uuid v1.2.0 diff --git a/go.sum b/go.sum index b2c6e4632..0eeaad327 100644 --- a/go.sum +++ b/go.sum @@ -321,6 +321,8 @@ github.com/openziti/edge v0.15.30 h1:5JXP/c/AxzZxm37d3nVDzlys8WJ2Th2p755xGpE5y7Y github.com/openziti/edge v0.15.30/go.mod h1:Me4HblJi5+ZC29sii2QpwVYgrC92Y2PYTrtFAe0nIW0= github.com/openziti/fabric v0.11.58 h1:wPiUZxCQrJm+t9envMf0ksfgEjCWSI97KRmeG1W1R94= github.com/openziti/fabric v0.11.58/go.mod h1:KJN8zm/krVb4aeMRR/8QcEszHQOoaGxUmE+jm88K4iA= +github.com/openziti/fabric v0.11.59 h1:DwZrSHPHakMScD0w7A01dCV8bbN6nE1hRsdJDC5uKuA= +github.com/openziti/fabric v0.11.59/go.mod h1:KJN8zm/krVb4aeMRR/8QcEszHQOoaGxUmE+jm88K4iA= github.com/openziti/foundation v0.11.16 h1:8pWIEHGM6Stmc9QxW/R+saRzxkzDsTDlAcyO21qrh4k= github.com/openziti/foundation v0.11.16/go.mod h1:4Jgs3HGDiP1aVk5nNzXJqwJwWhrjOYy1F+4Xtxcgxvo= github.com/openziti/sdk-golang v0.13.27 h1:J0LSHHplqZiJLI6f22bo40+94n3k4QKQHSjvBM9DTX8=