From 2b7db109137cb68e3103fb7f4e8a41535c611cac Mon Sep 17 00:00:00 2001 From: Paul Lorenz Date: Mon, 7 Mar 2022 12:22:17 -0500 Subject: [PATCH] Allow routers more control over link management (#318) * Allow routers more control over link management --- controller/api_impl/link_api_model.go | 1 + controller/api_impl/router_api_model.go | 11 + controller/controller.go | 3 +- controller/handler_common/common.go | 5 +- controller/handler_ctrl/accept.go | 31 +- controller/handler_ctrl/bind.go | 22 +- controller/handler_ctrl/connect.go | 5 +- controller/handler_ctrl/fault.go | 26 +- .../{link.go => link_connected.go} | 24 +- controller/handler_ctrl/router_link.go | 68 ++ controller/idgen/id.go | 10 + controller/network/assembly.go | 28 +- controller/network/inspect.go | 26 +- controller/network/link.go | 4 +- controller/network/link_controller.go | 203 ++--- controller/network/network.go | 33 +- controller/network/path_test.go | 36 +- controller/network/route_perf_test.go | 8 +- controller/network/router.go | 97 +-- events/metrics.go | 2 +- pb/ctrl_pb/ctrl.pb.go | 736 +++++++++++++----- pb/ctrl_pb/ctrl.proto | 28 +- pb/ctrl_pb/decoder.go | 49 +- pb/ctrl_pb/impl.go | 8 +- rest_model/inspect_response_value.go | 6 +- rest_model/link_detail.go | 17 + rest_model/links.go | 5 - rest_model/router_detail.go | 68 +- rest_model/router_listener.go | 112 +++ rest_model/terminator_cost_map.go | 5 - rest_model/terminator_precedence.go | 8 +- rest_model/terminator_precedence_map.go | 5 - rest_server/embedded_spec.go | 66 +- router/accepter.go | 10 +- router/handler_ctrl/bind.go | 12 +- router/handler_ctrl/dial.go | 87 ++- router/handler_ctrl/fault.go | 70 ++ router/handler_ctrl/inspect.go | 36 +- router/handler_link/bind.go | 107 ++- router/handler_link/close.go | 42 +- router/handler_link/control.go | 2 +- router/link_registry.go | 248 ++++++ router/router.go | 57 +- router/xlink/xlink.go | 42 +- router/xlink_transport/config.go | 10 + router/xlink_transport/dialer.go | 52 +- router/xlink_transport/factory.go | 18 +- router/xlink_transport/listener.go | 48 +- router/xlink_transport/xlink.go | 31 +- router/xlink_transport/xlink_split.go | 33 +- specs/swagger.yml | 25 +- tests/context.go | 28 +- tests/control.go | 21 + tests/link_test.go | 311 +++++++- tests/testdata/config/router-1.yml | 2 + tests/testdata/config/router-2.yml | 54 ++ tests/testutil/control.go | 25 + tests/testutil/message_collector.go | 59 ++ tests/testutil/timeout_underlay.go | 42 + trace/controller.go | 24 +- 60 files changed, 2572 insertions(+), 680 deletions(-) rename controller/handler_ctrl/{link.go => link_connected.go} (65%) create mode 100644 controller/handler_ctrl/router_link.go create mode 100644 rest_model/router_listener.go create mode 100644 router/handler_ctrl/fault.go create mode 100644 router/link_registry.go create mode 100644 tests/control.go create mode 100644 tests/testdata/config/router-2.yml create mode 100644 tests/testutil/control.go create mode 100644 tests/testutil/message_collector.go create mode 100644 tests/testutil/timeout_underlay.go diff --git a/controller/api_impl/link_api_model.go b/controller/api_impl/link_api_model.go index 54086ab4c..cd112cf89 100644 --- a/controller/api_impl/link_api_model.go +++ b/controller/api_impl/link_api_model.go @@ -61,6 +61,7 @@ func MapLinkToRestModel(_ *network.Network, _ api.RequestContext, link *network. SourceRouter: ToEntityRef(link.Src.Name, link.Src, RouterLinkFactory), State: &linkStateStr, StaticCost: &staticCost, + Type: &link.Type, } return ret, nil } diff --git a/controller/api_impl/router_api_model.go b/controller/api_impl/router_api_model.go index de05abaf1..8f16314a4 100644 --- a/controller/api_impl/router_api_model.go +++ b/controller/api_impl/router_api_model.go @@ -141,5 +141,16 @@ func MapRouterToRestModel(n *network.Network, _ api.RequestContext, router *netw NoTraversal: &router.NoTraversal, } + if connected != nil { + for _, listener := range connected.Listeners { + advAddr := listener.AdvertiseAddress() + linkType := listener.Type() + ret.ListenerAddresses = append(ret.ListenerAddresses, &rest_model.RouterListener{ + Address: &advAddr, + Type: &linkType, + }) + } + } + return ret, nil } diff --git a/controller/controller.go b/controller/controller.go index a377ff110..ec3f2c44b 100644 --- a/controller/controller.go +++ b/controller/controller.go @@ -231,7 +231,6 @@ func (c *Controller) startProfiling() { } func (c *Controller) loadEventHandlers() { - if e, ok := c.config.src["events"]; ok { if em, ok := e.(map[interface{}]interface{}); ok { for k, v := range em { @@ -250,7 +249,7 @@ func (c *Controller) registerXts() { } func (c *Controller) registerComponents() error { - c.ctrlConnectHandler = handler_ctrl.NewConnectHandler(c.config.Id, c.network, c.xctrls) + c.ctrlConnectHandler = handler_ctrl.NewConnectHandler(c.config.Id, c.network) c.mgmtConnectHandler = handler_mgmt.NewConnectHandler(c.config.Id, c.network) //add default REST XWeb diff --git a/controller/handler_common/common.go b/controller/handler_common/common.go index c227581d1..94e450786 100644 --- a/controller/handler_common/common.go +++ b/controller/handler_common/common.go @@ -20,6 +20,7 @@ func SendResult(request *channel.Message, ch channel.Channel, message string, su response := channel.NewResult(success, message) response.ReplyTo(request) - _ = ch.Send(response) - log.Debug("success") + if err := ch.Send(response); err != nil { + log.WithError(err).Error("failed to send result") + } } diff --git a/controller/handler_ctrl/accept.go b/controller/handler_ctrl/accept.go index b7d7ca4aa..1f66650ba 100644 --- a/controller/handler_ctrl/accept.go +++ b/controller/handler_ctrl/accept.go @@ -17,10 +17,13 @@ package handler_ctrl import ( + "github.com/golang/protobuf/proto" "github.com/michaelquigley/pfxlog" "github.com/openziti/channel" "github.com/openziti/fabric/controller/network" "github.com/openziti/fabric/controller/xctrl" + "github.com/openziti/fabric/pb/ctrl_pb" + "github.com/openziti/foundation/transport" "github.com/pkg/errors" ) @@ -67,11 +70,11 @@ func (self *CtrlAccepter) Run() { } func (self *CtrlAccepter) Bind(binding channel.Binding) error { - log := pfxlog.Logger() - binding.GetChannel().SetLogicalName(binding.GetChannel().Id().Token) ch := binding.GetChannel() + log := pfxlog.Logger().WithField("routerId", ch.Id().Token) + if r, err := self.network.GetRouter(ch.Id().Token); err == nil { if ch.Underlay().Headers() != nil { if versionValue, found := ch.Underlay().Headers()[channel.HelloVersionHeader]; found { @@ -83,10 +86,26 @@ func (self *CtrlAccepter) Bind(binding channel.Binding) error { } else { return errors.New("no version info header, closing router connection") } - - if listenerValue, found := ch.Underlay().Headers()[channel.HelloRouterAdvertisementsHeader]; found { - listenerString := string(listenerValue) - r.AdvertisedListener = listenerString + r.Listeners = nil + if val, found := ch.Underlay().Headers()[int32(ctrl_pb.ContentType_ListenersHeader)]; found { + log.Debug("router reported listeners using listeners header") + listeners := &ctrl_pb.Listeners{} + if err := proto.Unmarshal(val, listeners); err != nil { + log.WithError(err).Error("unable to unmarshall listeners value") + } else { + for _, listener := range listeners.Listeners { + log.WithField("address", listener.GetAddress()).WithField("type", listener.GetType()).Debug("router listener") + r.AddLinkListener(listener.GetAddress(), listener.GetType()) + } + } + } else if listenerValue, found := ch.Underlay().Headers()[channel.HelloRouterAdvertisementsHeader]; found { + log.Debug("router reported listener using advertisement header") + addr := string(listenerValue) + linkType := "default" + if addr, _ := transport.ParseAddress(addr); addr != nil { + linkType = addr.Type() + } + r.AddLinkListener(addr, linkType) } else { log.Warn("no advertised listeners") } diff --git a/controller/handler_ctrl/bind.go b/controller/handler_ctrl/bind.go index b7cd776a6..0b997c1fb 100644 --- a/controller/handler_ctrl/bind.go +++ b/controller/handler_ctrl/bind.go @@ -45,7 +45,8 @@ func (self *bindHandler) BindChannel(binding channel.Binding) error { binding.AddTypedReceiveHandler(newCreateTerminatorHandler(self.network, self.router)) binding.AddTypedReceiveHandler(newRemoveTerminatorHandler(self.network)) binding.AddTypedReceiveHandler(newUpdateTerminatorHandler(self.network)) - binding.AddTypedReceiveHandler(newLinkHandler(self.router, self.network)) + binding.AddTypedReceiveHandler(newLinkConnectedHandler(self.router, self.network)) + binding.AddTypedReceiveHandler(newRouterLinkHandler(self.router, self.network)) binding.AddTypedReceiveHandler(newVerifyLinkHandler(self.router, self.network)) binding.AddTypedReceiveHandler(newFaultHandler(self.router, self.network)) binding.AddTypedReceiveHandler(newMetricsHandler(self.network)) @@ -56,11 +57,17 @@ func (self *bindHandler) BindChannel(binding channel.Binding) error { binding.AddPeekHandler(metrics2.NewCtrlChannelPeekHandler(self.router.Id, self.network.GetMetricsRegistry())) if self.router.VersionInfo.HasMinimumVersion("0.18.7") { + roundTripHistogram := self.network.GetMetricsRegistry().Histogram("ctrl.latency:" + self.router.Id) + queueTimeHistogram := self.network.GetMetricsRegistry().Histogram("ctrl.queue_time:" + self.router.Id) latencyHandler := &ctrlChannelLatencyHandler{ - roundTripHistogram: self.network.GetMetricsRegistry().Histogram("ctrl.latency:" + self.router.Id), - queueTimeHistogram: self.network.GetMetricsRegistry().Histogram("ctrl.queue_time:" + self.router.Id), + roundTripHistogram: roundTripHistogram, + queueTimeHistogram: queueTimeHistogram, } - latency.AddLatencyProbe(binding.GetChannel(), binding, self.network.GetOptions().CtrlChanLatencyInterval/time.Duration(10), 10, latencyHandler) + latency.AddLatencyProbe(binding.GetChannel(), binding, self.network.GetOptions().CtrlChanLatencyInterval/time.Duration(10), 10, latencyHandler.HandleLatency) + binding.AddCloseHandler(channel.CloseHandlerF(func(ch channel.Channel) { + roundTripHistogram.Dispose() + queueTimeHistogram.Dispose() + })) } xctrlDone := make(chan struct{}) @@ -85,15 +92,10 @@ type ctrlChannelLatencyHandler struct { queueTimeHistogram metrics.Histogram } -func (self *ctrlChannelLatencyHandler) LatencyReported(latencyType latency.Type, elapsed time.Duration) { +func (self *ctrlChannelLatencyHandler) HandleLatency(latencyType latency.Type, elapsed time.Duration) { if latencyType == latency.RoundTripType { self.roundTripHistogram.Update(elapsed.Nanoseconds()) } else if latencyType == latency.BeforeSendType { self.queueTimeHistogram.Update(elapsed.Nanoseconds()) } } - -func (self *ctrlChannelLatencyHandler) ChannelClosed() { - self.roundTripHistogram.Dispose() - self.queueTimeHistogram.Dispose() -} diff --git a/controller/handler_ctrl/connect.go b/controller/handler_ctrl/connect.go index 9358367d6..c2ba727c3 100644 --- a/controller/handler_ctrl/connect.go +++ b/controller/handler_ctrl/connect.go @@ -23,7 +23,6 @@ import ( "github.com/michaelquigley/pfxlog" "github.com/openziti/channel" "github.com/openziti/fabric/controller/network" - "github.com/openziti/fabric/controller/xctrl" "github.com/openziti/foundation/identity/identity" "github.com/openziti/foundation/util/errorz" "github.com/openziti/foundation/util/stringz" @@ -33,14 +32,12 @@ import ( type ConnectHandler struct { identity identity.Identity network *network.Network - xctrls []xctrl.Xctrl } -func NewConnectHandler(identity identity.Identity, network *network.Network, xctrls []xctrl.Xctrl) *ConnectHandler { +func NewConnectHandler(identity identity.Identity, network *network.Network) *ConnectHandler { return &ConnectHandler{ identity: identity, network: network, - xctrls: xctrls, } } diff --git a/controller/handler_ctrl/fault.go b/controller/handler_ctrl/fault.go index 4c6f0447c..afefc3faa 100644 --- a/controller/handler_ctrl/fault.go +++ b/controller/handler_ctrl/fault.go @@ -20,9 +20,10 @@ import ( "errors" "github.com/golang/protobuf/proto" "github.com/michaelquigley/pfxlog" + "github.com/openziti/channel" + "github.com/openziti/channel/protobufs" "github.com/openziti/fabric/controller/network" "github.com/openziti/fabric/pb/ctrl_pb" - "github.com/openziti/channel" "strings" ) @@ -51,17 +52,32 @@ func (h *faultHandler) HandleReceive(msg *channel.Message, ch channel.Channel) { go h.handleFault(msg, ch, fault) } -func (h *faultHandler) handleFault(msg *channel.Message, ch channel.Channel, fault *ctrl_pb.Fault) { +func (h *faultHandler) handleFault(_ *channel.Message, ch channel.Channel, fault *ctrl_pb.Fault) { log := pfxlog.ContextLogger(ch.Label()) switch fault.Subject { case ctrl_pb.FaultSubject_LinkFault: linkId := fault.Id - if err := h.network.LinkConnected(linkId, false); err == nil { - if link, found := h.network.GetLink(linkId); found { + if link, found := h.network.GetLink(linkId); found { + wasConnected := link.IsUsable() + if err := h.network.LinkConnected(linkId, false); err == nil { h.network.LinkChanged(link) + otherRouter := link.Src + if link.Src.Id == h.r.Id { + otherRouter = link.Dst + } + + if wasConnected { + if ctrl := otherRouter.Control; ctrl != nil && otherRouter.Connected.Get() { + if err := protobufs.MarshalTyped(fault).Send(ctrl); err != nil { + log.WithField("linkId", linkId).WithField("routerId", otherRouter.Id). + WithError(err).Error("failed to forward link fault to other router") + } + } + } + + log.Infof("link fault [l/%s]", linkId) } - log.Infof("link fault [l/%s]", linkId) } case ctrl_pb.FaultSubject_IngressFault: diff --git a/controller/handler_ctrl/link.go b/controller/handler_ctrl/link_connected.go similarity index 65% rename from controller/handler_ctrl/link.go rename to controller/handler_ctrl/link_connected.go index 589a5084b..fc028757c 100644 --- a/controller/handler_ctrl/link.go +++ b/controller/handler_ctrl/link_connected.go @@ -19,41 +19,41 @@ package handler_ctrl import ( "github.com/golang/protobuf/proto" "github.com/michaelquigley/pfxlog" + "github.com/openziti/channel" "github.com/openziti/fabric/controller/network" "github.com/openziti/fabric/pb/ctrl_pb" - "github.com/openziti/channel" ) -type linkHandler struct { +type linkConnectedHandler struct { r *network.Router network *network.Network } -func newLinkHandler(r *network.Router, network *network.Network) *linkHandler { - return &linkHandler{r: r, network: network} +func newLinkConnectedHandler(r *network.Router, network *network.Network) *linkConnectedHandler { + return &linkConnectedHandler{r: r, network: network} } -func (h *linkHandler) ContentType() int32 { - return int32(ctrl_pb.ContentType_LinkType) +func (h *linkConnectedHandler) ContentType() int32 { + return int32(ctrl_pb.ContentType_LinkConnectedType) } -func (h *linkHandler) HandleReceive(msg *channel.Message, ch channel.Channel) { +func (h *linkConnectedHandler) HandleReceive(msg *channel.Message, ch channel.Channel) { log := pfxlog.ContextLogger(ch.Label()) - link := &ctrl_pb.Link{} + link := &ctrl_pb.LinkConnected{} if err := proto.Unmarshal(msg.Body, link); err != nil { log.WithError(err).Error("failed to unmarshal link message") return } - go h.HandleLink(msg, ch, link) + go h.HandleLink(ch, link) } -func (h *linkHandler) HandleLink(msg *channel.Message, ch channel.Channel, link *ctrl_pb.Link) { - log := pfxlog.ContextLogger(ch.Label()) +func (h *linkConnectedHandler) HandleLink(ch channel.Channel, link *ctrl_pb.LinkConnected) { + log := pfxlog.ContextLogger(ch.Label()).WithField("linkId", link.Id) if err := h.network.LinkConnected(link.Id, true); err == nil { - log.Infof("link connected [l/%s]", link.Id) + log.Info("link connected") } else { log.WithError(err).Error("unexpected error marking link connected") } diff --git a/controller/handler_ctrl/router_link.go b/controller/handler_ctrl/router_link.go new file mode 100644 index 000000000..4555a5c4d --- /dev/null +++ b/controller/handler_ctrl/router_link.go @@ -0,0 +1,68 @@ +/* + Copyright NetFoundry, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package handler_ctrl + +import ( + "github.com/golang/protobuf/proto" + "github.com/michaelquigley/pfxlog" + "github.com/openziti/channel" + "github.com/openziti/fabric/controller/network" + "github.com/openziti/fabric/pb/ctrl_pb" +) + +type routerLinkHandler struct { + r *network.Router + network *network.Network +} + +func newRouterLinkHandler(r *network.Router, network *network.Network) *routerLinkHandler { + return &routerLinkHandler{r: r, network: network} +} + +func (h *routerLinkHandler) ContentType() int32 { + return int32(ctrl_pb.ContentType_RouterLinksType) +} + +func (h *routerLinkHandler) HandleReceive(msg *channel.Message, ch channel.Channel) { + log := pfxlog.ContextLogger(ch.Label()) + + link := &ctrl_pb.RouterLinks{} + if err := proto.Unmarshal(msg.Body, link); err != nil { + log.WithError(err).Error("failed to unmarshal link message") + return + } + + go h.HandleLinks(ch, link) +} + +func (h *routerLinkHandler) HandleLinks(ch channel.Channel, links *ctrl_pb.RouterLinks) { + log := pfxlog.ContextLogger(ch.Label()).WithField("routerId", ch.Id().Token) + + for _, link := range links.Links { + linkLog := log.WithField("linkId", link.Id). + WithField("destRouterId", link.DestRouterId) + + created, err := h.network.NotifyExistingLink(link.Id, link.LinkType, h.r, link.DestRouterId) + if err != nil { + linkLog.WithError(err).Error("unexpected error adding router reported link") + } else if created { + linkLog.Info("router reported link added") + } else { + linkLog.Info("router reported link already known") + } + } +} diff --git a/controller/idgen/id.go b/controller/idgen/id.go index 71ac29127..ac141570f 100644 --- a/controller/idgen/id.go +++ b/controller/idgen/id.go @@ -3,7 +3,10 @@ package idgen import ( "crypto/rand" "encoding/binary" + "github.com/dineshappavoo/basex" + "github.com/google/uuid" "github.com/teris-io/shortid" + "math/big" ) const Alphabet = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ.-" @@ -52,3 +55,10 @@ func (self *shortIdGenerator) NextAlphaNumericPrefixedId() (string, error) { } } } + +func NewUUIDString() (string, error) { + id := uuid.New() + v := &big.Int{} + v.SetBytes(id[:]) + return basex.EncodeInt(v) +} diff --git a/controller/network/assembly.go b/controller/network/assembly.go index fdd5710e9..7ad13bc60 100644 --- a/controller/network/assembly.go +++ b/controller/network/assembly.go @@ -27,27 +27,39 @@ func (network *Network) assemble() { log := pfxlog.Logger() if network.Routers.connectedCount() > 1 { - log.Debugf("assembling with [%d] routers", network.Routers.connectedCount()) + log.Tracef("assembling with [%d] routers", network.Routers.connectedCount()) missingLinks, err := network.linkController.missingLinks(network.Routers.allConnected(), network.options.PendingLinkTimeout) if err == nil { for _, missingLink := range missingLinks { network.linkController.add(missingLink) - dial := &ctrl_pb.Dial{ - LinkId: missingLink.Id, - Address: missingLink.Dst.AdvertisedListener, - RouterId: missingLink.Dst.Id, - } + for _, listener := range missingLink.Dst.Listeners { + if listener.Type() != missingLink.Type { + continue + } + dial := &ctrl_pb.Dial{ + LinkId: missingLink.Id, + Address: listener.AdvertiseAddress(), + RouterId: missingLink.Dst.Id, + LinkType: listener.Type(), + } - if err := protobufs.MarshalTyped(dial).Send(missingLink.Src.Control); err != nil { - log.WithError(err).Error("unexpected error sending dial") + if versionInfo := missingLink.Dst.VersionInfo; versionInfo != nil { + dial.RouterVersion = missingLink.Dst.VersionInfo.Version + } + + if err := protobufs.MarshalTyped(dial).Send(missingLink.Src.Control); err != nil { + log.WithError(err).Error("unexpected error sending dial") + } } } } else { log.WithField("err", err).Error("missing link enumeration failed") } + + network.linkController.clearExpiredPending(network.options.PendingLinkTimeout) } } diff --git a/controller/network/inspect.go b/controller/network/inspect.go index 2e3c143c9..68ef97f7f 100644 --- a/controller/network/inspect.go +++ b/controller/network/inspect.go @@ -17,12 +17,14 @@ package network import ( + "encoding/json" "fmt" "github.com/michaelquigley/pfxlog" "github.com/openziti/channel/protobufs" "github.com/openziti/fabric/pb/ctrl_pb" "github.com/openziti/foundation/util/concurrenz" "regexp" + "strings" "sync" "time" ) @@ -30,7 +32,7 @@ import ( type InspectResultValue struct { AppId string Name string - Value string + Value interface{} } type InspectResult struct { @@ -113,7 +115,7 @@ func (ctx *inspectRequestContext) inspectLocal() { for _, requested := range ctx.requestedValues { result := ctx.network.Inspect(requested) if result != nil { - ctx.appendValue(ctx.network.GetAppId(), requested, *result) + ctx.appendValue(ctx.network.GetAppId(), requested, result) } } close(notifier) @@ -158,11 +160,27 @@ func (ctx *inspectRequestContext) handleRouterMessaging(router *Router, notifier } for _, val := range resp.Values { - ctx.appendValue(router.Id, val.Name, val.Value) + handled := false + if strings.HasPrefix(val.Value, "{") { + mapVal := map[string]interface{}{} + if err := json.Unmarshal([]byte(val.Value), mapVal); err == nil { + ctx.appendValue(router.Id, val.Name, mapVal) + handled = true + } + } else if strings.HasPrefix(val.Value, "[") { + var arrayVal []interface{} + if err := json.Unmarshal([]byte(val.Value), &arrayVal); err == nil { + ctx.appendValue(router.Id, val.Name, arrayVal) + handled = true + } + } + if !handled { + ctx.appendValue(router.Id, val.Name, val.Value) + } } } -func (ctx *inspectRequestContext) appendValue(appId string, name string, value string) { +func (ctx *inspectRequestContext) appendValue(appId string, name string, value interface{}) { ctx.lock.Lock() defer ctx.lock.Unlock() if !ctx.complete { diff --git a/controller/network/link.go b/controller/network/link.go index 77cf9c153..2461e125b 100644 --- a/controller/network/link.go +++ b/controller/network/link.go @@ -27,6 +27,7 @@ type Link struct { Id string Src *Router Dst *Router + Type string state []*LinkState down bool StaticCost int32 @@ -37,9 +38,10 @@ type Link struct { lock sync.Mutex } -func newLink(id string) *Link { +func newLink(id string, linkType string) *Link { l := &Link{ Id: id, + Type: linkType, state: make([]*LinkState, 0), down: false, StaticCost: 1, diff --git a/controller/network/link_controller.go b/controller/network/link_controller.go index 3e75e0a6f..a1bc0dd54 100644 --- a/controller/network/link_controller.go +++ b/controller/network/link_controller.go @@ -21,12 +21,14 @@ import ( "github.com/openziti/foundation/util/info" "github.com/orcaman/concurrent-map" "math" + "sync" "time" ) type linkController struct { linkTable *linkTable idGenerator idgen.Generator + lock sync.Mutex } func newLinkController() *linkController { @@ -46,6 +48,22 @@ func (linkController *linkController) has(link *Link) bool { return linkController.linkTable.has(link) } +func (linkController *linkController) routerReportedLink(linkId, linkType string, src, dst *Router) (*Link, bool) { + linkController.lock.Lock() + defer linkController.lock.Unlock() + + if link, found := linkController.get(linkId); found { + return link, false + } + + link := newLink(linkId, linkType) + link.Src = src + link.Dst = dst + link.addState(newLinkState(Connected)) + linkController.add(link) + return link, true +} + func (linkController *linkController) get(linkId string) (*Link, bool) { link, found := linkController.linkTable.get(linkId) return link, found @@ -123,16 +141,18 @@ func (linkController *linkController) missingLinks(routers []*Router, pendingTim missingLinks := make([]*Link, 0) for _, srcR := range routers { for _, dstR := range routers { - if srcR != dstR && dstR.AdvertisedListener != "" { - if !linkController.hasLink(srcR, dstR, pendingLimit) { - id, err := linkController.idGenerator.NextAlphaNumericPrefixedId() - if err != nil { - return nil, err + if srcR != dstR && len(dstR.Listeners) > 0 { + for _, listener := range dstR.Listeners { + if !linkController.hasLink(srcR, dstR, listener.Type(), pendingLimit) { + id, err := idgen.NewUUIDString() + if err != nil { + return nil, err + } + link := newLink(id, listener.Type()) + link.Src = srcR + link.Dst = dstR + missingLinks = append(missingLinks, link) } - link := newLink(id) - link.Src = srcR - link.Dst = dstR - missingLinks = append(missingLinks, link) } } } @@ -141,15 +161,28 @@ func (linkController *linkController) missingLinks(routers []*Router, pendingTim return missingLinks, nil } -func (linkController *linkController) hasLink(a, b *Router, pendingLimit int64) bool { - return linkController.hasDirectedLink(a, b, pendingLimit) || linkController.hasDirectedLink(b, a, pendingLimit) +func (linkController *linkController) clearExpiredPending(pendingTimeout time.Duration) { + pendingLimit := info.NowInMilliseconds() - pendingTimeout.Milliseconds() + + toRemove := linkController.linkTable.matching(func(link *Link) bool { + state := link.CurrentState() + return state != nil && state.Mode == Pending && state.Timestamp < pendingLimit + }) + + for _, link := range toRemove { + linkController.remove(link) + } } -func (linkController *linkController) hasDirectedLink(a, b *Router, pendingLimit int64) bool { +func (linkController *linkController) hasLink(a, b *Router, linkType string, pendingLimit int64) bool { + return linkController.hasDirectedLink(a, b, linkType, pendingLimit) || linkController.hasDirectedLink(b, a, linkType, pendingLimit) +} + +func (linkController *linkController) hasDirectedLink(a, b *Router, linkType string, pendingLimit int64) bool { links := a.routerLinks.GetLinks() for _, link := range links { state := link.CurrentState() - if link.Src == a && link.Dst == b && state != nil { + if link.Src == a && link.Dst == b && state != nil && link.Type == linkType { if state.Mode == Connected || (state.Mode == Pending && state.Timestamp > pendingLimit) { return true } @@ -214,140 +247,16 @@ func (lt *linkTable) allInMode(mode LinkMode) []*Link { return links } +func (lt *linkTable) matching(f func(*Link) bool) []*Link { + var links []*Link + for i := range lt.links.IterBuffered() { + if link, ok := i.Val.(*Link); ok && f(link) { + links = append(links, link) + } + } + return links +} + func (lt *linkTable) remove(link *Link) { lt.links.Remove(link.Id) } - -/* - * adjacencyTable - */ - -type adjacencyTable struct { - adjacency cmap.ConcurrentMap // map[Router.Id.Token]*routerLinksTable -} - -func newAdjacencyTable() *adjacencyTable { - return &adjacencyTable{adjacency: cmap.New()} -} - -func (at *adjacencyTable) add(link *Link) { - // src->dst - var rlt *routerLinksTable - if i, found := at.adjacency.Get(link.Src.Id); found { - rlt = i.(*routerLinksTable) - } else { - rlt = newRouterLinksTable() - } - rlt.addLinkForRouter(link.Dst.Id, link) - at.adjacency.Set(link.Src.Id, rlt) - - // dst->src - if i, found := at.adjacency.Get(link.Dst.Id); found { - rlt = i.(*routerLinksTable) - } else { - rlt = newRouterLinksTable() - } - rlt.addLinkForRouter(link.Src.Id, link) - at.adjacency.Set(link.Dst.Id, rlt) -} - -func (at *adjacencyTable) get(routerId string) (*routerLinksTable, bool) { - i, found := at.adjacency.Get(routerId) - if i != nil { - return i.(*routerLinksTable), found - } - return nil, found -} - -func (at *adjacencyTable) remove(link *Link) { - // src->dst - if i, found := at.adjacency.Get(link.Src.Id); found { - rlt := i.(*routerLinksTable) - rlt.removeLinkFromRouter(link.Dst.Id, link) - if rlt.size() > 0 { - at.adjacency.Set(link.Src.Id, rlt) - } else { - at.adjacency.Remove(link.Src.Id) - } - } - - // dst->src - if i, found := at.adjacency.Get(link.Dst.Id); found { - rlt := i.(*routerLinksTable) - rlt.removeLinkFromRouter(link.Src.Id, link) - if rlt.size() > 0 { - at.adjacency.Set(link.Dst.Id, rlt) - } else { - at.adjacency.Remove(link.Dst.Id) - } - } -} - -/* - * routerLinksTable - */ - -type routerLinksTable struct { - routerLinks cmap.ConcurrentMap // map[Router.Id.Token][]*Link -} - -func newRouterLinksTable() *routerLinksTable { - return &routerLinksTable{routerLinks: cmap.New()} -} - -func (rlt *routerLinksTable) addLinkForRouter(routerId string, link *Link) { - var links []*Link - if i, found := rlt.routerLinks.Get(routerId); found { - links = i.([]*Link) - } else { - links = make([]*Link, 0) - } - - links = append(links, link) - rlt.routerLinks.Set(routerId, links) -} - -func (rlt *routerLinksTable) allLinksForRouter(routerId string) ([]*Link, bool) { - if i, found := rlt.routerLinks.Get(routerId); found { - return i.([]*Link), true - } - return nil, false -} - -func (rlt *routerLinksTable) allLinksForAllRouters() []*Link { - allLinks := make([]*Link, 0, rlt.routerLinks.Count()) - for i := range rlt.routerLinks.IterBuffered() { - links := i.Val.([]*Link) - for _, link := range links { - allLinks = append(allLinks, link) - } - } - return allLinks -} - -func (rlt *routerLinksTable) size() int { - return rlt.routerLinks.Count() -} - -func (rlt *routerLinksTable) removeLinkFromRouter(routerId string, link *Link) { - var links []*Link - if i, found := rlt.routerLinks.Get(routerId); found { - links = i.([]*Link) - if len(links) == 1 && links[0] == link { - rlt.routerLinks.Remove(routerId) - - } else { - i := -1 - for j, jLink := range links { - if jLink == link { - i = j - break - } - } - if i != -1 { - links = append(links[:i], links[i+1:]...) - } - rlt.routerLinks.Set(routerId, links) - } - } -} diff --git a/controller/network/network.go b/controller/network/network.go index d7ba5d62a..a9ecacc99 100644 --- a/controller/network/network.go +++ b/controller/network/network.go @@ -250,7 +250,8 @@ func (network *Network) ConnectedRouter(id string) bool { func (network *Network) ConnectRouter(r *Router) { network.Routers.markConnected(r) - network.routerChanged <- r + + time.AfterFunc(250*time.Millisecond, func() { network.routerChanged <- r }) for _, h := range network.routerPresenceHandlers { go h.RouterConnected(r) @@ -304,6 +305,15 @@ func (network *Network) DisconnectRouter(r *Router) { } } +func (network *Network) NotifyExistingLink(id, linkType string, srcRouter *Router, dstRouterId string) (bool, error) { + dst := network.Routers.getConnected(dstRouterId) + if dst == nil { + return false, errors.New("destination router not connected") + } + _, created := network.linkController.routerReportedLink(id, linkType, srcRouter, dst) + return created, nil +} + func (network *Network) LinkConnected(id string, connected bool) error { log := pfxlog.Logger().WithField("linkId", id) @@ -833,11 +843,24 @@ func (network *Network) AcceptMetrics(metrics *metrics_pb.MetricsMessage) { for _, link := range network.GetAllLinksForRouter(router.Id) { metricId := "link." + link.Id + ".latency" + var latencyCost int64 + var found bool if latency, ok := metrics.Histograms[metricId]; ok { + latencyCost = int64(latency.Mean) + found = true + } + + metricId = "link." + link.Id + ".queue_time" + if latency, ok := metrics.Histograms[metricId]; ok { + latencyCost += int64(latency.Mean) + found = true + } + + if found { if link.Src.Id == router.Id { - link.SetSrcLatency(int64(latency.Mean)) // latency is in nanoseconds + link.SetSrcLatency(latencyCost) // latency is in nanoseconds } else if link.Dst.Id == router.Id { - link.SetDstLatency(int64(latency.Mean)) // latency is in nanoseconds + link.SetDstLatency(latencyCost) // latency is in nanoseconds } else { log.Warnf("link not for router") } @@ -895,10 +918,10 @@ func (network *Network) showOptions() { } } -func (network *Network) Inspect(name string) *string { +func (network *Network) Inspect(name string) interface{} { if strings.ToLower(name) == "stackdump" { result := debugz.GenerateStack() - return &result + return result } return nil } diff --git a/controller/network/path_test.go b/controller/network/path_test.go index e8a4c943e..e6d34d6e4 100644 --- a/controller/network/path_test.go +++ b/controller/network/path_test.go @@ -50,7 +50,7 @@ func TestSimplePath2(t *testing.T) { r1 := newRouterForTest("r1", "", transportAddr, nil, 0, false) network.Routers.markConnected(r1) - l0 := newLink("l0") + l0 := newLink("l0", "tls") l0.Src = r0 l0.Dst = r1 l0.addState(newLinkState(Connected)) @@ -116,13 +116,13 @@ func TestTransitPath2(t *testing.T) { r2 := newRouterForTest("r2", "", transportAddr, nil, 0, false) network.Routers.markConnected(r2) - l0 := newLink("l0") + l0 := newLink("l0", "tls") l0.Src = r0 l0.Dst = r1 l0.addState(newLinkState(Connected)) network.linkController.add(l0) - l1 := newLink("l1") + l1 := newLink("l1", "tls") l1.Src = r1 l1.Dst = r2 l1.addState(newLinkState(Connected)) @@ -186,7 +186,7 @@ func newRouterForTest(id string, fingerprint string, advLstnr transport.Address, NoTraversal: noTraversal, } if advLstnr != nil { - r.AdvertisedListener = advLstnr.String() + r.AddLinkListener(advLstnr.String(), advLstnr.Type()) } return r } @@ -256,7 +256,7 @@ func TestShortestPath(t *testing.T) { r3 := newRouterForTest("r3", "", transportAddr, nil, 4, false) network.Routers.markConnected(r3) - link := newLink("l0") + link := newLink("l0", "tls") link.SetStaticCost(2) link.SetDstLatency(10 * 1_000_000) link.SetSrcLatency(11 * 1_000_000) @@ -265,7 +265,7 @@ func TestShortestPath(t *testing.T) { link.addState(newLinkState(Connected)) network.linkController.add(link) - link = newLink("l1") + link = newLink("l1", "tls") link.SetStaticCost(5) link.SetDstLatency(15 * 1_000_000) link.SetSrcLatency(16 * 1_000_000) @@ -274,7 +274,7 @@ func TestShortestPath(t *testing.T) { link.addState(newLinkState(Connected)) network.linkController.add(link) - link = newLink("l2") + link = newLink("l2", "tls") link.SetStaticCost(9) link.SetDstLatency(20 * 1_000_000) link.SetSrcLatency(21 * 1_000_000) @@ -283,7 +283,7 @@ func TestShortestPath(t *testing.T) { link.addState(newLinkState(Connected)) network.linkController.add(link) - link = newLink("l3") + link = newLink("l3", "tls") link.SetStaticCost(13) link.SetDstLatency(25 * 1_000_000) link.SetSrcLatency(26 * 1_000_000) @@ -332,7 +332,7 @@ func TestShortestPathWithUntraversableRouter(t *testing.T) { r3 := newRouterForTest("r3", "", transportAddr, nil, 4, false) network.Routers.markConnected(r3) - link := newLink("l0") + link := newLink("l0", "tls") link.SetStaticCost(2) link.SetDstLatency(10 * 1_000_000) link.SetSrcLatency(11 * 1_000_000) @@ -341,7 +341,7 @@ func TestShortestPathWithUntraversableRouter(t *testing.T) { link.addState(newLinkState(Connected)) network.linkController.add(link) - link = newLink("l1") + link = newLink("l1", "tls") link.SetStaticCost(5) link.SetDstLatency(15 * 1_000_000) link.SetSrcLatency(16 * 1_000_000) @@ -350,7 +350,7 @@ func TestShortestPathWithUntraversableRouter(t *testing.T) { link.addState(newLinkState(Connected)) network.linkController.add(link) - link = newLink("l2") + link = newLink("l2", "tls") link.SetStaticCost(9) link.SetDstLatency(20 * 1_000_000) link.SetSrcLatency(21 * 1_000_000) @@ -359,7 +359,7 @@ func TestShortestPathWithUntraversableRouter(t *testing.T) { link.addState(newLinkState(Connected)) network.linkController.add(link) - link = newLink("l3") + link = newLink("l3", "tls") link.SetStaticCost(13) link.SetDstLatency(25 * 1_000_000) link.SetSrcLatency(26 * 1_000_000) @@ -402,7 +402,7 @@ func TestShortestPathWithOnlyUntraversableRouter(t *testing.T) { r1 := newRouterForTest("r1", "", transportAddr, nil, 2, true) network.Routers.markConnected(r1) - link := newLink("l0") + link := newLink("l0", "tls") link.SetStaticCost(2) link.SetDstLatency(10 * 1_000_000) link.SetSrcLatency(11 * 1_000_000) @@ -444,7 +444,7 @@ func TestShortestPathWithUntraversableEdgeRouters(t *testing.T) { r1 := newRouterForTest("r1", "", transportAddr, nil, 2, true) network.Routers.markConnected(r1) - link := newLink("l0") + link := newLink("l0", "tls") link.SetStaticCost(3) link.SetDstLatency(10 * 1_000_000) link.SetSrcLatency(11 * 1_000_000) @@ -489,7 +489,7 @@ func TestShortestPathWithUntraversableEdgeRoutersAndTraversableMiddle(t *testing r2 := newRouterForTest("r2", "", transportAddr, nil, 3, true) network.Routers.markConnected(r2) - link := newLink("l0") + link := newLink("l0", "tls") link.SetStaticCost(2) link.SetDstLatency(10 * 1_000_000) link.SetSrcLatency(11 * 1_000_000) @@ -498,7 +498,7 @@ func TestShortestPathWithUntraversableEdgeRoutersAndTraversableMiddle(t *testing link.addState(newLinkState(Connected)) network.linkController.add(link) - link = newLink("l1") + link = newLink("l1", "tls") link.SetStaticCost(3) link.SetDstLatency(12 * 1_000_000) link.SetSrcLatency(15 * 1_000_000) @@ -545,7 +545,7 @@ func TestShortestPathWithUntraversableEdgeRoutersAndUntraversableMiddle(t *testi r2 := newRouterForTest("r2", "", transportAddr, nil, 2, true) network.Routers.markConnected(r2) - link := newLink("l0") + link := newLink("l0", "tls") link.SetStaticCost(2) link.SetDstLatency(10 * 1_000_000) link.SetSrcLatency(11 * 1_000_000) @@ -554,7 +554,7 @@ func TestShortestPathWithUntraversableEdgeRoutersAndUntraversableMiddle(t *testi link.addState(newLinkState(Connected)) network.linkController.add(link) - link = newLink("l2") + link = newLink("l2", "tls") link.SetStaticCost(2) link.SetDstLatency(10 * 1_000_000) link.SetSrcLatency(11 * 1_000_000) diff --git a/controller/network/route_perf_test.go b/controller/network/route_perf_test.go index 4904b3f9e..be647ced4 100644 --- a/controller/network/route_perf_test.go +++ b/controller/network/route_perf_test.go @@ -57,7 +57,7 @@ func TestShortestPathAgainstEstablished(t *testing.T) { addLink := func(srcRouter, dstRouter *Router) { if srcRouter != dstRouter { - link := newLink(fmt.Sprintf("link-%04d", linkIdx)) + link := newLink(fmt.Sprintf("link-%04d", linkIdx), "tls") link.SetStaticCost(int32(nextCost())) link.SetDstLatency(nextCost() * 100_000) link.SetSrcLatency(nextCost() * 100_000) @@ -183,7 +183,7 @@ func BenchmarkShortestPathPerfWithRouterChanges(b *testing.B) { addLink := func(srcRouter, dstRouter *Router) { if srcRouter != dstRouter { - link := newLink(fmt.Sprintf("link-%04d", linkIdx)) + link := newLink(fmt.Sprintf("link-%04d", linkIdx), "tls") link.SetStaticCost(int32(nextCost())) link.SetDstLatency(nextCost() * 100_000) link.SetSrcLatency(nextCost() * 100_000) @@ -277,7 +277,7 @@ func BenchmarkShortestPathPerf(b *testing.B) { addLink := func(srcRouter, dstRouter *Router) { if srcRouter != dstRouter { - link := newLink(fmt.Sprintf("link-%04d", linkIdx)) + link := newLink(fmt.Sprintf("link-%04d", linkIdx), "tls") link.SetStaticCost(int32(nextCost())) link.SetDstLatency(nextCost() * 100_000) link.SetSrcLatency(nextCost() * 100_000) @@ -351,7 +351,7 @@ func BenchmarkMoreRealisticShortestPathPerf(b *testing.B) { addLink := func(srcRouter, dstRouter *Router) { if srcRouter != dstRouter { - link := newLink(fmt.Sprintf("link-%04d", linkIdx)) + link := newLink(fmt.Sprintf("link-%04d", linkIdx), "tls") link.SetStaticCost(int32(nextCost())) link.SetDstLatency(nextCost() * 100_000) link.SetSrcLatency(nextCost() * 100_000) diff --git a/controller/network/router.go b/controller/network/router.go index b0ab69f7d..bc4b084d3 100644 --- a/controller/network/router.go +++ b/controller/network/router.go @@ -33,17 +33,22 @@ import ( "go.etcd.io/bbolt" ) +type Listener interface { + AdvertiseAddress() string + Type() string +} + type Router struct { models.BaseEntity - Name string - Fingerprint *string - AdvertisedListener string - Control channel.Channel - Connected concurrenz.AtomicBoolean - VersionInfo *common.VersionInfo - routerLinks RouterLinks - Cost uint16 - NoTraversal bool + Name string + Fingerprint *string + Listeners []Listener + Control channel.Channel + Connected concurrenz.AtomicBoolean + VersionInfo *common.VersionInfo + routerLinks RouterLinks + Cost uint16 + NoTraversal bool } func (entity *Router) fillFrom(_ Controller, _ *bbolt.Tx, boltEntity boltz.Entity) error { @@ -69,6 +74,13 @@ func (entity *Router) toBolt() boltz.Entity { } } +func (entity *Router) AddLinkListener(addr, linkType string) { + entity.Listeners = append(entity.Listeners, linkListener{ + addr: addr, + linkType: linkType, + }) +} + func NewRouter(id, name, fingerprint string, cost uint16, noTraversal bool) *Router { if name == "" { name = id @@ -85,60 +97,6 @@ func NewRouter(id, name, fingerprint string, cost uint16, noTraversal bool) *Rou return result } -type routerCopyOnWriteMap struct { - sync.Mutex - atomic.Value -} - -func newRouterCopyOnWriteMap() *routerCopyOnWriteMap { - result := &routerCopyOnWriteMap{} - result.Store(map[string]*Router{}) - return result -} - -func (self *routerCopyOnWriteMap) Has(id string) bool { - _, found := self.getCurrentMap()[id] - return found -} - -func (self *routerCopyOnWriteMap) Count() int { - return len(self.getCurrentMap()) -} - -func (self *routerCopyOnWriteMap) Get(id string) (*Router, bool) { - r, found := self.getCurrentMap()[id] - return r, found -} - -func (self *routerCopyOnWriteMap) Set(id string, router *Router) { - self.Lock() - defer self.Unlock() - m := self.getMapCopy() - m[id] = router - self.Store(m) -} - -func (self *routerCopyOnWriteMap) Remove(id string) { - self.Lock() - defer self.Unlock() - m := self.getMapCopy() - delete(m, id) - self.Store(m) -} - -func (self *routerCopyOnWriteMap) getCurrentMap() map[string]*Router { - return self.Load().(map[string]*Router) -} - -func (self *routerCopyOnWriteMap) getMapCopy() map[string]*Router { - m := self.getCurrentMap() - mapCopy := map[string]*Router{} - for k, v := range m { - mapCopy[k] = v - } - return mapCopy -} - type RouterController struct { baseController cache cmap.ConcurrentMap @@ -420,3 +378,16 @@ func (self *RouterLinks) Clear() { self.allLinks.Store([]*Link{}) self.linkByRouter.Store(map[string][]*Link{}) } + +type linkListener struct { + addr string + linkType string +} + +func (self linkListener) AdvertiseAddress() string { + return self.addr +} + +func (self linkListener) Type() string { + return self.linkType +} diff --git a/events/metrics.go b/events/metrics.go index dc8330b5e..b4b2d998d 100644 --- a/events/metrics.go +++ b/events/metrics.go @@ -32,7 +32,7 @@ func mapLinkMetrics(_ *metrics_pb.MetricsMessage, event *MetricsEvent) { if strings.HasPrefix(event.Metric, "link.") { var name, linkId string - if strings.HasSuffix(event.Metric, "latency") { + if strings.HasSuffix(event.Metric, "latency") || strings.HasSuffix(event.Metric, "queue_time") { name, linkId = ExtractId(event.Metric, "link.", 1) } else { name, linkId = ExtractId(event.Metric, "link.", 2) diff --git a/pb/ctrl_pb/ctrl.pb.go b/pb/ctrl_pb/ctrl.pb.go index 130b9303b..49f3c6f2a 100644 --- a/pb/ctrl_pb/ctrl.pb.go +++ b/pb/ctrl_pb/ctrl.pb.go @@ -28,7 +28,7 @@ const ( // defined in ctrl_msg/messages.go now // CircuitSuccessType = 1001; ContentType_DialType ContentType = 1002 - ContentType_LinkType ContentType = 1003 + ContentType_LinkConnectedType ContentType = 1003 ContentType_FaultType ContentType = 1004 ContentType_RouteType ContentType = 1005 ContentType_UnrouteType ContentType = 1006 @@ -46,6 +46,8 @@ const ( ContentType_VerifyLinkType ContentType = 1019 ContentType_SettingsType ContentType = 1020 ContentType_CircuitConfirmationType ContentType = 1034 + ContentType_RouterLinksType ContentType = 1035 + ContentType_ListenersHeader ContentType = 10 ) // Enum value maps for ContentType. @@ -54,7 +56,7 @@ var ( 0: "Zero", 1000: "CircuitRequestType", 1002: "DialType", - 1003: "LinkType", + 1003: "LinkConnectedType", 1004: "FaultType", 1005: "RouteType", 1006: "UnrouteType", @@ -70,12 +72,14 @@ var ( 1019: "VerifyLinkType", 1020: "SettingsType", 1034: "CircuitConfirmationType", + 1035: "RouterLinksType", + 10: "ListenersHeader", } ContentType_value = map[string]int32{ "Zero": 0, "CircuitRequestType": 1000, "DialType": 1002, - "LinkType": 1003, + "LinkConnectedType": 1003, "FaultType": 1004, "RouteType": 1005, "UnrouteType": 1006, @@ -91,6 +95,8 @@ var ( "VerifyLinkType": 1019, "SettingsType": 1020, "CircuitConfirmationType": 1034, + "RouterLinksType": 1035, + "ListenersHeader": 10, } ) @@ -774,9 +780,11 @@ type Dial struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - LinkId string `protobuf:"bytes,1,opt,name=linkId,proto3" json:"linkId,omitempty"` - Address string `protobuf:"bytes,2,opt,name=address,proto3" json:"address,omitempty"` - RouterId string `protobuf:"bytes,3,opt,name=routerId,proto3" json:"routerId,omitempty"` + LinkId string `protobuf:"bytes,1,opt,name=linkId,proto3" json:"linkId,omitempty"` + Address string `protobuf:"bytes,2,opt,name=address,proto3" json:"address,omitempty"` + RouterId string `protobuf:"bytes,3,opt,name=routerId,proto3" json:"routerId,omitempty"` + LinkType string `protobuf:"bytes,4,opt,name=linkType,proto3" json:"linkType,omitempty"` + RouterVersion string `protobuf:"bytes,5,opt,name=routerVersion,proto3" json:"routerVersion,omitempty"` } func (x *Dial) Reset() { @@ -832,7 +840,21 @@ func (x *Dial) GetRouterId() string { return "" } -type Link struct { +func (x *Dial) GetLinkType() string { + if x != nil { + return x.LinkType + } + return "" +} + +func (x *Dial) GetRouterVersion() string { + if x != nil { + return x.RouterVersion + } + return "" +} + +type LinkConnected struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields @@ -840,8 +862,8 @@ type Link struct { Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` } -func (x *Link) Reset() { - *x = Link{} +func (x *LinkConnected) Reset() { + *x = LinkConnected{} if protoimpl.UnsafeEnabled { mi := &file_ctrl_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -849,13 +871,13 @@ func (x *Link) Reset() { } } -func (x *Link) String() string { +func (x *LinkConnected) String() string { return protoimpl.X.MessageStringOf(x) } -func (*Link) ProtoMessage() {} +func (*LinkConnected) ProtoMessage() {} -func (x *Link) ProtoReflect() protoreflect.Message { +func (x *LinkConnected) ProtoReflect() protoreflect.Message { mi := &file_ctrl_proto_msgTypes[9] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -867,18 +889,65 @@ func (x *Link) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use Link.ProtoReflect.Descriptor instead. -func (*Link) Descriptor() ([]byte, []int) { +// Deprecated: Use LinkConnected.ProtoReflect.Descriptor instead. +func (*LinkConnected) Descriptor() ([]byte, []int) { return file_ctrl_proto_rawDescGZIP(), []int{9} } -func (x *Link) GetId() string { +func (x *LinkConnected) GetId() string { if x != nil { return x.Id } return "" } +type RouterLinks struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Links []*RouterLinks_RouterLink `protobuf:"bytes,1,rep,name=links,proto3" json:"links,omitempty"` +} + +func (x *RouterLinks) Reset() { + *x = RouterLinks{} + if protoimpl.UnsafeEnabled { + mi := &file_ctrl_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RouterLinks) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RouterLinks) ProtoMessage() {} + +func (x *RouterLinks) ProtoReflect() protoreflect.Message { + mi := &file_ctrl_proto_msgTypes[10] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RouterLinks.ProtoReflect.Descriptor instead. +func (*RouterLinks) Descriptor() ([]byte, []int) { + return file_ctrl_proto_rawDescGZIP(), []int{10} +} + +func (x *RouterLinks) GetLinks() []*RouterLinks_RouterLink { + if x != nil { + return x.Links + } + return nil +} + type Fault struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -891,7 +960,7 @@ type Fault struct { func (x *Fault) Reset() { *x = Fault{} if protoimpl.UnsafeEnabled { - mi := &file_ctrl_proto_msgTypes[10] + mi := &file_ctrl_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -904,7 +973,7 @@ func (x *Fault) String() string { func (*Fault) ProtoMessage() {} func (x *Fault) ProtoReflect() protoreflect.Message { - mi := &file_ctrl_proto_msgTypes[10] + mi := &file_ctrl_proto_msgTypes[11] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -917,7 +986,7 @@ func (x *Fault) ProtoReflect() protoreflect.Message { // Deprecated: Use Fault.ProtoReflect.Descriptor instead. func (*Fault) Descriptor() ([]byte, []int) { - return file_ctrl_proto_rawDescGZIP(), []int{10} + return file_ctrl_proto_rawDescGZIP(), []int{11} } func (x *Fault) GetSubject() FaultSubject { @@ -946,7 +1015,7 @@ type Context struct { func (x *Context) Reset() { *x = Context{} if protoimpl.UnsafeEnabled { - mi := &file_ctrl_proto_msgTypes[11] + mi := &file_ctrl_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -959,7 +1028,7 @@ func (x *Context) String() string { func (*Context) ProtoMessage() {} func (x *Context) ProtoReflect() protoreflect.Message { - mi := &file_ctrl_proto_msgTypes[11] + mi := &file_ctrl_proto_msgTypes[12] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -972,7 +1041,7 @@ func (x *Context) ProtoReflect() protoreflect.Message { // Deprecated: Use Context.ProtoReflect.Descriptor instead. func (*Context) Descriptor() ([]byte, []int) { - return file_ctrl_proto_rawDescGZIP(), []int{11} + return file_ctrl_proto_rawDescGZIP(), []int{12} } func (x *Context) GetFields() map[string]string { @@ -1004,7 +1073,7 @@ type Route struct { func (x *Route) Reset() { *x = Route{} if protoimpl.UnsafeEnabled { - mi := &file_ctrl_proto_msgTypes[12] + mi := &file_ctrl_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1017,7 +1086,7 @@ func (x *Route) String() string { func (*Route) ProtoMessage() {} func (x *Route) ProtoReflect() protoreflect.Message { - mi := &file_ctrl_proto_msgTypes[12] + mi := &file_ctrl_proto_msgTypes[13] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1030,7 +1099,7 @@ func (x *Route) ProtoReflect() protoreflect.Message { // Deprecated: Use Route.ProtoReflect.Descriptor instead. func (*Route) Descriptor() ([]byte, []int) { - return file_ctrl_proto_rawDescGZIP(), []int{12} + return file_ctrl_proto_rawDescGZIP(), []int{13} } func (x *Route) GetCircuitId() string { @@ -1080,7 +1149,7 @@ type Unroute struct { func (x *Unroute) Reset() { *x = Unroute{} if protoimpl.UnsafeEnabled { - mi := &file_ctrl_proto_msgTypes[13] + mi := &file_ctrl_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1093,7 +1162,7 @@ func (x *Unroute) String() string { func (*Unroute) ProtoMessage() {} func (x *Unroute) ProtoReflect() protoreflect.Message { - mi := &file_ctrl_proto_msgTypes[13] + mi := &file_ctrl_proto_msgTypes[14] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1106,7 +1175,7 @@ func (x *Unroute) ProtoReflect() protoreflect.Message { // Deprecated: Use Unroute.ProtoReflect.Descriptor instead. func (*Unroute) Descriptor() ([]byte, []int) { - return file_ctrl_proto_rawDescGZIP(), []int{13} + return file_ctrl_proto_rawDescGZIP(), []int{14} } func (x *Unroute) GetCircuitId() string { @@ -1134,7 +1203,7 @@ type InspectRequest struct { func (x *InspectRequest) Reset() { *x = InspectRequest{} if protoimpl.UnsafeEnabled { - mi := &file_ctrl_proto_msgTypes[14] + mi := &file_ctrl_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1147,7 +1216,7 @@ func (x *InspectRequest) String() string { func (*InspectRequest) ProtoMessage() {} func (x *InspectRequest) ProtoReflect() protoreflect.Message { - mi := &file_ctrl_proto_msgTypes[14] + mi := &file_ctrl_proto_msgTypes[15] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1160,7 +1229,7 @@ func (x *InspectRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use InspectRequest.ProtoReflect.Descriptor instead. func (*InspectRequest) Descriptor() ([]byte, []int) { - return file_ctrl_proto_rawDescGZIP(), []int{14} + return file_ctrl_proto_rawDescGZIP(), []int{15} } func (x *InspectRequest) GetRequestedValues() []string { @@ -1183,7 +1252,7 @@ type InspectResponse struct { func (x *InspectResponse) Reset() { *x = InspectResponse{} if protoimpl.UnsafeEnabled { - mi := &file_ctrl_proto_msgTypes[15] + mi := &file_ctrl_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1196,7 +1265,7 @@ func (x *InspectResponse) String() string { func (*InspectResponse) ProtoMessage() {} func (x *InspectResponse) ProtoReflect() protoreflect.Message { - mi := &file_ctrl_proto_msgTypes[15] + mi := &file_ctrl_proto_msgTypes[16] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1209,7 +1278,7 @@ func (x *InspectResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use InspectResponse.ProtoReflect.Descriptor instead. func (*InspectResponse) Descriptor() ([]byte, []int) { - return file_ctrl_proto_rawDescGZIP(), []int{15} + return file_ctrl_proto_rawDescGZIP(), []int{16} } func (x *InspectResponse) GetSuccess() bool { @@ -1245,7 +1314,7 @@ type VerifyLink struct { func (x *VerifyLink) Reset() { *x = VerifyLink{} if protoimpl.UnsafeEnabled { - mi := &file_ctrl_proto_msgTypes[16] + mi := &file_ctrl_proto_msgTypes[17] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1258,7 +1327,7 @@ func (x *VerifyLink) String() string { func (*VerifyLink) ProtoMessage() {} func (x *VerifyLink) ProtoReflect() protoreflect.Message { - mi := &file_ctrl_proto_msgTypes[16] + mi := &file_ctrl_proto_msgTypes[17] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1271,7 +1340,7 @@ func (x *VerifyLink) ProtoReflect() protoreflect.Message { // Deprecated: Use VerifyLink.ProtoReflect.Descriptor instead. func (*VerifyLink) Descriptor() ([]byte, []int) { - return file_ctrl_proto_rawDescGZIP(), []int{16} + return file_ctrl_proto_rawDescGZIP(), []int{17} } func (x *VerifyLink) GetLinkId() string { @@ -1288,6 +1357,171 @@ func (x *VerifyLink) GetFingerprints() []string { return nil } +type Listener struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Address string `protobuf:"bytes,1,opt,name=address,proto3" json:"address,omitempty"` + Type string `protobuf:"bytes,2,opt,name=type,proto3" json:"type,omitempty"` +} + +func (x *Listener) Reset() { + *x = Listener{} + if protoimpl.UnsafeEnabled { + mi := &file_ctrl_proto_msgTypes[18] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Listener) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Listener) ProtoMessage() {} + +func (x *Listener) ProtoReflect() protoreflect.Message { + mi := &file_ctrl_proto_msgTypes[18] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Listener.ProtoReflect.Descriptor instead. +func (*Listener) Descriptor() ([]byte, []int) { + return file_ctrl_proto_rawDescGZIP(), []int{18} +} + +func (x *Listener) GetAddress() string { + if x != nil { + return x.Address + } + return "" +} + +func (x *Listener) GetType() string { + if x != nil { + return x.Type + } + return "" +} + +type Listeners struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Listeners []*Listener `protobuf:"bytes,1,rep,name=listeners,proto3" json:"listeners,omitempty"` +} + +func (x *Listeners) Reset() { + *x = Listeners{} + if protoimpl.UnsafeEnabled { + mi := &file_ctrl_proto_msgTypes[19] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Listeners) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Listeners) ProtoMessage() {} + +func (x *Listeners) ProtoReflect() protoreflect.Message { + mi := &file_ctrl_proto_msgTypes[19] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Listeners.ProtoReflect.Descriptor instead. +func (*Listeners) Descriptor() ([]byte, []int) { + return file_ctrl_proto_rawDescGZIP(), []int{19} +} + +func (x *Listeners) GetListeners() []*Listener { + if x != nil { + return x.Listeners + } + return nil +} + +type RouterLinks_RouterLink struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + DestRouterId string `protobuf:"bytes,2,opt,name=destRouterId,proto3" json:"destRouterId,omitempty"` + LinkType string `protobuf:"bytes,3,opt,name=linkType,proto3" json:"linkType,omitempty"` +} + +func (x *RouterLinks_RouterLink) Reset() { + *x = RouterLinks_RouterLink{} + if protoimpl.UnsafeEnabled { + mi := &file_ctrl_proto_msgTypes[23] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RouterLinks_RouterLink) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RouterLinks_RouterLink) ProtoMessage() {} + +func (x *RouterLinks_RouterLink) ProtoReflect() protoreflect.Message { + mi := &file_ctrl_proto_msgTypes[23] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RouterLinks_RouterLink.ProtoReflect.Descriptor instead. +func (*RouterLinks_RouterLink) Descriptor() ([]byte, []int) { + return file_ctrl_proto_rawDescGZIP(), []int{10, 0} +} + +func (x *RouterLinks_RouterLink) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *RouterLinks_RouterLink) GetDestRouterId() string { + if x != nil { + return x.DestRouterId + } + return "" +} + +func (x *RouterLinks_RouterLink) GetLinkType() string { + if x != nil { + return x.LinkType + } + return "" +} + type Route_Egress struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -1302,7 +1536,7 @@ type Route_Egress struct { func (x *Route_Egress) Reset() { *x = Route_Egress{} if protoimpl.UnsafeEnabled { - mi := &file_ctrl_proto_msgTypes[21] + mi := &file_ctrl_proto_msgTypes[25] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1315,7 +1549,7 @@ func (x *Route_Egress) String() string { func (*Route_Egress) ProtoMessage() {} func (x *Route_Egress) ProtoReflect() protoreflect.Message { - mi := &file_ctrl_proto_msgTypes[21] + mi := &file_ctrl_proto_msgTypes[25] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1328,7 +1562,7 @@ func (x *Route_Egress) ProtoReflect() protoreflect.Message { // Deprecated: Use Route_Egress.ProtoReflect.Descriptor instead. func (*Route_Egress) Descriptor() ([]byte, []int) { - return file_ctrl_proto_rawDescGZIP(), []int{12, 0} + return file_ctrl_proto_rawDescGZIP(), []int{13, 0} } func (x *Route_Egress) GetBinding() string { @@ -1371,7 +1605,7 @@ type Route_Forward struct { func (x *Route_Forward) Reset() { *x = Route_Forward{} if protoimpl.UnsafeEnabled { - mi := &file_ctrl_proto_msgTypes[22] + mi := &file_ctrl_proto_msgTypes[26] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1384,7 +1618,7 @@ func (x *Route_Forward) String() string { func (*Route_Forward) ProtoMessage() {} func (x *Route_Forward) ProtoReflect() protoreflect.Message { - mi := &file_ctrl_proto_msgTypes[22] + mi := &file_ctrl_proto_msgTypes[26] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1397,7 +1631,7 @@ func (x *Route_Forward) ProtoReflect() protoreflect.Message { // Deprecated: Use Route_Forward.ProtoReflect.Descriptor instead. func (*Route_Forward) Descriptor() ([]byte, []int) { - return file_ctrl_proto_rawDescGZIP(), []int{12, 1} + return file_ctrl_proto_rawDescGZIP(), []int{13, 1} } func (x *Route_Forward) GetSrcAddress() string { @@ -1426,7 +1660,7 @@ type InspectResponse_InspectValue struct { func (x *InspectResponse_InspectValue) Reset() { *x = InspectResponse_InspectValue{} if protoimpl.UnsafeEnabled { - mi := &file_ctrl_proto_msgTypes[24] + mi := &file_ctrl_proto_msgTypes[28] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1439,7 +1673,7 @@ func (x *InspectResponse_InspectValue) String() string { func (*InspectResponse_InspectValue) ProtoMessage() {} func (x *InspectResponse_InspectValue) ProtoReflect() protoreflect.Message { - mi := &file_ctrl_proto_msgTypes[24] + mi := &file_ctrl_proto_msgTypes[28] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1452,7 +1686,7 @@ func (x *InspectResponse_InspectValue) ProtoReflect() protoreflect.Message { // Deprecated: Use InspectResponse_InspectValue.ProtoReflect.Descriptor instead. func (*InspectResponse_InspectValue) Descriptor() ([]byte, []int) { - return file_ctrl_proto_rawDescGZIP(), []int{15, 0} + return file_ctrl_proto_rawDescGZIP(), []int{16, 0} } func (x *InspectResponse_InspectValue) GetName() string { @@ -1552,131 +1786,157 @@ var file_ctrl_proto_rawDesc = []byte{ 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x6f, 0x72, 0x50, 0x72, 0x65, 0x63, 0x65, 0x64, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x0a, 0x70, 0x72, 0x65, 0x63, 0x65, 0x64, 0x65, 0x6e, 0x63, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x63, 0x6f, 0x73, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x63, - 0x6f, 0x73, 0x74, 0x22, 0x54, 0x0a, 0x04, 0x44, 0x69, 0x61, 0x6c, 0x12, 0x16, 0x0a, 0x06, 0x6c, - 0x69, 0x6e, 0x6b, 0x49, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x6c, 0x69, 0x6e, - 0x6b, 0x49, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x1a, 0x0a, - 0x08, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x72, 0x49, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x08, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x72, 0x49, 0x64, 0x22, 0x16, 0x0a, 0x04, 0x4c, 0x69, 0x6e, - 0x6b, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, - 0x64, 0x22, 0x4d, 0x0a, 0x05, 0x46, 0x61, 0x75, 0x6c, 0x74, 0x12, 0x34, 0x0a, 0x07, 0x73, 0x75, - 0x62, 0x6a, 0x65, 0x63, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1a, 0x2e, 0x7a, 0x69, - 0x74, 0x69, 0x2e, 0x63, 0x74, 0x72, 0x6c, 0x2e, 0x70, 0x62, 0x2e, 0x46, 0x61, 0x75, 0x6c, 0x74, - 0x53, 0x75, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x07, 0x73, 0x75, 0x62, 0x6a, 0x65, 0x63, 0x74, - 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, - 0x22, 0xa1, 0x01, 0x0a, 0x07, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x12, 0x39, 0x0a, 0x06, - 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x7a, - 0x69, 0x74, 0x69, 0x2e, 0x63, 0x74, 0x72, 0x6c, 0x2e, 0x70, 0x62, 0x2e, 0x43, 0x6f, 0x6e, 0x74, - 0x65, 0x78, 0x74, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, - 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x12, 0x20, 0x0a, 0x0b, 0x63, 0x68, 0x61, 0x6e, 0x6e, - 0x65, 0x6c, 0x4d, 0x61, 0x73, 0x6b, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0b, 0x63, 0x68, - 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x4d, 0x61, 0x73, 0x6b, 0x1a, 0x39, 0x0a, 0x0b, 0x46, 0x69, 0x65, - 0x6c, 0x64, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x3a, 0x02, 0x38, 0x01, 0x22, 0x8c, 0x04, 0x0a, 0x05, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x12, 0x1c, - 0x0a, 0x09, 0x63, 0x69, 0x72, 0x63, 0x75, 0x69, 0x74, 0x49, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x09, 0x63, 0x69, 0x72, 0x63, 0x75, 0x69, 0x74, 0x49, 0x64, 0x12, 0x18, 0x0a, 0x07, - 0x61, 0x74, 0x74, 0x65, 0x6d, 0x70, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x07, 0x61, - 0x74, 0x74, 0x65, 0x6d, 0x70, 0x74, 0x12, 0x32, 0x0a, 0x06, 0x65, 0x67, 0x72, 0x65, 0x73, 0x73, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x7a, 0x69, 0x74, 0x69, 0x2e, 0x63, 0x74, - 0x72, 0x6c, 0x2e, 0x70, 0x62, 0x2e, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x2e, 0x45, 0x67, 0x72, 0x65, - 0x73, 0x73, 0x52, 0x06, 0x65, 0x67, 0x72, 0x65, 0x73, 0x73, 0x12, 0x37, 0x0a, 0x08, 0x66, 0x6f, - 0x72, 0x77, 0x61, 0x72, 0x64, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x7a, - 0x69, 0x74, 0x69, 0x2e, 0x63, 0x74, 0x72, 0x6c, 0x2e, 0x70, 0x62, 0x2e, 0x52, 0x6f, 0x75, 0x74, - 0x65, 0x2e, 0x46, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x52, 0x08, 0x66, 0x6f, 0x72, 0x77, 0x61, - 0x72, 0x64, 0x73, 0x12, 0x2f, 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x18, 0x05, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x7a, 0x69, 0x74, 0x69, 0x2e, 0x63, 0x74, 0x72, 0x6c, - 0x2e, 0x70, 0x62, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x07, 0x63, 0x6f, 0x6e, - 0x74, 0x65, 0x78, 0x74, 0x1a, 0xe1, 0x01, 0x0a, 0x06, 0x45, 0x67, 0x72, 0x65, 0x73, 0x73, 0x12, - 0x18, 0x0a, 0x07, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x07, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x12, 0x18, 0x0a, 0x07, 0x61, 0x64, 0x64, - 0x72, 0x65, 0x73, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x61, 0x64, 0x64, 0x72, - 0x65, 0x73, 0x73, 0x12, 0x20, 0x0a, 0x0b, 0x64, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x64, 0x65, 0x73, 0x74, 0x69, 0x6e, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x44, 0x0a, 0x08, 0x70, 0x65, 0x65, 0x72, 0x44, 0x61, 0x74, - 0x61, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x7a, 0x69, 0x74, 0x69, 0x2e, 0x63, - 0x74, 0x72, 0x6c, 0x2e, 0x70, 0x62, 0x2e, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x2e, 0x45, 0x67, 0x72, - 0x65, 0x73, 0x73, 0x2e, 0x50, 0x65, 0x65, 0x72, 0x44, 0x61, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, - 0x79, 0x52, 0x08, 0x70, 0x65, 0x65, 0x72, 0x44, 0x61, 0x74, 0x61, 0x1a, 0x3b, 0x0a, 0x0d, 0x50, - 0x65, 0x65, 0x72, 0x44, 0x61, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, - 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, - 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x49, 0x0a, 0x07, 0x46, 0x6f, 0x72, 0x77, - 0x61, 0x72, 0x64, 0x12, 0x1e, 0x0a, 0x0a, 0x73, 0x72, 0x63, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, - 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x73, 0x72, 0x63, 0x41, 0x64, 0x64, 0x72, - 0x65, 0x73, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x64, 0x73, 0x74, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, - 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x64, 0x73, 0x74, 0x41, 0x64, 0x64, 0x72, - 0x65, 0x73, 0x73, 0x22, 0x39, 0x0a, 0x07, 0x55, 0x6e, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x12, 0x1c, - 0x0a, 0x09, 0x63, 0x69, 0x72, 0x63, 0x75, 0x69, 0x74, 0x49, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x09, 0x63, 0x69, 0x72, 0x63, 0x75, 0x69, 0x74, 0x49, 0x64, 0x12, 0x10, 0x0a, 0x03, - 0x6e, 0x6f, 0x77, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x03, 0x6e, 0x6f, 0x77, 0x22, 0x3a, - 0x0a, 0x0e, 0x49, 0x6e, 0x73, 0x70, 0x65, 0x63, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x12, 0x28, 0x0a, 0x0f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x65, 0x64, 0x56, 0x61, 0x6c, - 0x75, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0f, 0x72, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x65, 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x22, 0xc1, 0x01, 0x0a, 0x0f, 0x49, - 0x6e, 0x73, 0x70, 0x65, 0x63, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x18, - 0x0a, 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, - 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x65, 0x72, 0x72, 0x6f, - 0x72, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x06, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x73, - 0x12, 0x42, 0x0a, 0x06, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x2a, 0x2e, 0x7a, 0x69, 0x74, 0x69, 0x2e, 0x63, 0x74, 0x72, 0x6c, 0x2e, 0x70, 0x62, 0x2e, - 0x49, 0x6e, 0x73, 0x70, 0x65, 0x63, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, - 0x49, 0x6e, 0x73, 0x70, 0x65, 0x63, 0x74, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x06, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x73, 0x1a, 0x38, 0x0a, 0x0c, 0x49, 0x6e, 0x73, 0x70, 0x65, 0x63, 0x74, 0x56, - 0x61, 0x6c, 0x75, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x48, - 0x0a, 0x0a, 0x56, 0x65, 0x72, 0x69, 0x66, 0x79, 0x4c, 0x69, 0x6e, 0x6b, 0x12, 0x16, 0x0a, 0x06, + 0x6f, 0x73, 0x74, 0x22, 0x96, 0x01, 0x0a, 0x04, 0x44, 0x69, 0x61, 0x6c, 0x12, 0x16, 0x0a, 0x06, 0x6c, 0x69, 0x6e, 0x6b, 0x49, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x6c, 0x69, - 0x6e, 0x6b, 0x49, 0x64, 0x12, 0x22, 0x0a, 0x0c, 0x66, 0x69, 0x6e, 0x67, 0x65, 0x72, 0x70, 0x72, - 0x69, 0x6e, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x66, 0x69, 0x6e, 0x67, - 0x65, 0x72, 0x70, 0x72, 0x69, 0x6e, 0x74, 0x73, 0x2a, 0xcd, 0x03, 0x0a, 0x0b, 0x43, 0x6f, 0x6e, - 0x74, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x08, 0x0a, 0x04, 0x5a, 0x65, 0x72, 0x6f, - 0x10, 0x00, 0x12, 0x17, 0x0a, 0x12, 0x43, 0x69, 0x72, 0x63, 0x75, 0x69, 0x74, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x54, 0x79, 0x70, 0x65, 0x10, 0xe8, 0x07, 0x12, 0x0d, 0x0a, 0x08, 0x44, - 0x69, 0x61, 0x6c, 0x54, 0x79, 0x70, 0x65, 0x10, 0xea, 0x07, 0x12, 0x0d, 0x0a, 0x08, 0x4c, 0x69, - 0x6e, 0x6b, 0x54, 0x79, 0x70, 0x65, 0x10, 0xeb, 0x07, 0x12, 0x0e, 0x0a, 0x09, 0x46, 0x61, 0x75, - 0x6c, 0x74, 0x54, 0x79, 0x70, 0x65, 0x10, 0xec, 0x07, 0x12, 0x0e, 0x0a, 0x09, 0x52, 0x6f, 0x75, - 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x10, 0xed, 0x07, 0x12, 0x10, 0x0a, 0x0b, 0x55, 0x6e, 0x72, - 0x6f, 0x75, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x10, 0xee, 0x07, 0x12, 0x10, 0x0a, 0x0b, 0x4d, - 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x54, 0x79, 0x70, 0x65, 0x10, 0xef, 0x07, 0x12, 0x20, 0x0a, - 0x1b, 0x54, 0x6f, 0x67, 0x67, 0x6c, 0x65, 0x50, 0x69, 0x70, 0x65, 0x54, 0x72, 0x61, 0x63, 0x65, - 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x54, 0x79, 0x70, 0x65, 0x10, 0xf0, 0x07, 0x12, - 0x13, 0x0a, 0x0e, 0x54, 0x72, 0x61, 0x63, 0x65, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, - 0x65, 0x10, 0xf2, 0x07, 0x12, 0x20, 0x0a, 0x1b, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, 0x65, - 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x54, - 0x79, 0x70, 0x65, 0x10, 0xf3, 0x07, 0x12, 0x20, 0x0a, 0x1b, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, - 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x54, 0x79, 0x70, 0x65, 0x10, 0xf4, 0x07, 0x12, 0x17, 0x0a, 0x12, 0x49, 0x6e, 0x73, 0x70, - 0x65, 0x63, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x54, 0x79, 0x70, 0x65, 0x10, 0xf5, - 0x07, 0x12, 0x18, 0x0a, 0x13, 0x49, 0x6e, 0x73, 0x70, 0x65, 0x63, 0x74, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x54, 0x79, 0x70, 0x65, 0x10, 0xf6, 0x07, 0x12, 0x23, 0x0a, 0x1e, 0x56, - 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x6f, - 0x72, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x54, 0x79, 0x70, 0x65, 0x10, 0xf9, 0x07, - 0x12, 0x20, 0x0a, 0x1b, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, + 0x6e, 0x6b, 0x49, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x1a, + 0x0a, 0x08, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x72, 0x49, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x08, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x72, 0x49, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x6c, 0x69, + 0x6e, 0x6b, 0x54, 0x79, 0x70, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6c, 0x69, + 0x6e, 0x6b, 0x54, 0x79, 0x70, 0x65, 0x12, 0x24, 0x0a, 0x0d, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x72, + 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x72, + 0x6f, 0x75, 0x74, 0x65, 0x72, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0x1f, 0x0a, 0x0d, + 0x4c, 0x69, 0x6e, 0x6b, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x12, 0x0e, 0x0a, + 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x22, 0xa7, 0x01, + 0x0a, 0x0b, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x72, 0x4c, 0x69, 0x6e, 0x6b, 0x73, 0x12, 0x3a, 0x0a, + 0x05, 0x6c, 0x69, 0x6e, 0x6b, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x7a, + 0x69, 0x74, 0x69, 0x2e, 0x63, 0x74, 0x72, 0x6c, 0x2e, 0x70, 0x62, 0x2e, 0x52, 0x6f, 0x75, 0x74, + 0x65, 0x72, 0x4c, 0x69, 0x6e, 0x6b, 0x73, 0x2e, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x72, 0x4c, 0x69, + 0x6e, 0x6b, 0x52, 0x05, 0x6c, 0x69, 0x6e, 0x6b, 0x73, 0x1a, 0x5c, 0x0a, 0x0a, 0x52, 0x6f, 0x75, + 0x74, 0x65, 0x72, 0x4c, 0x69, 0x6e, 0x6b, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x22, 0x0a, 0x0c, 0x64, 0x65, 0x73, 0x74, 0x52, + 0x6f, 0x75, 0x74, 0x65, 0x72, 0x49, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x64, + 0x65, 0x73, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x72, 0x49, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x6c, + 0x69, 0x6e, 0x6b, 0x54, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6c, + 0x69, 0x6e, 0x6b, 0x54, 0x79, 0x70, 0x65, 0x22, 0x4d, 0x0a, 0x05, 0x46, 0x61, 0x75, 0x6c, 0x74, + 0x12, 0x34, 0x0a, 0x07, 0x73, 0x75, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0e, 0x32, 0x1a, 0x2e, 0x7a, 0x69, 0x74, 0x69, 0x2e, 0x63, 0x74, 0x72, 0x6c, 0x2e, 0x70, 0x62, + 0x2e, 0x46, 0x61, 0x75, 0x6c, 0x74, 0x53, 0x75, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x07, 0x73, + 0x75, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x22, 0xa1, 0x01, 0x0a, 0x07, 0x43, 0x6f, 0x6e, 0x74, 0x65, + 0x78, 0x74, 0x12, 0x39, 0x0a, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x7a, 0x69, 0x74, 0x69, 0x2e, 0x63, 0x74, 0x72, 0x6c, 0x2e, 0x70, + 0x62, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x73, + 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x12, 0x20, 0x0a, + 0x0b, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x4d, 0x61, 0x73, 0x6b, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0d, 0x52, 0x0b, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x4d, 0x61, 0x73, 0x6b, 0x1a, + 0x39, 0x0a, 0x0b, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, + 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, + 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x8c, 0x04, 0x0a, 0x05, 0x52, + 0x6f, 0x75, 0x74, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x69, 0x72, 0x63, 0x75, 0x69, 0x74, 0x49, + 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x69, 0x72, 0x63, 0x75, 0x69, 0x74, + 0x49, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x61, 0x74, 0x74, 0x65, 0x6d, 0x70, 0x74, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0d, 0x52, 0x07, 0x61, 0x74, 0x74, 0x65, 0x6d, 0x70, 0x74, 0x12, 0x32, 0x0a, 0x06, + 0x65, 0x67, 0x72, 0x65, 0x73, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x7a, + 0x69, 0x74, 0x69, 0x2e, 0x63, 0x74, 0x72, 0x6c, 0x2e, 0x70, 0x62, 0x2e, 0x52, 0x6f, 0x75, 0x74, + 0x65, 0x2e, 0x45, 0x67, 0x72, 0x65, 0x73, 0x73, 0x52, 0x06, 0x65, 0x67, 0x72, 0x65, 0x73, 0x73, + 0x12, 0x37, 0x0a, 0x08, 0x66, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x73, 0x18, 0x04, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x7a, 0x69, 0x74, 0x69, 0x2e, 0x63, 0x74, 0x72, 0x6c, 0x2e, 0x70, + 0x62, 0x2e, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x2e, 0x46, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x52, + 0x08, 0x66, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x73, 0x12, 0x2f, 0x0a, 0x07, 0x63, 0x6f, 0x6e, + 0x74, 0x65, 0x78, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x7a, 0x69, 0x74, + 0x69, 0x2e, 0x63, 0x74, 0x72, 0x6c, 0x2e, 0x70, 0x62, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, + 0x74, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x1a, 0xe1, 0x01, 0x0a, 0x06, 0x45, + 0x67, 0x72, 0x65, 0x73, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x12, + 0x18, 0x0a, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x20, 0x0a, 0x0b, 0x64, 0x65, 0x73, + 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, + 0x64, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x44, 0x0a, 0x08, 0x70, + 0x65, 0x65, 0x72, 0x44, 0x61, 0x74, 0x61, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x28, 0x2e, + 0x7a, 0x69, 0x74, 0x69, 0x2e, 0x63, 0x74, 0x72, 0x6c, 0x2e, 0x70, 0x62, 0x2e, 0x52, 0x6f, 0x75, + 0x74, 0x65, 0x2e, 0x45, 0x67, 0x72, 0x65, 0x73, 0x73, 0x2e, 0x50, 0x65, 0x65, 0x72, 0x44, 0x61, + 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x08, 0x70, 0x65, 0x65, 0x72, 0x44, 0x61, 0x74, + 0x61, 0x1a, 0x3b, 0x0a, 0x0d, 0x50, 0x65, 0x65, 0x72, 0x44, 0x61, 0x74, 0x61, 0x45, 0x6e, 0x74, + 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, + 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x49, + 0x0a, 0x07, 0x46, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x12, 0x1e, 0x0a, 0x0a, 0x73, 0x72, 0x63, + 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x73, + 0x72, 0x63, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x64, 0x73, 0x74, + 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x64, + 0x73, 0x74, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x22, 0x39, 0x0a, 0x07, 0x55, 0x6e, 0x72, + 0x6f, 0x75, 0x74, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x69, 0x72, 0x63, 0x75, 0x69, 0x74, 0x49, + 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x69, 0x72, 0x63, 0x75, 0x69, 0x74, + 0x49, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x6e, 0x6f, 0x77, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, + 0x03, 0x6e, 0x6f, 0x77, 0x22, 0x3a, 0x0a, 0x0e, 0x49, 0x6e, 0x73, 0x70, 0x65, 0x63, 0x74, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x28, 0x0a, 0x0f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x65, 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, + 0x0f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x65, 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, + 0x22, 0xc1, 0x01, 0x0a, 0x0f, 0x49, 0x6e, 0x73, 0x70, 0x65, 0x63, 0x74, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x12, 0x16, + 0x0a, 0x06, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x06, + 0x65, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x12, 0x42, 0x0a, 0x06, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, + 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x7a, 0x69, 0x74, 0x69, 0x2e, 0x63, 0x74, + 0x72, 0x6c, 0x2e, 0x70, 0x62, 0x2e, 0x49, 0x6e, 0x73, 0x70, 0x65, 0x63, 0x74, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x49, 0x6e, 0x73, 0x70, 0x65, 0x63, 0x74, 0x56, 0x61, 0x6c, + 0x75, 0x65, 0x52, 0x06, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x1a, 0x38, 0x0a, 0x0c, 0x49, 0x6e, + 0x73, 0x70, 0x65, 0x63, 0x74, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, + 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x14, + 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x22, 0x48, 0x0a, 0x0a, 0x56, 0x65, 0x72, 0x69, 0x66, 0x79, 0x4c, 0x69, + 0x6e, 0x6b, 0x12, 0x16, 0x0a, 0x06, 0x6c, 0x69, 0x6e, 0x6b, 0x49, 0x64, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x06, 0x6c, 0x69, 0x6e, 0x6b, 0x49, 0x64, 0x12, 0x22, 0x0a, 0x0c, 0x66, 0x69, + 0x6e, 0x67, 0x65, 0x72, 0x70, 0x72, 0x69, 0x6e, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, + 0x52, 0x0c, 0x66, 0x69, 0x6e, 0x67, 0x65, 0x72, 0x70, 0x72, 0x69, 0x6e, 0x74, 0x73, 0x22, 0x38, + 0x0a, 0x08, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x12, 0x18, 0x0a, 0x07, 0x61, 0x64, + 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x61, 0x64, 0x64, + 0x72, 0x65, 0x73, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x22, 0x41, 0x0a, 0x09, 0x4c, 0x69, 0x73, 0x74, + 0x65, 0x6e, 0x65, 0x72, 0x73, 0x12, 0x34, 0x0a, 0x09, 0x6c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, + 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x7a, 0x69, 0x74, 0x69, 0x2e, + 0x63, 0x74, 0x72, 0x6c, 0x2e, 0x70, 0x62, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, + 0x52, 0x09, 0x6c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x73, 0x2a, 0x81, 0x04, 0x0a, 0x0b, + 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x08, 0x0a, 0x04, 0x5a, + 0x65, 0x72, 0x6f, 0x10, 0x00, 0x12, 0x17, 0x0a, 0x12, 0x43, 0x69, 0x72, 0x63, 0x75, 0x69, 0x74, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x54, 0x79, 0x70, 0x65, 0x10, 0xe8, 0x07, 0x12, 0x0d, + 0x0a, 0x08, 0x44, 0x69, 0x61, 0x6c, 0x54, 0x79, 0x70, 0x65, 0x10, 0xea, 0x07, 0x12, 0x16, 0x0a, + 0x11, 0x4c, 0x69, 0x6e, 0x6b, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x54, 0x79, + 0x70, 0x65, 0x10, 0xeb, 0x07, 0x12, 0x0e, 0x0a, 0x09, 0x46, 0x61, 0x75, 0x6c, 0x74, 0x54, 0x79, + 0x70, 0x65, 0x10, 0xec, 0x07, 0x12, 0x0e, 0x0a, 0x09, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x54, 0x79, + 0x70, 0x65, 0x10, 0xed, 0x07, 0x12, 0x10, 0x0a, 0x0b, 0x55, 0x6e, 0x72, 0x6f, 0x75, 0x74, 0x65, + 0x54, 0x79, 0x70, 0x65, 0x10, 0xee, 0x07, 0x12, 0x10, 0x0a, 0x0b, 0x4d, 0x65, 0x74, 0x72, 0x69, + 0x63, 0x73, 0x54, 0x79, 0x70, 0x65, 0x10, 0xef, 0x07, 0x12, 0x20, 0x0a, 0x1b, 0x54, 0x6f, 0x67, + 0x67, 0x6c, 0x65, 0x50, 0x69, 0x70, 0x65, 0x54, 0x72, 0x61, 0x63, 0x65, 0x73, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x54, 0x79, 0x70, 0x65, 0x10, 0xf0, 0x07, 0x12, 0x13, 0x0a, 0x0e, 0x54, + 0x72, 0x61, 0x63, 0x65, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x10, 0xf2, 0x07, + 0x12, 0x20, 0x0a, 0x1b, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x54, 0x79, 0x70, 0x65, 0x10, - 0xfa, 0x07, 0x12, 0x13, 0x0a, 0x0e, 0x56, 0x65, 0x72, 0x69, 0x66, 0x79, 0x4c, 0x69, 0x6e, 0x6b, - 0x54, 0x79, 0x70, 0x65, 0x10, 0xfb, 0x07, 0x12, 0x11, 0x0a, 0x0c, 0x53, 0x65, 0x74, 0x74, 0x69, - 0x6e, 0x67, 0x73, 0x54, 0x79, 0x70, 0x65, 0x10, 0xfc, 0x07, 0x12, 0x1c, 0x0a, 0x17, 0x43, 0x69, - 0x72, 0x63, 0x75, 0x69, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x72, 0x6d, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x54, 0x79, 0x70, 0x65, 0x10, 0x8a, 0x08, 0x2a, 0x35, 0x0a, 0x0c, 0x53, 0x65, 0x74, 0x74, - 0x69, 0x6e, 0x67, 0x54, 0x79, 0x70, 0x65, 0x73, 0x12, 0x11, 0x0a, 0x0d, 0x55, 0x6e, 0x75, 0x73, - 0x65, 0x64, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x10, 0x00, 0x12, 0x12, 0x0a, 0x0e, 0x4e, - 0x65, 0x77, 0x43, 0x74, 0x72, 0x6c, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x10, 0x01, 0x2a, - 0x3d, 0x0a, 0x14, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x6f, 0x72, 0x50, 0x72, 0x65, - 0x63, 0x65, 0x64, 0x65, 0x6e, 0x63, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x44, 0x65, 0x66, 0x61, 0x75, - 0x6c, 0x74, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x52, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x64, - 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x10, 0x02, 0x2a, 0x52, - 0x0a, 0x0c, 0x46, 0x61, 0x75, 0x6c, 0x74, 0x53, 0x75, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x10, - 0x0a, 0x0c, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x46, 0x61, 0x75, 0x6c, 0x74, 0x10, 0x00, - 0x12, 0x0f, 0x0a, 0x0b, 0x45, 0x67, 0x72, 0x65, 0x73, 0x73, 0x46, 0x61, 0x75, 0x6c, 0x74, 0x10, - 0x01, 0x12, 0x0d, 0x0a, 0x09, 0x4c, 0x69, 0x6e, 0x6b, 0x46, 0x61, 0x75, 0x6c, 0x74, 0x10, 0x02, - 0x12, 0x10, 0x0a, 0x0c, 0x46, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x46, 0x61, 0x75, 0x6c, 0x74, - 0x10, 0x03, 0x42, 0x27, 0x5a, 0x25, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, - 0x2f, 0x6f, 0x70, 0x65, 0x6e, 0x7a, 0x69, 0x74, 0x69, 0x2f, 0x66, 0x61, 0x62, 0x72, 0x69, 0x63, - 0x2f, 0x70, 0x62, 0x2f, 0x63, 0x74, 0x72, 0x6c, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x33, + 0xf3, 0x07, 0x12, 0x20, 0x0a, 0x1b, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x54, 0x65, 0x72, 0x6d, + 0x69, 0x6e, 0x61, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x54, 0x79, 0x70, + 0x65, 0x10, 0xf4, 0x07, 0x12, 0x17, 0x0a, 0x12, 0x49, 0x6e, 0x73, 0x70, 0x65, 0x63, 0x74, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x54, 0x79, 0x70, 0x65, 0x10, 0xf5, 0x07, 0x12, 0x18, 0x0a, + 0x13, 0x49, 0x6e, 0x73, 0x70, 0x65, 0x63, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x54, 0x79, 0x70, 0x65, 0x10, 0xf6, 0x07, 0x12, 0x23, 0x0a, 0x1e, 0x56, 0x61, 0x6c, 0x69, 0x64, + 0x61, 0x74, 0x65, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x6f, 0x72, 0x73, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x54, 0x79, 0x70, 0x65, 0x10, 0xf9, 0x07, 0x12, 0x20, 0x0a, 0x1b, + 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x6f, 0x72, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x54, 0x79, 0x70, 0x65, 0x10, 0xfa, 0x07, 0x12, 0x13, + 0x0a, 0x0e, 0x56, 0x65, 0x72, 0x69, 0x66, 0x79, 0x4c, 0x69, 0x6e, 0x6b, 0x54, 0x79, 0x70, 0x65, + 0x10, 0xfb, 0x07, 0x12, 0x11, 0x0a, 0x0c, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x54, + 0x79, 0x70, 0x65, 0x10, 0xfc, 0x07, 0x12, 0x1c, 0x0a, 0x17, 0x43, 0x69, 0x72, 0x63, 0x75, 0x69, + 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x72, 0x6d, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, + 0x65, 0x10, 0x8a, 0x08, 0x12, 0x14, 0x0a, 0x0f, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x72, 0x4c, 0x69, + 0x6e, 0x6b, 0x73, 0x54, 0x79, 0x70, 0x65, 0x10, 0x8b, 0x08, 0x12, 0x13, 0x0a, 0x0f, 0x4c, 0x69, + 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x73, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x10, 0x0a, 0x2a, + 0x35, 0x0a, 0x0c, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x54, 0x79, 0x70, 0x65, 0x73, 0x12, + 0x11, 0x0a, 0x0d, 0x55, 0x6e, 0x75, 0x73, 0x65, 0x64, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, + 0x10, 0x00, 0x12, 0x12, 0x0a, 0x0e, 0x4e, 0x65, 0x77, 0x43, 0x74, 0x72, 0x6c, 0x41, 0x64, 0x64, + 0x72, 0x65, 0x73, 0x73, 0x10, 0x01, 0x2a, 0x3d, 0x0a, 0x14, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, + 0x61, 0x74, 0x6f, 0x72, 0x50, 0x72, 0x65, 0x63, 0x65, 0x64, 0x65, 0x6e, 0x63, 0x65, 0x12, 0x0b, + 0x0a, 0x07, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x52, + 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x64, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x46, 0x61, 0x69, + 0x6c, 0x65, 0x64, 0x10, 0x02, 0x2a, 0x52, 0x0a, 0x0c, 0x46, 0x61, 0x75, 0x6c, 0x74, 0x53, 0x75, + 0x62, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x10, 0x0a, 0x0c, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, + 0x46, 0x61, 0x75, 0x6c, 0x74, 0x10, 0x00, 0x12, 0x0f, 0x0a, 0x0b, 0x45, 0x67, 0x72, 0x65, 0x73, + 0x73, 0x46, 0x61, 0x75, 0x6c, 0x74, 0x10, 0x01, 0x12, 0x0d, 0x0a, 0x09, 0x4c, 0x69, 0x6e, 0x6b, + 0x46, 0x61, 0x75, 0x6c, 0x74, 0x10, 0x02, 0x12, 0x10, 0x0a, 0x0c, 0x46, 0x6f, 0x72, 0x77, 0x61, + 0x72, 0x64, 0x46, 0x61, 0x75, 0x6c, 0x74, 0x10, 0x03, 0x42, 0x27, 0x5a, 0x25, 0x67, 0x69, 0x74, + 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6f, 0x70, 0x65, 0x6e, 0x7a, 0x69, 0x74, 0x69, + 0x2f, 0x66, 0x61, 0x62, 0x72, 0x69, 0x63, 0x2f, 0x70, 0x62, 0x2f, 0x63, 0x74, 0x72, 0x6c, 0x5f, + 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -1692,7 +1952,7 @@ func file_ctrl_proto_rawDescGZIP() []byte { } var file_ctrl_proto_enumTypes = make([]protoimpl.EnumInfo, 4) -var file_ctrl_proto_msgTypes = make([]protoimpl.MessageInfo, 25) +var file_ctrl_proto_msgTypes = make([]protoimpl.MessageInfo, 29) var file_ctrl_proto_goTypes = []interface{}{ (ContentType)(0), // 0: ziti.ctrl.pb.ContentType (SettingTypes)(0), // 1: ziti.ctrl.pb.SettingTypes @@ -1707,42 +1967,48 @@ var file_ctrl_proto_goTypes = []interface{}{ (*ValidateTerminatorsRequest)(nil), // 10: ziti.ctrl.pb.ValidateTerminatorsRequest (*UpdateTerminatorRequest)(nil), // 11: ziti.ctrl.pb.UpdateTerminatorRequest (*Dial)(nil), // 12: ziti.ctrl.pb.Dial - (*Link)(nil), // 13: ziti.ctrl.pb.Link - (*Fault)(nil), // 14: ziti.ctrl.pb.Fault - (*Context)(nil), // 15: ziti.ctrl.pb.Context - (*Route)(nil), // 16: ziti.ctrl.pb.Route - (*Unroute)(nil), // 17: ziti.ctrl.pb.Unroute - (*InspectRequest)(nil), // 18: ziti.ctrl.pb.InspectRequest - (*InspectResponse)(nil), // 19: ziti.ctrl.pb.InspectResponse - (*VerifyLink)(nil), // 20: ziti.ctrl.pb.VerifyLink - nil, // 21: ziti.ctrl.pb.Settings.DataEntry - nil, // 22: ziti.ctrl.pb.CircuitRequest.PeerDataEntry - nil, // 23: ziti.ctrl.pb.CreateTerminatorRequest.PeerDataEntry - nil, // 24: ziti.ctrl.pb.Context.FieldsEntry - (*Route_Egress)(nil), // 25: ziti.ctrl.pb.Route.Egress - (*Route_Forward)(nil), // 26: ziti.ctrl.pb.Route.Forward - nil, // 27: ziti.ctrl.pb.Route.Egress.PeerDataEntry - (*InspectResponse_InspectValue)(nil), // 28: ziti.ctrl.pb.InspectResponse.InspectValue + (*LinkConnected)(nil), // 13: ziti.ctrl.pb.LinkConnected + (*RouterLinks)(nil), // 14: ziti.ctrl.pb.RouterLinks + (*Fault)(nil), // 15: ziti.ctrl.pb.Fault + (*Context)(nil), // 16: ziti.ctrl.pb.Context + (*Route)(nil), // 17: ziti.ctrl.pb.Route + (*Unroute)(nil), // 18: ziti.ctrl.pb.Unroute + (*InspectRequest)(nil), // 19: ziti.ctrl.pb.InspectRequest + (*InspectResponse)(nil), // 20: ziti.ctrl.pb.InspectResponse + (*VerifyLink)(nil), // 21: ziti.ctrl.pb.VerifyLink + (*Listener)(nil), // 22: ziti.ctrl.pb.Listener + (*Listeners)(nil), // 23: ziti.ctrl.pb.Listeners + nil, // 24: ziti.ctrl.pb.Settings.DataEntry + nil, // 25: ziti.ctrl.pb.CircuitRequest.PeerDataEntry + nil, // 26: ziti.ctrl.pb.CreateTerminatorRequest.PeerDataEntry + (*RouterLinks_RouterLink)(nil), // 27: ziti.ctrl.pb.RouterLinks.RouterLink + nil, // 28: ziti.ctrl.pb.Context.FieldsEntry + (*Route_Egress)(nil), // 29: ziti.ctrl.pb.Route.Egress + (*Route_Forward)(nil), // 30: ziti.ctrl.pb.Route.Forward + nil, // 31: ziti.ctrl.pb.Route.Egress.PeerDataEntry + (*InspectResponse_InspectValue)(nil), // 32: ziti.ctrl.pb.InspectResponse.InspectValue } var file_ctrl_proto_depIdxs = []int32{ - 21, // 0: ziti.ctrl.pb.Settings.data:type_name -> ziti.ctrl.pb.Settings.DataEntry - 22, // 1: ziti.ctrl.pb.CircuitRequest.peerData:type_name -> ziti.ctrl.pb.CircuitRequest.PeerDataEntry - 23, // 2: ziti.ctrl.pb.CreateTerminatorRequest.peerData:type_name -> ziti.ctrl.pb.CreateTerminatorRequest.PeerDataEntry + 24, // 0: ziti.ctrl.pb.Settings.data:type_name -> ziti.ctrl.pb.Settings.DataEntry + 25, // 1: ziti.ctrl.pb.CircuitRequest.peerData:type_name -> ziti.ctrl.pb.CircuitRequest.PeerDataEntry + 26, // 2: ziti.ctrl.pb.CreateTerminatorRequest.peerData:type_name -> ziti.ctrl.pb.CreateTerminatorRequest.PeerDataEntry 2, // 3: ziti.ctrl.pb.CreateTerminatorRequest.precedence:type_name -> ziti.ctrl.pb.TerminatorPrecedence 9, // 4: ziti.ctrl.pb.ValidateTerminatorsRequest.terminators:type_name -> ziti.ctrl.pb.Terminator 2, // 5: ziti.ctrl.pb.UpdateTerminatorRequest.precedence:type_name -> ziti.ctrl.pb.TerminatorPrecedence - 3, // 6: ziti.ctrl.pb.Fault.subject:type_name -> ziti.ctrl.pb.FaultSubject - 24, // 7: ziti.ctrl.pb.Context.fields:type_name -> ziti.ctrl.pb.Context.FieldsEntry - 25, // 8: ziti.ctrl.pb.Route.egress:type_name -> ziti.ctrl.pb.Route.Egress - 26, // 9: ziti.ctrl.pb.Route.forwards:type_name -> ziti.ctrl.pb.Route.Forward - 15, // 10: ziti.ctrl.pb.Route.context:type_name -> ziti.ctrl.pb.Context - 28, // 11: ziti.ctrl.pb.InspectResponse.values:type_name -> ziti.ctrl.pb.InspectResponse.InspectValue - 27, // 12: ziti.ctrl.pb.Route.Egress.peerData:type_name -> ziti.ctrl.pb.Route.Egress.PeerDataEntry - 13, // [13:13] is the sub-list for method output_type - 13, // [13:13] is the sub-list for method input_type - 13, // [13:13] is the sub-list for extension type_name - 13, // [13:13] is the sub-list for extension extendee - 0, // [0:13] is the sub-list for field type_name + 27, // 6: ziti.ctrl.pb.RouterLinks.links:type_name -> ziti.ctrl.pb.RouterLinks.RouterLink + 3, // 7: ziti.ctrl.pb.Fault.subject:type_name -> ziti.ctrl.pb.FaultSubject + 28, // 8: ziti.ctrl.pb.Context.fields:type_name -> ziti.ctrl.pb.Context.FieldsEntry + 29, // 9: ziti.ctrl.pb.Route.egress:type_name -> ziti.ctrl.pb.Route.Egress + 30, // 10: ziti.ctrl.pb.Route.forwards:type_name -> ziti.ctrl.pb.Route.Forward + 16, // 11: ziti.ctrl.pb.Route.context:type_name -> ziti.ctrl.pb.Context + 32, // 12: ziti.ctrl.pb.InspectResponse.values:type_name -> ziti.ctrl.pb.InspectResponse.InspectValue + 22, // 13: ziti.ctrl.pb.Listeners.listeners:type_name -> ziti.ctrl.pb.Listener + 31, // 14: ziti.ctrl.pb.Route.Egress.peerData:type_name -> ziti.ctrl.pb.Route.Egress.PeerDataEntry + 15, // [15:15] is the sub-list for method output_type + 15, // [15:15] is the sub-list for method input_type + 15, // [15:15] is the sub-list for extension type_name + 15, // [15:15] is the sub-list for extension extendee + 0, // [0:15] is the sub-list for field type_name } func init() { file_ctrl_proto_init() } @@ -1860,7 +2126,7 @@ func file_ctrl_proto_init() { } } file_ctrl_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Link); i { + switch v := v.(*LinkConnected); i { case 0: return &v.state case 1: @@ -1872,7 +2138,7 @@ func file_ctrl_proto_init() { } } file_ctrl_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Fault); i { + switch v := v.(*RouterLinks); i { case 0: return &v.state case 1: @@ -1884,7 +2150,7 @@ func file_ctrl_proto_init() { } } file_ctrl_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Context); i { + switch v := v.(*Fault); i { case 0: return &v.state case 1: @@ -1896,7 +2162,7 @@ func file_ctrl_proto_init() { } } file_ctrl_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Route); i { + switch v := v.(*Context); i { case 0: return &v.state case 1: @@ -1908,7 +2174,7 @@ func file_ctrl_proto_init() { } } file_ctrl_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Unroute); i { + switch v := v.(*Route); i { case 0: return &v.state case 1: @@ -1920,7 +2186,7 @@ func file_ctrl_proto_init() { } } file_ctrl_proto_msgTypes[14].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*InspectRequest); i { + switch v := v.(*Unroute); i { case 0: return &v.state case 1: @@ -1932,7 +2198,7 @@ func file_ctrl_proto_init() { } } file_ctrl_proto_msgTypes[15].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*InspectResponse); i { + switch v := v.(*InspectRequest); i { case 0: return &v.state case 1: @@ -1944,6 +2210,18 @@ func file_ctrl_proto_init() { } } file_ctrl_proto_msgTypes[16].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*InspectResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_ctrl_proto_msgTypes[17].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*VerifyLink); i { case 0: return &v.state @@ -1955,7 +2233,43 @@ func file_ctrl_proto_init() { return nil } } - file_ctrl_proto_msgTypes[21].Exporter = func(v interface{}, i int) interface{} { + file_ctrl_proto_msgTypes[18].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Listener); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_ctrl_proto_msgTypes[19].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Listeners); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_ctrl_proto_msgTypes[23].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*RouterLinks_RouterLink); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_ctrl_proto_msgTypes[25].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Route_Egress); i { case 0: return &v.state @@ -1967,7 +2281,7 @@ func file_ctrl_proto_init() { return nil } } - file_ctrl_proto_msgTypes[22].Exporter = func(v interface{}, i int) interface{} { + file_ctrl_proto_msgTypes[26].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Route_Forward); i { case 0: return &v.state @@ -1979,7 +2293,7 @@ func file_ctrl_proto_init() { return nil } } - file_ctrl_proto_msgTypes[24].Exporter = func(v interface{}, i int) interface{} { + file_ctrl_proto_msgTypes[28].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*InspectResponse_InspectValue); i { case 0: return &v.state @@ -1998,7 +2312,7 @@ func file_ctrl_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_ctrl_proto_rawDesc, NumEnums: 4, - NumMessages: 25, + NumMessages: 29, NumExtensions: 0, NumServices: 0, }, diff --git a/pb/ctrl_pb/ctrl.proto b/pb/ctrl_pb/ctrl.proto index 12d4c49f9..50ad2daea 100644 --- a/pb/ctrl_pb/ctrl.proto +++ b/pb/ctrl_pb/ctrl.proto @@ -10,7 +10,7 @@ enum ContentType { // defined in ctrl_msg/messages.go now // CircuitSuccessType = 1001; DialType = 1002; - LinkType = 1003; + LinkConnectedType = 1003; FaultType = 1004; RouteType = 1005; UnrouteType = 1006; @@ -28,6 +28,9 @@ enum ContentType { VerifyLinkType = 1019; SettingsType = 1020; CircuitConfirmationType = 1034; + RouterLinksType = 1035; + + ListenersHeader = 10; } // SettingTypes are used with the Settings message send arbitrary settings to routers. @@ -97,12 +100,24 @@ message Dial { string linkId = 1; string address = 2; string routerId = 3; + string linkType = 4; + string routerVersion = 5; } -message Link { +message LinkConnected { string id = 1; } +message RouterLinks { + message RouterLink { + string id = 1; + string destRouterId = 2; + string linkType = 3; + } + + repeated RouterLink links = 1; +} + enum FaultSubject { IngressFault = 0; EgressFault = 1; @@ -163,4 +178,13 @@ message InspectResponse { message VerifyLink { string linkId = 1; repeated string fingerprints = 2; +} + +message Listener { + string address = 1; + string type = 2; +} + +message Listeners { + repeated Listener listeners = 1; } \ No newline at end of file diff --git a/pb/ctrl_pb/decoder.go b/pb/ctrl_pb/decoder.go index c4d0768d0..907f2730f 100644 --- a/pb/ctrl_pb/decoder.go +++ b/pb/ctrl_pb/decoder.go @@ -113,6 +113,24 @@ func (d Decoder) Decode(msg *channel.Message) ([]byte, bool) { return nil, true } + case int32(ContentType_VerifyLinkType): + request := &VerifyLink{} + if err := proto.Unmarshal(msg.Body, request); err == nil { + meta := channel.NewTraceMessageDecode(DECODER, "Verify Link") + meta["linkId"] = request.LinkId + data, err := meta.MarshalTraceMessageDecode() + if err != nil { + pfxlog.Logger().Errorf("unexpected error (%s)", err) + return nil, true + } + + return data, true + + } else { + pfxlog.Logger().Errorf("unexpected error (%s)", err) + return nil, true + } + case int32(ctrl_msg.CircuitSuccessType): meta := channel.NewTraceMessageDecode(DECODER, "Circuit Success Response") meta["circuitId"] = string(msg.Body) @@ -159,10 +177,10 @@ func (d Decoder) Decode(msg *channel.Message) ([]byte, bool) { return nil, true } - case int32(ContentType_LinkType): - link := &Link{} + case int32(ContentType_LinkConnectedType): + link := &LinkConnected{} if err := proto.Unmarshal(msg.Body, link); err == nil { - meta := channel.NewTraceMessageDecode(DECODER, "Link") + meta := channel.NewTraceMessageDecode(DECODER, "LinkConnected") meta["id"] = link.Id data, err := meta.MarshalTraceMessageDecode() @@ -177,6 +195,31 @@ func (d Decoder) Decode(msg *channel.Message) ([]byte, bool) { return nil, true } + case int32(ContentType_RouterLinksType): + links := &RouterLinks{} + if err := proto.Unmarshal(msg.Body, links); err == nil { + meta := channel.NewTraceMessageDecode(DECODER, "RouterLinks") + var linksD []map[string]interface{} + for _, link := range links.Links { + linksD = append(linksD, map[string]interface{}{ + "id": link.Id, + "dest": link.DestRouterId, + }) + } + meta["links"] = linksD + + data, err := meta.MarshalTraceMessageDecode() + if err != nil { + return nil, true + } + + return data, true + + } else { + pfxlog.Logger().Errorf("unexpected error (%s)", err) + return nil, true + } + case int32(ContentType_FaultType): fault := &Fault{} if err := proto.Unmarshal(msg.Body, fault); err == nil { diff --git a/pb/ctrl_pb/impl.go b/pb/ctrl_pb/impl.go index 253ca51bf..fd5c4b0e9 100644 --- a/pb/ctrl_pb/impl.go +++ b/pb/ctrl_pb/impl.go @@ -22,8 +22,12 @@ func (request *CircuitConfirmation) GetContentType() int32 { return int32(ContentType_CircuitConfirmationType) } -func (request *Link) GetContentType() int32 { - return int32(ContentType_LinkType) +func (request *LinkConnected) GetContentType() int32 { + return int32(ContentType_LinkConnectedType) +} + +func (request *RouterLinks) GetContentType() int32 { + return int32(ContentType_RouterLinksType) } func (request *VerifyLink) GetContentType() int32 { diff --git a/rest_model/inspect_response_value.go b/rest_model/inspect_response_value.go index 9b305de2b..33af0610e 100644 --- a/rest_model/inspect_response_value.go +++ b/rest_model/inspect_response_value.go @@ -53,7 +53,7 @@ type InspectResponseValue struct { // value // Required: true - Value *string `json:"value"` + Value interface{} `json:"value"` } // Validate validates this inspect response value @@ -98,8 +98,8 @@ func (m *InspectResponseValue) validateName(formats strfmt.Registry) error { func (m *InspectResponseValue) validateValue(formats strfmt.Registry) error { - if err := validate.Required("value", "body", m.Value); err != nil { - return err + if m.Value == nil { + return errors.Required("value", "body", nil) } return nil diff --git a/rest_model/link_detail.go b/rest_model/link_detail.go index 555cabc0d..583e3bd37 100644 --- a/rest_model/link_detail.go +++ b/rest_model/link_detail.go @@ -78,6 +78,10 @@ type LinkDetail struct { // static cost // Required: true StaticCost *int64 `json:"staticCost"` + + // type + // Required: true + Type *string `json:"type"` } // Validate validates this link detail @@ -120,6 +124,10 @@ func (m *LinkDetail) Validate(formats strfmt.Registry) error { res = append(res, err) } + if err := m.validateType(formats); err != nil { + res = append(res, err) + } + if len(res) > 0 { return errors.CompositeValidationError(res...) } @@ -229,6 +237,15 @@ func (m *LinkDetail) validateStaticCost(formats strfmt.Registry) error { return nil } +func (m *LinkDetail) validateType(formats strfmt.Registry) error { + + if err := validate.Required("type", "body", m.Type); err != nil { + return err + } + + return nil +} + // ContextValidate validate this link detail based on the context it is used func (m *LinkDetail) ContextValidate(ctx context.Context, formats strfmt.Registry) error { var res []error diff --git a/rest_model/links.go b/rest_model/links.go index cc09e97e6..84a6d2251 100644 --- a/rest_model/links.go +++ b/rest_model/links.go @@ -53,11 +53,6 @@ func (m Links) Validate(formats strfmt.Registry) error { } if val, ok := m[k]; ok { if err := val.Validate(formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName(k) - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName(k) - } return err } } diff --git a/rest_model/router_detail.go b/rest_model/router_detail.go index a1b3eaf97..478794014 100644 --- a/rest_model/router_detail.go +++ b/rest_model/router_detail.go @@ -31,6 +31,7 @@ package rest_model import ( "context" + "strconv" "github.com/go-openapi/errors" "github.com/go-openapi/strfmt" @@ -58,8 +59,8 @@ type RouterDetail struct { // Required: true Fingerprint *string `json:"fingerprint"` - // listener address - ListenerAddress string `json:"listenerAddress,omitempty"` + // listener addresses + ListenerAddresses []*RouterListener `json:"listenerAddresses"` // name // Required: true @@ -90,7 +91,7 @@ func (m *RouterDetail) UnmarshalJSON(raw []byte) error { Fingerprint *string `json:"fingerprint"` - ListenerAddress string `json:"listenerAddress,omitempty"` + ListenerAddresses []*RouterListener `json:"listenerAddresses"` Name *string `json:"name"` @@ -108,7 +109,7 @@ func (m *RouterDetail) UnmarshalJSON(raw []byte) error { m.Fingerprint = dataAO1.Fingerprint - m.ListenerAddress = dataAO1.ListenerAddress + m.ListenerAddresses = dataAO1.ListenerAddresses m.Name = dataAO1.Name @@ -135,7 +136,7 @@ func (m RouterDetail) MarshalJSON() ([]byte, error) { Fingerprint *string `json:"fingerprint"` - ListenerAddress string `json:"listenerAddress,omitempty"` + ListenerAddresses []*RouterListener `json:"listenerAddresses"` Name *string `json:"name"` @@ -150,7 +151,7 @@ func (m RouterDetail) MarshalJSON() ([]byte, error) { dataAO1.Fingerprint = m.Fingerprint - dataAO1.ListenerAddress = m.ListenerAddress + dataAO1.ListenerAddresses = m.ListenerAddresses dataAO1.Name = m.Name @@ -187,6 +188,10 @@ func (m *RouterDetail) Validate(formats strfmt.Registry) error { res = append(res, err) } + if err := m.validateListenerAddresses(formats); err != nil { + res = append(res, err) + } + if err := m.validateName(formats); err != nil { res = append(res, err) } @@ -240,6 +245,33 @@ func (m *RouterDetail) validateFingerprint(formats strfmt.Registry) error { return nil } +func (m *RouterDetail) validateListenerAddresses(formats strfmt.Registry) error { + + if swag.IsZero(m.ListenerAddresses) { // not required + return nil + } + + for i := 0; i < len(m.ListenerAddresses); i++ { + if swag.IsZero(m.ListenerAddresses[i]) { // not required + continue + } + + if m.ListenerAddresses[i] != nil { + if err := m.ListenerAddresses[i].Validate(formats); err != nil { + if ve, ok := err.(*errors.Validation); ok { + return ve.ValidateName("listenerAddresses" + "." + strconv.Itoa(i)) + } else if ce, ok := err.(*errors.CompositeError); ok { + return ce.ValidateName("listenerAddresses" + "." + strconv.Itoa(i)) + } + return err + } + } + + } + + return nil +} + func (m *RouterDetail) validateName(formats strfmt.Registry) error { if err := validate.Required("name", "body", m.Name); err != nil { @@ -287,6 +319,10 @@ func (m *RouterDetail) ContextValidate(ctx context.Context, formats strfmt.Regis res = append(res, err) } + if err := m.contextValidateListenerAddresses(ctx, formats); err != nil { + res = append(res, err) + } + if err := m.contextValidateVersionInfo(ctx, formats); err != nil { res = append(res, err) } @@ -297,6 +333,26 @@ func (m *RouterDetail) ContextValidate(ctx context.Context, formats strfmt.Regis return nil } +func (m *RouterDetail) contextValidateListenerAddresses(ctx context.Context, formats strfmt.Registry) error { + + for i := 0; i < len(m.ListenerAddresses); i++ { + + if m.ListenerAddresses[i] != nil { + if err := m.ListenerAddresses[i].ContextValidate(ctx, formats); err != nil { + if ve, ok := err.(*errors.Validation); ok { + return ve.ValidateName("listenerAddresses" + "." + strconv.Itoa(i)) + } else if ce, ok := err.(*errors.CompositeError); ok { + return ce.ValidateName("listenerAddresses" + "." + strconv.Itoa(i)) + } + return err + } + } + + } + + return nil +} + func (m *RouterDetail) contextValidateVersionInfo(ctx context.Context, formats strfmt.Registry) error { if m.VersionInfo != nil { diff --git a/rest_model/router_listener.go b/rest_model/router_listener.go new file mode 100644 index 000000000..e048ce49e --- /dev/null +++ b/rest_model/router_listener.go @@ -0,0 +1,112 @@ +// Code generated by go-swagger; DO NOT EDIT. + +// +// Copyright NetFoundry, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// __ __ _ +// \ \ / / (_) +// \ \ /\ / /_ _ _ __ _ __ _ _ __ __ _ +// \ \/ \/ / _` | '__| '_ \| | '_ \ / _` | +// \ /\ / (_| | | | | | | | | | | (_| | : This file is generated, do not edit it. +// \/ \/ \__,_|_| |_| |_|_|_| |_|\__, | +// __/ | +// |___/ + +package rest_model + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the swagger generate command + +import ( + "context" + + "github.com/go-openapi/errors" + "github.com/go-openapi/strfmt" + "github.com/go-openapi/swag" + "github.com/go-openapi/validate" +) + +// RouterListener router listener +// +// swagger:model routerListener +type RouterListener struct { + + // address + // Required: true + Address *string `json:"address"` + + // type + // Required: true + Type *string `json:"type"` +} + +// Validate validates this router listener +func (m *RouterListener) Validate(formats strfmt.Registry) error { + var res []error + + if err := m.validateAddress(formats); err != nil { + res = append(res, err) + } + + if err := m.validateType(formats); err != nil { + res = append(res, err) + } + + if len(res) > 0 { + return errors.CompositeValidationError(res...) + } + return nil +} + +func (m *RouterListener) validateAddress(formats strfmt.Registry) error { + + if err := validate.Required("address", "body", m.Address); err != nil { + return err + } + + return nil +} + +func (m *RouterListener) validateType(formats strfmt.Registry) error { + + if err := validate.Required("type", "body", m.Type); err != nil { + return err + } + + return nil +} + +// ContextValidate validates this router listener based on context it is used +func (m *RouterListener) ContextValidate(ctx context.Context, formats strfmt.Registry) error { + return nil +} + +// MarshalBinary interface implementation +func (m *RouterListener) MarshalBinary() ([]byte, error) { + if m == nil { + return nil, nil + } + return swag.WriteJSON(m) +} + +// UnmarshalBinary interface implementation +func (m *RouterListener) UnmarshalBinary(b []byte) error { + var res RouterListener + if err := swag.ReadJSON(b, &res); err != nil { + return err + } + *m = res + return nil +} diff --git a/rest_model/terminator_cost_map.go b/rest_model/terminator_cost_map.go index 96dc641c6..279384270 100644 --- a/rest_model/terminator_cost_map.go +++ b/rest_model/terminator_cost_map.go @@ -53,11 +53,6 @@ func (m TerminatorCostMap) Validate(formats strfmt.Registry) error { } if val, ok := m[k]; ok { if err := val.Validate(formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName(k) - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName(k) - } return err } } diff --git a/rest_model/terminator_precedence.go b/rest_model/terminator_precedence.go index 20ed96bd0..37ad50267 100644 --- a/rest_model/terminator_precedence.go +++ b/rest_model/terminator_precedence.go @@ -44,12 +44,8 @@ import ( type TerminatorPrecedence string func NewTerminatorPrecedence(value TerminatorPrecedence) *TerminatorPrecedence { - return &value -} - -// Pointer returns a pointer to a freshly-allocated TerminatorPrecedence. -func (m TerminatorPrecedence) Pointer() *TerminatorPrecedence { - return &m + v := value + return &v } const ( diff --git a/rest_model/terminator_precedence_map.go b/rest_model/terminator_precedence_map.go index ae9e45ff7..4595ded07 100644 --- a/rest_model/terminator_precedence_map.go +++ b/rest_model/terminator_precedence_map.go @@ -49,11 +49,6 @@ func (m TerminatorPrecedenceMap) Validate(formats strfmt.Registry) error { if val, ok := m[k]; ok { if err := val.Validate(formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName(k) - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName(k) - } return err } } diff --git a/rest_server/embedded_spec.go b/rest_server/embedded_spec.go index 91ca041bf..51fc3db92 100644 --- a/rest_server/embedded_spec.go +++ b/rest_server/embedded_spec.go @@ -1188,7 +1188,12 @@ func init() { "type": "string" }, "value": { - "type": "string" + "type": [ + "object", + "string", + "number", + "boolean" + ] } } }, @@ -1215,6 +1220,7 @@ func init() { "type": "object", "required": [ "id", + "type", "sourceRouter", "destRouter", "state", @@ -1251,6 +1257,9 @@ func init() { }, "staticCost": { "type": "integer" + }, + "type": { + "type": "string" } } }, @@ -1454,8 +1463,11 @@ func init() { "fingerprint": { "type": "string" }, - "listenerAddress": { - "type": "string" + "listenerAddresses": { + "type": "array", + "items": { + "$ref": "#/definitions/routerListener" + } }, "name": { "type": "string" @@ -1476,6 +1488,21 @@ func init() { "$ref": "#/definitions/routerDetail" } }, + "routerListener": { + "type": "object", + "required": [ + "address", + "type" + ], + "properties": { + "address": { + "type": "string" + }, + "type": { + "type": "string" + } + } + }, "routerPatch": { "type": "object", "properties": { @@ -4917,7 +4944,12 @@ func init() { "type": "string" }, "value": { - "type": "string" + "type": [ + "object", + "string", + "number", + "boolean" + ] } } }, @@ -4944,6 +4976,7 @@ func init() { "type": "object", "required": [ "id", + "type", "sourceRouter", "destRouter", "state", @@ -4980,6 +5013,9 @@ func init() { }, "staticCost": { "type": "integer" + }, + "type": { + "type": "string" } } }, @@ -5185,8 +5221,11 @@ func init() { "fingerprint": { "type": "string" }, - "listenerAddress": { - "type": "string" + "listenerAddresses": { + "type": "array", + "items": { + "$ref": "#/definitions/routerListener" + } }, "name": { "type": "string" @@ -5207,6 +5246,21 @@ func init() { "$ref": "#/definitions/routerDetail" } }, + "routerListener": { + "type": "object", + "required": [ + "address", + "type" + ], + "properties": { + "address": { + "type": "string" + }, + "type": { + "type": "string" + } + } + }, "routerPatch": { "type": "object", "properties": { diff --git a/router/accepter.go b/router/accepter.go index 12dd9d969..1aca53c54 100644 --- a/router/accepter.go +++ b/router/accepter.go @@ -1,13 +1,15 @@ package router import ( - forwarder2 "github.com/openziti/fabric/router/forwarder" + "github.com/openziti/fabric/router/forwarder" "github.com/openziti/fabric/router/xlink" "github.com/sirupsen/logrus" ) -func newXlinkAccepter(f *forwarder2.Forwarder) xlink.Acceptor { - return &xlinkAccepter{forwarder: f} +func newXlinkAccepter(f *forwarder.Forwarder) xlink.Acceptor { + return &xlinkAccepter{ + forwarder: f, + } } func (self *xlinkAccepter) Accept(xlink xlink.Xlink) error { @@ -19,5 +21,5 @@ func (self *xlinkAccepter) Accept(xlink xlink.Xlink) error { } type xlinkAccepter struct { - forwarder *forwarder2.Forwarder + forwarder *forwarder.Forwarder } diff --git a/router/handler_ctrl/bind.go b/router/handler_ctrl/bind.go index 418873bc4..91e4b6a41 100644 --- a/router/handler_ctrl/bind.go +++ b/router/handler_ctrl/bind.go @@ -37,6 +37,7 @@ type bindHandler struct { closeNotify chan struct{} ctrlAddressChanger CtrlAddressChanger traceHandler *channel.TraceHandler + linkRegistry xlink.Registry } func NewBindHandler(id *identity.TokenId, @@ -47,6 +48,7 @@ func NewBindHandler(id *identity.TokenId, xctrls []xctrl.Xctrl, ctrlAddressChanger CtrlAddressChanger, traceHandler *channel.TraceHandler, + linkRegistry xlink.Registry, closeNotify chan struct{}) channel.BindHandler { return &bindHandler{ id: id, @@ -58,17 +60,23 @@ func NewBindHandler(id *identity.TokenId, closeNotify: closeNotify, ctrlAddressChanger: ctrlAddressChanger, traceHandler: traceHandler, + linkRegistry: linkRegistry, } } func (self *bindHandler) BindChannel(binding channel.Binding) error { - binding.AddTypedReceiveHandler(newDialHandler(self.id, self.ctrl, self.xlinkDialers, self.forwarder, self.closeNotify)) + linkDialerPool := &handlerPool{ + options: self.forwarder.Options.LinkDial, + closeNotify: self.closeNotify, + } + binding.AddTypedReceiveHandler(newDialHandler(self.id, self.ctrl, self.xlinkDialers, linkDialerPool, self.linkRegistry)) binding.AddTypedReceiveHandler(newRouteHandler(self.id, self.ctrl, self.dialerCfg, self.forwarder, self.closeNotify)) binding.AddTypedReceiveHandler(newValidateTerminatorsHandler(self.ctrl, self.dialerCfg)) binding.AddTypedReceiveHandler(newUnrouteHandler(self.forwarder)) binding.AddTypedReceiveHandler(newTraceHandler(self.id, self.forwarder.TraceController())) - binding.AddTypedReceiveHandler(newInspectHandler(self.id)) + binding.AddTypedReceiveHandler(newInspectHandler(self.id, self.linkRegistry)) binding.AddTypedReceiveHandler(newSettingsHandler(self.ctrlAddressChanger)) + binding.AddTypedReceiveHandler(newFaultHandler(self.linkRegistry)) binding.AddPeekHandler(trace.NewChannelPeekHandler(self.id.Token, binding.GetChannel(), self.forwarder.TraceController(), trace.NewChannelSink(binding.GetChannel()))) latency.AddLatencyProbeResponder(binding) diff --git a/router/handler_ctrl/dial.go b/router/handler_ctrl/dial.go index 24abd5e55..c93bad9e5 100644 --- a/router/handler_ctrl/dial.go +++ b/router/handler_ctrl/dial.go @@ -22,7 +22,6 @@ import ( "github.com/openziti/channel" "github.com/openziti/channel/protobufs" "github.com/openziti/fabric/pb/ctrl_pb" - "github.com/openziti/fabric/router/forwarder" "github.com/openziti/fabric/router/xgress" "github.com/openziti/fabric/router/xlink" "github.com/openziti/foundation/identity/identity" @@ -31,21 +30,20 @@ import ( ) type dialHandler struct { - id *identity.TokenId - ctrl xgress.CtrlChannel - dialers []xlink.Dialer - pool handlerPool + id *identity.TokenId + ctrl xgress.CtrlChannel + dialers []xlink.Dialer + registry xlink.Registry + pool *handlerPool } -func newDialHandler(id *identity.TokenId, ctrl xgress.CtrlChannel, dialers []xlink.Dialer, forwarder *forwarder.Forwarder, closeNotify chan struct{}) *dialHandler { +func newDialHandler(id *identity.TokenId, ctrl xgress.CtrlChannel, dialers []xlink.Dialer, pool *handlerPool, registry xlink.Registry) *dialHandler { handler := &dialHandler{ - id: id, - ctrl: ctrl, - dialers: dialers, - pool: handlerPool{ - options: forwarder.Options.LinkDial, - closeNotify: closeNotify, - }, + id: id, + ctrl: ctrl, + dialers: dialers, + pool: pool, + registry: registry, } handler.pool.Start() @@ -57,44 +55,69 @@ func (self *dialHandler) ContentType() int32 { } func (self *dialHandler) HandleReceive(msg *channel.Message, ch channel.Channel) { + dial := &ctrl_pb.Dial{} + if err := proto.Unmarshal(msg.Body, dial); err != nil { + logrus.WithError(err).Error("error unmarshalling dial message") + return + } + self.pool.Queue(func() { - self.handle(msg, ch) + self.handle(dial, ch) }) } -func (self *dialHandler) handle(msg *channel.Message, _ channel.Channel) { - dial := &ctrl_pb.Dial{} - if err := proto.Unmarshal(msg.Body, dial); err == nil { - log := pfxlog.ChannelLogger("link", "linkDialer"). - WithFields(logrus.Fields{"linkId": dial.LinkId, "routerId": dial.RouterId, "address": dial.Address}) +func (self *dialHandler) handle(dial *ctrl_pb.Dial, _ channel.Channel) { + log := pfxlog.ChannelLogger("link", "linkDialer"). + WithFields(logrus.Fields{ + "linkId": dial.LinkId, + "routerId": dial.RouterId, + "address": dial.Address, + "linkType": dial.LinkType, + "routerVersion": dial.RouterVersion, + }) - linkId := self.id.ShallowCloneWithNewToken(dial.LinkId) - if len(self.dialers) == 1 { - log.Info("dialing link") - if err := self.dialers[0].Dial(dial.Address, linkId, self.id.Token); err == nil { + if len(self.dialers) != 1 { + log.Errorf("invalid Xlink dialers configuration") + if err := self.sendLinkFault(dial.LinkId); err != nil { + log.WithError(err).Error("error sending link fault") + } + return + } + + link, lockAcquired := self.registry.GetDialLock(dial) + if link != nil { + log.WithField("existingLinkId", link.Id().Token).Info("existing link found") + if err := self.sendLinkFault(dial.LinkId); err != nil { + log.WithError(err).Error("error sending link fault") + } + return + } + + if lockAcquired { + log.Info("dialing link") + if link, err := self.dialers[0].Dial(dial); err == nil { + if existingLink, success := self.registry.DialSucceeded(link); success { + log.Info("link registered") if err := self.sendLinkMessage(dial.LinkId); err != nil { log.WithError(err).Error("error sending link message ") } - log.Info("link established") } else { - log.WithError(err).Error("link dialing failed") - if err := self.sendLinkFault(dial.LinkId); err != nil { - log.WithError(err).Error("error sending fault") - } + log.WithField("existingLinkId", existingLink.Id().Token).Info("existing link found, new link closed") } } else { - log.Errorf("invalid Xlink dialers configuration") + log.WithError(err).Error("link dialing failed") + self.registry.DialFailed(dial) if err := self.sendLinkFault(dial.LinkId); err != nil { - log.WithError(err).Error("error sending link fault") + log.WithError(err).Error("error sending fault") } } } else { - logrus.WithError(err).Error("error unmarshalling dial message") + log.Info("unable to dial, dial already in progress") } } func (self *dialHandler) sendLinkMessage(linkId string) error { - linkMsg := &ctrl_pb.Link{Id: linkId} + linkMsg := &ctrl_pb.LinkConnected{Id: linkId} if err := protobufs.MarshalTyped(linkMsg).Send(self.ctrl.Channel()); err != nil { return errors.Wrap(err, "error sending link message") } diff --git a/router/handler_ctrl/fault.go b/router/handler_ctrl/fault.go new file mode 100644 index 000000000..4b6d58944 --- /dev/null +++ b/router/handler_ctrl/fault.go @@ -0,0 +1,70 @@ +/* + Copyright NetFoundry, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package handler_ctrl + +import ( + "github.com/golang/protobuf/proto" + "github.com/michaelquigley/pfxlog" + "github.com/openziti/channel" + "github.com/openziti/fabric/pb/ctrl_pb" + "github.com/openziti/fabric/router/xlink" +) + +type faultHandler struct { + xlinkRegistry xlink.Registry +} + +func newFaultHandler(registry xlink.Registry) *faultHandler { + return &faultHandler{xlinkRegistry: registry} +} + +func (self *faultHandler) ContentType() int32 { + return int32(ctrl_pb.ContentType_FaultType) +} + +func (self *faultHandler) HandleReceive(msg *channel.Message, ch channel.Channel) { + log := pfxlog.ContextLogger(ch.Label()) + + fault := &ctrl_pb.Fault{} + if err := proto.Unmarshal(msg.Body, fault); err != nil { + log.WithError(err).Error("failed to unmarshal fault message") + return + } + + go self.handleFault(msg, ch, fault) +} + +func (self *faultHandler) handleFault(_ *channel.Message, ch channel.Channel, fault *ctrl_pb.Fault) { + log := pfxlog.ContextLogger(ch.Label()).Entry + + switch fault.Subject { + case ctrl_pb.FaultSubject_LinkFault: + linkId := fault.Id + log = log.WithField("linkId", linkId) + if link, _ := self.xlinkRegistry.GetLinkById(linkId); link != nil { + log.Info("link fault reported, closing") + if err := link.CloseNotified(); err != nil { + log.WithError(err).Error("failure closing link") + } + } else { + log.Info("link fault reported, link already closed or unknown") + } + + default: + log.WithField("subject", fault.Subject.String()).Error("unhandled fault subject") + } +} diff --git a/router/handler_ctrl/inspect.go b/router/handler_ctrl/inspect.go index d25c54309..80765db66 100644 --- a/router/handler_ctrl/inspect.go +++ b/router/handler_ctrl/inspect.go @@ -17,21 +17,28 @@ package handler_ctrl import ( + "encoding/json" "github.com/golang/protobuf/proto" "github.com/michaelquigley/pfxlog" "github.com/openziti/channel" "github.com/openziti/fabric/pb/ctrl_pb" + "github.com/openziti/fabric/router/xlink" "github.com/openziti/foundation/identity/identity" "github.com/openziti/foundation/util/debugz" + "github.com/pkg/errors" "strings" ) type inspectHandler struct { - id *identity.TokenId + id *identity.TokenId + xlinkRegistry xlink.Registry } -func newInspectHandler(id *identity.TokenId) *inspectHandler { - return &inspectHandler{id: id} +func newInspectHandler(id *identity.TokenId, xlinkRegistry xlink.Registry) *inspectHandler { + return &inspectHandler{ + id: id, + xlinkRegistry: xlinkRegistry, + } } func (*inspectHandler) ContentType() int32 { @@ -73,6 +80,22 @@ func (context *inspectRequestContext) processLocal() { for _, requested := range context.request.RequestedValues { if strings.ToLower(requested) == "stackdump" { context.appendValue(requested, debugz.GenerateStack()) + } else if strings.ToLower(requested) == "links" { + var result []*linkInspectResult + for link := range context.handler.xlinkRegistry.Iter() { + result = append(result, &linkInspectResult{ + Id: link.Id().Token, + Type: link.LinkType(), + Dest: link.DestinationId(), + DestVersion: link.DestVersion(), + }) + } + js, err := json.Marshal(result) + if err != nil { + context.appendError(errors.Wrap(err, "failed to marshal links to json").Error()) + } else { + context.appendValue(requested, string(js)) + } } } } @@ -102,3 +125,10 @@ func (context *inspectRequestContext) appendError(err string) { context.response.Success = false context.response.Errors = append(context.response.Errors, err) } + +type linkInspectResult struct { + Id string `json:"id"` + Type string `json:"type"` + Dest string `json:"dest"` + DestVersion string `json:"destVersion"` +} diff --git a/router/handler_link/bind.go b/router/handler_link/bind.go index ea06e4f0f..28cf4d772 100644 --- a/router/handler_link/bind.go +++ b/router/handler_link/bind.go @@ -11,19 +11,22 @@ import ( "github.com/openziti/fabric/router/xgress" "github.com/openziti/fabric/router/xlink" "github.com/openziti/fabric/trace" + "github.com/openziti/foundation/common" "github.com/openziti/foundation/metrics" + "github.com/openziti/foundation/util/concurrenz" nfpem "github.com/openziti/foundation/util/pem" "github.com/pkg/errors" "github.com/sirupsen/logrus" "time" ) -func NewBindHandlerFactory(c xgress.CtrlChannel, f *forwarder.Forwarder, fo *forwarder.Options, mr metrics.Registry) *bindHandlerFactory { +func NewBindHandlerFactory(c xgress.CtrlChannel, f *forwarder.Forwarder, fo *forwarder.Options, mr metrics.Registry, registry xlink.Registry) *bindHandlerFactory { return &bindHandlerFactory{ ctrl: c, forwarder: f, forwarderOptions: fo, metricsRegistry: mr, + xlinkRegistry: registry, } } @@ -32,6 +35,7 @@ type bindHandlerFactory struct { forwarder *forwarder.Forwarder forwarderOptions *forwarder.Options metricsRegistry metrics.Registry + xlinkRegistry xlink.Registry } func (self *bindHandlerFactory) NewBindHandler(link xlink.Xlink, latency bool, listenerSide bool) channel.BindHandler { @@ -58,21 +62,46 @@ func (self *bindHandler) BindChannel(binding channel.Binding) error { } } + log := pfxlog.Logger().WithFields(map[string]interface{}{ + "linkId": self.xlink.Id().Token, + "routerId": self.xlink.DestinationId(), + "routerVersion": self.xlink.DestVersion(), + }) + closeNotify := make(chan struct{}) binding.GetChannel().SetLogicalName("l/" + self.xlink.Id().Token) binding.SetUserData(self.xlink.Id().Token) - binding.AddCloseHandler(newCloseHandler(self.xlink, self.ctrl, self.forwarder, closeNotify)) + binding.AddCloseHandler(newCloseHandler(self.xlink, self.ctrl, self.forwarder, closeNotify, self.xlinkRegistry)) binding.AddErrorHandler(newErrorHandler(self.xlink, self.ctrl)) binding.AddTypedReceiveHandler(newPayloadHandler(self.xlink, self.forwarder)) binding.AddTypedReceiveHandler(newQueuingAckHandler(self.xlink, self.forwarder, closeNotify)) binding.AddTypedReceiveHandler(&latency.LatencyHandler{}) - binding.AddTypedReceiveHandler(newControlHandler(self.xlink, self.forwarder, closeNotify)) + binding.AddTypedReceiveHandler(newControlHandler(self.xlink, self.forwarder)) binding.AddPeekHandler(metrics2.NewChannelPeekHandler(self.xlink.Id().Token, self.forwarder.MetricsRegistry())) binding.AddPeekHandler(trace.NewChannelPeekHandler(self.xlink.Id().Token, ch, self.forwarder.TraceController(), trace.NewChannelSink(self.ctrl.Channel()))) - if self.trackLatency { - latencyMetric := self.metricsRegistry.Histogram("link." + self.xlink.Id().Token + ".latency") + doHeartbeat := self.getDestVersionInfo().HasMinimumVersion("0.25.0") + + latencyMetric := self.metricsRegistry.Histogram("link." + self.xlink.Id().Token + ".latency") + queueTimeMetric := self.metricsRegistry.Histogram("link." + self.xlink.Id().Token + ".queue_time") + binding.AddCloseHandler(channel.CloseHandlerF(func(ch channel.Channel) { + latencyMetric.Dispose() + queueTimeMetric.Dispose() + })) + + if doHeartbeat { + log.Info("link destination support heartbeats") + cb := &heartbeatCallback{ + latencyMetric: latencyMetric, + queueTimeMetric: queueTimeMetric, + ch: binding.GetChannel(), + latencySemaphore: concurrenz.NewSemaphore(2), + } + + channel.ConfigureHeartbeat(binding, 10*time.Second, time.Second, cb) + } else if self.trackLatency { + log.Info("link destination does not support heartbeats, using latency probe") latencyTimeout := self.forwarderOptions.LatencyProbeTimeout config := &latency.ProbeConfig{ Channel: ch, @@ -84,9 +113,6 @@ func (self *bindHandler) BindChannel(binding channel.Binding) error { TimeoutHandler: func() { pfxlog.ContextLogger(ch.Label()).Errorf("latency timeout after [%s] on channel [%s]", latencyTimeout, ch.Label()) }, - ExitHandler: func() { - latencyMetric.Dispose() - }, } go latency.ProbeLatencyConfigurable(config) @@ -95,10 +121,16 @@ func (self *bindHandler) BindChannel(binding channel.Binding) error { return nil } +func (self *bindHandler) getDestVersionInfo() *common.VersionInfo { + return &common.VersionInfo{ + Version: self.xlink.DestVersion(), + } +} + func (self *bindHandler) verifyLink(l xlink.Xlink, ch channel.Channel) error { var fingerprints []string for _, cert := range ch.Certificates() { - fingerprints = append(fingerprints, nfpem.FingerprintFromX509(cert)) + fingerprints = append(fingerprints, nfpem.FingerprintFromCertificate(cert)) } verifyLink := &ctrl_pb.VerifyLink{ @@ -123,3 +155,60 @@ func (self *bindHandler) verifyLink(l xlink.Xlink, ch channel.Channel) error { return errors.Errorf("unable to verify link [%v]", result.Message) } + +type heartbeatCallback struct { + latencyMetric metrics.Histogram + queueTimeMetric metrics.Histogram + firstSent int64 + lastResponse int64 + ch channel.Channel + latencySemaphore concurrenz.Semaphore +} + +func (self *heartbeatCallback) HeartbeatTx(int64) { + if self.firstSent == 0 { + self.firstSent = time.Now().UnixMilli() + } +} + +func (self *heartbeatCallback) HeartbeatRx(int64) {} + +func (self *heartbeatCallback) HeartbeatRespTx(int64) {} + +func (self *heartbeatCallback) HeartbeatRespRx(ts int64) { + now := time.Now() + self.lastResponse = now.UnixMilli() + self.latencyMetric.Update(time.Now().UnixNano() - ts) +} + +func (self *heartbeatCallback) CheckHeartBeat() { + log := pfxlog.Logger().WithField("channelId", self.ch.Label()) + now := time.Now().UnixMilli() + if self.firstSent != 0 && (now-self.firstSent > 30000) && (now-self.lastResponse > 30000) { + log.Error("heartbeat not received in time, closing link") + if err := self.ch.Close(); err != nil { + log.WithError(err).Error("error while closing link") + } + } + go self.checkQueueTime() +} + +func (self *heartbeatCallback) checkQueueTime() { + log := pfxlog.Logger().WithField("linkId", self.ch.Id().Token) + if !self.latencySemaphore.TryAcquire() { + log.Warn("unable to check queue time, too many check already running") + return + } + + defer self.latencySemaphore.Release() + + sendTracker := &latency.SendTimeTracker{ + Handler: func(latencyType latency.Type, latency time.Duration) { + self.queueTimeMetric.Update(latency.Nanoseconds()) + }, + StartTime: time.Now(), + } + if err := self.ch.Send(sendTracker); err != nil && !self.ch.IsClosed() { + log.WithError(err).Error("unable to send queue time tracer") + } +} diff --git a/router/handler_link/close.go b/router/handler_link/close.go index c71be4652..39053094f 100644 --- a/router/handler_link/close.go +++ b/router/handler_link/close.go @@ -28,19 +28,21 @@ import ( ) type closeHandler struct { - link xlink.Xlink - ctrl xgress.CtrlChannel - forwarder *forwarder.Forwarder - closeNotify chan struct{} - closed concurrenz.AtomicBoolean + link xlink.Xlink + ctrl xgress.CtrlChannel + forwarder *forwarder.Forwarder + closeNotify chan struct{} + closed concurrenz.AtomicBoolean + xlinkRegistry xlink.Registry } -func newCloseHandler(link xlink.Xlink, ctrl xgress.CtrlChannel, forwarder *forwarder.Forwarder, closeNotify chan struct{}) *closeHandler { +func newCloseHandler(link xlink.Xlink, ctrl xgress.CtrlChannel, forwarder *forwarder.Forwarder, closeNotify chan struct{}, registry xlink.Registry) *closeHandler { return &closeHandler{ - link: link, - ctrl: ctrl, - forwarder: forwarder, - closeNotify: closeNotify, + link: link, + ctrl: ctrl, + forwarder: forwarder, + closeNotify: closeNotify, + xlinkRegistry: registry, } } @@ -51,16 +53,22 @@ func (self *closeHandler) HandleClose(ch channel.Channel) { WithField("routerId", self.link.DestinationId()) // ensure that both parts of a split link are closed, if one side closes - go func() { _ = self.link.Close() }() + go func() { + _ = self.link.Close() + // Close can be called from the link registry, so we can't call back into it from the same go-routine + self.xlinkRegistry.LinkClosed(self.link) + }() log.Info("link closed") - fault := &ctrl_pb.Fault{Subject: ctrl_pb.FaultSubject_LinkFault, Id: self.link.Id().Token} - if err := protobufs.MarshalTyped(fault).Send(self.ctrl.Channel()); err == nil { - log.Debug("transmitted link fault") - } else { - log.WithError(err).Error("unexpected error transmitting link fault") - } + self.link.HandleCloseNotification(func() { + fault := &ctrl_pb.Fault{Subject: ctrl_pb.FaultSubject_LinkFault, Id: self.link.Id().Token} + if err := protobufs.MarshalTyped(fault).Send(self.ctrl.Channel()); err == nil { + log.Debug("transmitted link fault") + } else { + log.WithError(err).Error("unexpected error transmitting link fault") + } + }) self.forwarder.UnregisterLink(self.link) close(self.closeNotify) diff --git a/router/handler_link/control.go b/router/handler_link/control.go index 5683f38b9..ac888a8a1 100644 --- a/router/handler_link/control.go +++ b/router/handler_link/control.go @@ -29,7 +29,7 @@ type controlHandler struct { forwarder *forwarder.Forwarder } -func newControlHandler(link xlink.Xlink, forwarder *forwarder.Forwarder, closeNotify <-chan struct{}) *controlHandler { +func newControlHandler(link xlink.Xlink, forwarder *forwarder.Forwarder) *controlHandler { result := &controlHandler{ link: link, forwarder: forwarder, diff --git a/router/link_registry.go b/router/link_registry.go new file mode 100644 index 000000000..47263634d --- /dev/null +++ b/router/link_registry.go @@ -0,0 +1,248 @@ +package router + +import ( + "fmt" + "github.com/michaelquigley/pfxlog" + "github.com/openziti/channel" + "github.com/openziti/channel/protobufs" + "github.com/openziti/fabric/pb/ctrl_pb" + "github.com/openziti/fabric/router/xlink" + "github.com/openziti/foundation/storage/boltz" + "github.com/sirupsen/logrus" + "sync" + "time" +) + +func NewLinkRegistry() xlink.Registry { + return &linkRegistryImpl{ + linkMap: map[string]xlink.Xlink{}, + linkByIdMap: map[string]xlink.Xlink{}, + dialLocks: map[string]int64{}, + } +} + +type linkRegistryImpl struct { + linkMap map[string]xlink.Xlink + linkByIdMap map[string]xlink.Xlink + dialLocks map[string]int64 + sync.Mutex + ctrlCh channel.Channel +} + +func (self *linkRegistryImpl) GetLink(routerId, linkType string) (xlink.Xlink, bool) { + self.Lock() + defer self.Unlock() + + key := self.getLookupKey(routerId, linkType) + val, found := self.linkMap[key] + if found { + return val, true + } + return nil, false +} + +func (self *linkRegistryImpl) GetLinkById(linkId string) (xlink.Xlink, bool) { + self.Lock() + defer self.Unlock() + + link, found := self.linkByIdMap[linkId] + return link, found +} + +func (self *linkRegistryImpl) GetDialLock(dial xlink.Dial) (xlink.Xlink, bool) { + self.Lock() + defer self.Unlock() + key := self.getDialLookupKey(dial) + + val, found := self.linkMap[key] + if found { + self.sendRouterLinkMessage(val) + return val, false + } + + if val, found := self.dialLocks[key]; found { + if val > time.Now().Add(-time.Minute).UnixMilli() { + return nil, false + } + } + + if len(self.dialLocks) > 100 { + self.purgeOldDialLocks() + } + + self.dialLocks[key] = time.Now().UnixMilli() + return nil, true +} + +func (self *linkRegistryImpl) purgeOldDialLocks() { + count := 0 + minuteAgo := time.Now().Add(-time.Minute).UnixMilli() + for k, v := range self.dialLocks { + if v < minuteAgo { + delete(self.dialLocks, k) + count++ + } + } + if count > 0 { + logrus.WithField("locksDelete", count).Warn("found old link dial locks") + } +} + +func (self *linkRegistryImpl) getDialLookupKey(dial xlink.Dial) string { + return self.getLookupKey(dial.GetRouterId(), dial.GetLinkType()) +} + +func (self *linkRegistryImpl) getLinkLookupKey(link xlink.Xlink) string { + return self.getLookupKey(link.DestinationId(), link.LinkType()) +} + +func (self *linkRegistryImpl) getLookupKey(routerId, linkType string) string { + key := fmt.Sprintf("%v#%v", routerId, linkType) + return key +} + +func (self *linkRegistryImpl) DialFailed(dial xlink.Dial) { + self.Lock() + defer self.Unlock() + key := self.getDialLookupKey(dial) + delete(self.dialLocks, key) +} + +func (self *linkRegistryImpl) LinkAccepted(link xlink.Xlink) (xlink.Xlink, bool) { + self.Lock() + defer self.Unlock() + return self.applyLink(link) +} + +func (self *linkRegistryImpl) DialSucceeded(link xlink.Xlink) (xlink.Xlink, bool) { + self.Lock() + defer self.Unlock() + key := self.getLinkLookupKey(link) + delete(self.dialLocks, key) + return self.applyLink(link) +} + +func (self *linkRegistryImpl) applyLink(link xlink.Xlink) (xlink.Xlink, bool) { + key := self.getLinkLookupKey(link) + if existing := self.linkMap[key]; existing != nil { + log := logrus.WithField("dest", link.DestinationId()). + WithField("linkType", link.LinkType()). + WithField("currentLinkId", existing.Id().Token). + WithField("newLinkId", link.Id().Token) + if existing.Id().Token < link.Id().Token { + log.Info("duplicate link detected. closing new link (new link id is > than current link id)") + self.sendRouterLinkMessage(existing) + if err := link.Close(); err != nil { + log.WithError(err).Error("error closing duplicate link") + } + return existing, false + } + log.Info("duplicate link detected. closing current link (current link id is > than new link id)") + + // report link fault, then close link after allowing some time for circuits to be re-routed + fault := &ctrl_pb.Fault{ + Id: existing.Id().Token, + Subject: ctrl_pb.FaultSubject_LinkFault, + } + + if err := protobufs.MarshalTyped(fault).Send(self.ctrlCh); err != nil { + logrus.WithError(err).Error("failed to send router fault when duplicate link detected") + } + + time.AfterFunc(5*time.Minute, func() { + _ = existing.Close() + }) + } + self.linkMap[key] = link + self.linkByIdMap[link.Id().Token] = link + return nil, true +} + +func (self *linkRegistryImpl) LinkClosed(link xlink.Xlink) { + self.Lock() + defer self.Unlock() + key := self.getLinkLookupKey(link) + if val := self.linkMap[key]; val == link { + delete(self.linkMap, key) + } + delete(self.linkByIdMap, link.Id().Token) +} + +func (self *linkRegistryImpl) Shutdown() { + for link := range self.Iter() { + _ = link.Close() + } +} + +func (self *linkRegistryImpl) sendRouterLinkMessage(link xlink.Xlink) { + linkMsg := &ctrl_pb.RouterLinks{ + Links: []*ctrl_pb.RouterLinks_RouterLink{ + { + Id: link.Id().Token, + DestRouterId: link.DestinationId(), + LinkType: link.LinkType(), + }, + }, + } + if err := protobufs.MarshalTyped(linkMsg).Send(self.ctrlCh); err != nil { + pfxlog.Logger().WithField("linkId", link.Id().Token). + WithField("dest", link.DestinationId()). + WithField("linkType", link.LinkType()). + WithError(err).Error("error sending router link message") + } +} + +/* XCtrl implementation so we get reconnect notifications */ + +func (self *linkRegistryImpl) LoadConfig(map[interface{}]interface{}) error { + return nil +} + +func (self *linkRegistryImpl) BindChannel(channel.Binding) error { + return nil +} + +func (self *linkRegistryImpl) Enabled() bool { + return true +} + +func (self *linkRegistryImpl) Run(ctrl channel.Channel, _ boltz.Db, _ chan struct{}) error { + self.ctrlCh = ctrl + return nil +} + +func (self *linkRegistryImpl) Iter() <-chan xlink.Xlink { + result := make(chan xlink.Xlink, len(self.linkMap)) + go func() { + self.Lock() + defer self.Unlock() + + for _, link := range self.linkMap { + select { + case result <- link: + default: + } + } + close(result) + }() + return result +} + +func (self *linkRegistryImpl) NotifyOfReconnect() { + routerLinks := &ctrl_pb.RouterLinks{} + for link := range self.Iter() { + routerLinks.Links = append(routerLinks.Links, &ctrl_pb.RouterLinks_RouterLink{ + Id: link.Id().Token, + DestRouterId: link.DestinationId(), + LinkType: link.LinkType(), + }) + } + + if err := protobufs.MarshalTyped(routerLinks).Send(self.ctrlCh); err != nil { + logrus.WithError(err).Error("failed to send router links on reconnect") + } +} + +func (self *linkRegistryImpl) GetTraceDecoders() []channel.TraceMessageDecoder { + return nil +} diff --git a/router/router.go b/router/router.go index 1d81f8120..d66ec5fe6 100644 --- a/router/router.go +++ b/router/router.go @@ -68,6 +68,7 @@ type Router struct { xlinkFactories map[string]xlink.Factory xlinkListeners []xlink.Listener xlinkDialers []xlink.Dialer + xlinkRegistry xlink.Registry xgressListeners []xgress.Listener metricsRegistry metrics.UsageRegistry shutdownC chan struct{} @@ -124,6 +125,7 @@ func Create(config *Config, versionProvider common.VersionProvider) *Router { versionProvider: versionProvider, debugOperations: map[byte]func(c *bufio.ReadWriter) error{}, xwebFactoryRegistry: xweb.NewWebHandlerFactoryRegistryImpl(), + xlinkRegistry: NewLinkRegistry(), } } @@ -205,6 +207,8 @@ func (self *Router) Shutdown() error { } } + self.xlinkRegistry.Shutdown() + for _, xgressListener := range self.xgressListeners { if err := xgressListener.Close(); err != nil { errors = append(errors, err) @@ -266,12 +270,14 @@ func (self *Router) startProfiling() { func (self *Router) registerComponents() error { self.xlinkFactories = make(map[string]xlink.Factory) xlinkAccepter := newXlinkAccepter(self.forwarder) - xlinkChAccepter := handler_link.NewBindHandlerFactory(self, + xlinkChAccepter := handler_link.NewBindHandlerFactory( + self, self.forwarder, self.config.Forwarder, self.metricsRegistry, + self.xlinkRegistry, ) - self.xlinkFactories["transport"] = xlink_transport.NewFactory(xlinkAccepter, xlinkChAccepter, self.config.Transport) + self.xlinkFactories["transport"] = xlink_transport.NewFactory(xlinkAccepter, xlinkChAccepter, self.config.Transport, self.xlinkRegistry) xgress.GlobalRegistry().Register("proxy", xgress_proxy.NewFactory(self.config.Id, self, self.config.Transport)) xgress.GlobalRegistry().Register("proxy_udp", xgress_proxy_udp.NewFactory(self)) @@ -282,6 +288,10 @@ func (self *Router) registerComponents() error { return err } + if err := self.RegisterXctrl(self.xlinkRegistry); err != nil { + return err + } + return nil } @@ -383,9 +393,25 @@ func (self *Router) startControlPlane() error { attributes[channel.HelloRouterAdvertisementsHeader] = []byte(self.xlinkListeners[0].GetAdvertisement()) } + listeners := &ctrl_pb.Listeners{} + for _, listener := range self.xlinkListeners { + listeners.Listeners = append(listeners.Listeners, &ctrl_pb.Listener{ + Address: listener.GetAdvertisement(), + Type: listener.GetType(), + }) + } + + if len(listeners.Listeners) > 0 { + if buf, err := proto.Marshal(listeners); err != nil { + return errors.Wrap(err, "unable to marshal Listeners") + } else { + attributes[int32(ctrl_pb.ContentType_ListenersHeader)] = buf + } + } + reconnectHandler := func() { for _, x := range self.xctrls { - x.NotifyOfReconnect() + go x.NotifyOfReconnect() } } @@ -400,6 +426,7 @@ func (self *Router) startControlPlane() error { self.xctrls, self.config, self.config.Trace.Handler, + self.xlinkRegistry, self.shutdownC, ) @@ -458,6 +485,7 @@ const ( UpdateRoute byte = 2 CloseControlChannel byte = 3 OpenControlChannel byte = 4 + DumpLinks byte = 5 ) func (self *Router) RegisterDefaultDebugOps() { @@ -465,6 +493,7 @@ func (self *Router) RegisterDefaultDebugOps() { self.debugOperations[UpdateRoute] = self.debugOpUpdateRouter self.debugOperations[CloseControlChannel] = self.debugOpCloseControlChannel self.debugOperations[OpenControlChannel] = self.debugOpOpenControlChannel + self.debugOperations[DumpLinks] = self.debugOpWriteLinks } func (self *Router) RegisterDebugOp(opId byte, f func(c *bufio.ReadWriter) error) { @@ -477,6 +506,23 @@ func (self *Router) debugOpWriteForwarderTables(c *bufio.ReadWriter) error { return err } +func (self *Router) debugOpWriteLinks(c *bufio.ReadWriter) error { + noLinks := true + for link := range self.xlinkRegistry.Iter() { + line := fmt.Sprintf("id: %v dest: %v type: %v\n", link.Id().Token, link.DestinationId(), link.LinkType()) + _, err := c.WriteString(line) + if err != nil { + return err + } + noLinks = false + } + if noLinks { + _, err := c.WriteString("no links\n") + return err + } + return nil +} + func (self *Router) debugOpUpdateRouter(c *bufio.ReadWriter) error { logrus.Error("received debug operation to update routes") sizeBuf := make([]byte, 4) @@ -546,7 +592,10 @@ func (self *Router) HandleDebug(conn io.ReadWriter) error { } if opF, ok := self.debugOperations[op]; ok { - return opF(bconn) + if err := opF(bconn); err != nil { + return err + } + return bconn.Flush() } return errors.Errorf("invalid operation %v", op) } diff --git a/router/xlink/xlink.go b/router/xlink/xlink.go index 6a3266747..77ea73d86 100644 --- a/router/xlink/xlink.go +++ b/router/xlink/xlink.go @@ -17,11 +17,37 @@ package xlink import ( + "github.com/openziti/fabric/controller/xctrl" "github.com/openziti/fabric/router/xgress" "github.com/openziti/foundation/identity/identity" "github.com/openziti/foundation/transport" ) +// Registry contains known link instances and manages link de-duplication +type Registry interface { + xctrl.Xctrl + + // GetLink returns the link to the given router, of the given type, if one exists + GetLink(routerId, linkType string) (Xlink, bool) + // GetLinkById returns the link for the given id, if it exists + GetLinkById(linkId string) (Xlink, bool) + // GetDialLock tries to acquire a dial lock for the given dial attempt + GetDialLock(dial Dial) (Xlink, bool) + // DialFailed notifies the registry that a dial failed + DialFailed(dial Dial) + // DialSucceeded notifies the registry that a dial succeed and provides the resulting link + DialSucceeded(link Xlink) (Xlink, bool) + // LinkAccepted notifes the registry that a link listener accepted a dial and provides the resulting link + LinkAccepted(link Xlink) (Xlink, bool) + // LinkClosed notifies the registry that a link closed + LinkClosed(link Xlink) + // Iter provides a channel which returns all known links + Iter() <-chan Xlink + // Shutdown frees any resources owned by the registry + Shutdown() +} + +// A Factory creates link listeners and link dialers type Factory interface { CreateListener(id *identity.TokenId, f Forwarder, config transport.Configuration) (Listener, error) CreateDialer(id *identity.TokenId, f Forwarder, config transport.Configuration) (Dialer, error) @@ -30,6 +56,7 @@ type Factory interface { type Listener interface { Listen() error GetAdvertisement() string + GetType() string Close() error } @@ -37,8 +64,17 @@ type Acceptor interface { Accept(xlink Xlink) error } +// A Dial contains the information need to dial another router +type Dial interface { + GetLinkId() string + GetRouterId() string + GetAddress() string + GetLinkType() string + GetRouterVersion() string +} + type Dialer interface { - Dial(address string, id *identity.TokenId, routerId string) error + Dial(dial Dial) (Xlink, error) } type Xlink interface { @@ -47,7 +83,11 @@ type Xlink interface { SendAcknowledgement(acknowledgement *xgress.Acknowledgement) error SendControl(control *xgress.Control) error Close() error + CloseNotified() error DestinationId() string + DestVersion() string + LinkType() string + HandleCloseNotification(f func()) } type Forwarder interface { diff --git a/router/xlink_transport/config.go b/router/xlink_transport/config.go index 5f078e530..2d11090a3 100644 --- a/router/xlink_transport/config.go +++ b/router/xlink_transport/config.go @@ -46,6 +46,7 @@ func loadListenerConfig(data map[interface{}]interface{}) (*listenerConfig, erro if addressString, ok := value.(string); ok { if address, err := transport.ParseAddress(addressString); err == nil { config.advertise = address + config.linkType = address.Type() } else { return nil, fmt.Errorf("error parsing 'advertise' address in listener config") } @@ -54,6 +55,14 @@ func loadListenerConfig(data map[interface{}]interface{}) (*listenerConfig, erro } } + if value, found := data["type"]; found { + if typeString, ok := value.(string); ok { + config.linkType = typeString + } else { + return nil, fmt.Errorf("invalid 'type' address in listener config (%s)", reflect.TypeOf(value)) + } + } + if value, found := data["options"]; found { if submap, ok := value.(map[interface{}]interface{}); ok { options, err := channel.LoadOptions(submap) @@ -74,6 +83,7 @@ func loadListenerConfig(data map[interface{}]interface{}) (*listenerConfig, erro type listenerConfig struct { bind transport.Address advertise transport.Address + linkType string options *channel.Options } diff --git a/router/xlink_transport/dialer.go b/router/xlink_transport/dialer.go index ab2671a84..562575d36 100644 --- a/router/xlink_transport/dialer.go +++ b/router/xlink_transport/dialer.go @@ -34,46 +34,52 @@ type dialer struct { transportConfig transport.Configuration } -func (self *dialer) Dial(addressString string, linkId *identity.TokenId, routerId string) error { - address, err := transport.ParseAddress(addressString) +func (self *dialer) Dial(dial xlink.Dial) (xlink.Xlink, error) { + address, err := transport.ParseAddress(dial.GetAddress()) if err != nil { - return errors.Wrapf(err, "error parsing link address [%s]", addressString) + return nil, errors.Wrapf(err, "error parsing link address [%s]", dial.GetAddress()) } + linkId := self.id.ShallowCloneWithNewToken(dial.GetLinkId()) connId := uuid.New().String() var xli xlink.Xlink if self.config.split { - xli, err = self.dialSplit(linkId, address, routerId, connId) + xli, err = self.dialSplit(linkId, address, connId, dial) } else { - xli, err = self.dialSingle(linkId, address, routerId, connId) + xli, err = self.dialSingle(linkId, address, connId, dial) } if err != nil { - return errors.Wrapf(err, "error dialing outgoing link [l/%s]", linkId.Token) + return nil, errors.Wrapf(err, "error dialing outgoing link [l/%s]", linkId.Token) } if err := self.acceptor.Accept(xli); err != nil { - return errors.Wrapf(err, "error accepting link [l/%s]", linkId.Token) + return nil, errors.Wrapf(err, "error accepting link [l/%s]", linkId.Token) } - return nil + return xli, nil } -func (self *dialer) dialSplit(linkId *identity.TokenId, address transport.Address, routerId, connId string) (xlink.Xlink, error) { +func (self *dialer) dialSplit(linkId *identity.TokenId, address transport.Address, connId string, dial xlink.Dial) (xlink.Xlink, error) { logrus.Debugf("dialing link with split payload/ack channels [l/%s]", linkId.Token) payloadDialer := channel.NewClassicDialer(linkId, address, map[int32][]byte{ - LinkHeaderRouterId: []byte(routerId), - LinkHeaderConnId: []byte(connId), - LinkHeaderType: {byte(PayloadChannel)}, + LinkHeaderRouterId: []byte(self.id.Token), + LinkHeaderConnId: []byte(connId), + LinkHeaderType: {byte(PayloadChannel)}, + LinkHeaderRouterVersion: []byte(dial.GetRouterVersion()), }) logrus.Debugf("dialing payload channel for [l/%s]", linkId.Token) bindHandler := &splitDialBindHandler{ dialer: self, - link: &splitImpl{id: linkId, routerId: routerId}, + link: &splitImpl{id: linkId, + routerId: dial.GetRouterId(), + routerVersion: dial.GetRouterVersion(), + linkType: dial.GetLinkType(), + }, } payloadCh, err := channel.NewChannelWithTransportConfiguration("l/"+linkId.Token, payloadDialer, channel.BindHandlerF(bindHandler.bindPayloadChannel), self.config.options, self.transportConfig) @@ -84,8 +90,10 @@ func (self *dialer) dialSplit(linkId *identity.TokenId, address transport.Addres logrus.Debugf("dialing ack channel for [l/%s]", linkId.Token) ackDialer := channel.NewClassicDialer(linkId, address, map[int32][]byte{ - LinkHeaderConnId: []byte(connId), - LinkHeaderType: {byte(AckChannel)}, + LinkHeaderRouterId: []byte(self.id.Token), + LinkHeaderConnId: []byte(connId), + LinkHeaderType: {byte(AckChannel)}, + LinkHeaderRouterVersion: []byte(dial.GetRouterVersion()), }) _, err = channel.NewChannelWithTransportConfiguration("l/"+linkId.Token, ackDialer, channel.BindHandlerF(bindHandler.bindAckChannel), self.config.options, self.transportConfig) @@ -97,17 +105,23 @@ func (self *dialer) dialSplit(linkId *identity.TokenId, address transport.Addres return bindHandler.link, nil } -func (self *dialer) dialSingle(linkId *identity.TokenId, address transport.Address, routerId, connId string) (xlink.Xlink, error) { +func (self *dialer) dialSingle(linkId *identity.TokenId, address transport.Address, connId string, dial xlink.Dial) (xlink.Xlink, error) { logrus.Debugf("dialing link with single channel [l/%s]", linkId.Token) payloadDialer := channel.NewClassicDialer(linkId, address, map[int32][]byte{ - LinkHeaderRouterId: []byte(routerId), - LinkHeaderConnId: []byte(connId), + LinkHeaderRouterId: []byte(self.id.Token), + LinkHeaderConnId: []byte(connId), + LinkHeaderRouterVersion: []byte(dial.GetRouterVersion()), }) bindHandler := &dialBindHandler{ dialer: self, - link: &impl{id: linkId, routerId: routerId}, + link: &impl{ + id: linkId, + routerId: dial.GetRouterId(), + linkType: dial.GetLinkType(), + routerVersion: dial.GetRouterVersion(), + }, } _, err := channel.NewChannelWithTransportConfiguration("l/"+linkId.Token, payloadDialer, bindHandler, self.config.options, self.transportConfig) diff --git a/router/xlink_transport/factory.go b/router/xlink_transport/factory.go index 1b03ea6e9..eb8d9383a 100644 --- a/router/xlink_transport/factory.go +++ b/router/xlink_transport/factory.go @@ -26,9 +26,10 @@ import ( type channelType byte const ( - LinkHeaderConnId = 0 - LinkHeaderType = 1 - LinkHeaderRouterId = 2 + LinkHeaderConnId = 0 + LinkHeaderType = 1 + LinkHeaderRouterId = 2 + LinkHeaderRouterVersion = 3 PayloadChannel channelType = 1 AckChannel channelType = 2 @@ -44,8 +45,13 @@ func (self channelType) String() string { return "invalid" } -func NewFactory(accepter xlink.Acceptor, bindHandlerFactory BindHandlerFactory, c transport.Configuration) xlink.Factory { - return &factory{acceptor: accepter, bindHandlerFactory: bindHandlerFactory, transportConfig: c} +func NewFactory(accepter xlink.Acceptor, bindHandlerFactory BindHandlerFactory, c transport.Configuration, registry xlink.Registry) xlink.Factory { + return &factory{ + acceptor: accepter, + bindHandlerFactory: bindHandlerFactory, + transportConfig: c, + xlinkRegistry: registry, + } } func (self *factory) CreateListener(id *identity.TokenId, _ xlink.Forwarder, configData transport.Configuration) (xlink.Listener, error) { @@ -60,6 +66,7 @@ func (self *factory) CreateListener(id *identity.TokenId, _ xlink.Forwarder, con bindHandlerFactory: self.bindHandlerFactory, tcfg: self.transportConfig, pendingLinks: map[string]*pendingLink{}, + xlinkRegistery: self.xlinkRegistry, }, nil } @@ -81,4 +88,5 @@ type factory struct { acceptor xlink.Acceptor bindHandlerFactory BindHandlerFactory transportConfig transport.Configuration + xlinkRegistry xlink.Registry } diff --git a/router/xlink_transport/listener.go b/router/xlink_transport/listener.go index 2c9ae598a..cffb67684 100644 --- a/router/xlink_transport/listener.go +++ b/router/xlink_transport/listener.go @@ -38,6 +38,7 @@ type listener struct { tcfg transport.Configuration pendingLinks map[string]*pendingLink lock sync.Mutex + xlinkRegistery xlink.Registry } func (self *listener) Listen() error { @@ -57,6 +58,10 @@ func (self *listener) GetAdvertisement() string { return self.config.advertise.String() } +func (self *listener) GetType() string { + return self.config.linkType +} + func (self *listener) Close() error { return self.listener.Close() } @@ -76,11 +81,13 @@ func (self *listener) acceptLoop() { func (self *listener) BindChannel(binding channel.Binding) error { log := pfxlog.ChannelLogger("link", "linkListener"). + WithField("linkType", self.GetType()). WithField("linkId", binding.GetChannel().Id().Token) headers := binding.GetChannel().Underlay().Headers() var chanType channelType routerId := "" + routerVersion := "" if headers != nil { if v, ok := headers[LinkHeaderRouterId]; ok { @@ -91,17 +98,21 @@ func (self *listener) BindChannel(binding channel.Binding) error { if val, ok := headers[LinkHeaderType]; ok { chanType = channelType(val[0]) } + if val, ok := headers[LinkHeaderRouterVersion]; ok { + routerVersion = string(val) + log = log.WithField("routerVersion", routerVersion) + } } if chanType != 0 { log = log.WithField("channelType", chanType) - return self.bindSplitChannel(binding, chanType, routerId, log) + return self.bindSplitChannel(binding, chanType, routerId, routerVersion, log) } - return self.bindNonSplitChannel(binding, routerId, log) + return self.bindNonSplitChannel(binding, routerId, routerVersion, log) } -func (self *listener) bindSplitChannel(binding channel.Binding, chanType channelType, routerId string, log *logrus.Entry) error { +func (self *listener) bindSplitChannel(binding channel.Binding, chanType channelType, routerId, routerVersion string, log *logrus.Entry) error { headers := binding.GetChannel().Underlay().Headers() id, ok := headers[LinkHeaderConnId] if !ok { @@ -110,7 +121,7 @@ func (self *listener) bindSplitChannel(binding channel.Binding, chanType channel log.Info("accepted part of split conn") - xli, err := self.getOrCreateSplitLink(string(id), routerId, binding, chanType) + xli, err := self.getOrCreateSplitLink(string(id), routerId, routerVersion, binding, chanType) if err != nil { log.WithError(err).Error("erroring binding link channel") } @@ -123,15 +134,25 @@ func (self *listener) bindSplitChannel(binding channel.Binding, chanType channel if xli.payloadCh != nil && xli.ackCh != nil { if err := self.accepter.Accept(xli); err != nil { log.WithError(err).Error("error accepting incoming Xlink") - } + if err := xli.Close(); err != nil { + log.WithError(err).Debugf("error closing link") + } + return err + } log.Info("accepted link") + + if existingLink, applied := self.xlinkRegistery.LinkAccepted(xli); applied { + log.Info("link registered") + } else { + log.WithField("existingLinkId", existingLink.Id().Token).Info("existing link found, new link closed") + } } return nil } -func (self *listener) getOrCreateSplitLink(id, routerId string, binding channel.Binding, chanType channelType) (*splitImpl, error) { +func (self *listener) getOrCreateSplitLink(id, routerId, routerVersion string, binding channel.Binding, chanType channelType) (*splitImpl, error) { self.lock.Lock() defer self.lock.Unlock() @@ -143,8 +164,10 @@ func (self *listener) getOrCreateSplitLink(id, routerId string, binding channel. } else { pending = &pendingLink{ link: &splitImpl{ - id: binding.GetChannel().Id(), - routerId: routerId, + id: binding.GetChannel().Id(), + routerId: routerId, + routerVersion: routerVersion, + linkType: self.GetType(), }, eventTime: time.Now(), } @@ -171,11 +194,12 @@ func (self *listener) getOrCreateSplitLink(id, routerId string, binding channel. return link, nil } -func (self *listener) bindNonSplitChannel(binding channel.Binding, routerId string, log *logrus.Entry) error { +func (self *listener) bindNonSplitChannel(binding channel.Binding, routerId, routerVersion string, log *logrus.Entry) error { xli := &impl{ id: binding.GetChannel().Id(), ch: binding.GetChannel(), routerId: routerId, + linkType: self.GetType(), } bindHandler := self.bindHandlerFactory.NewBindHandler(xli, true, true) @@ -193,6 +217,12 @@ func (self *listener) bindNonSplitChannel(binding channel.Binding, routerId stri return err } + if existingLink, applied := self.xlinkRegistery.LinkAccepted(xli); applied { + log.Info("link registered") + } else { + log.WithField("existingLinkId", existingLink.Id().Token).Info("existing link found, new link closed") + } + log.Info("accepted link") return nil } diff --git a/router/xlink_transport/xlink.go b/router/xlink_transport/xlink.go index fbaa8d927..53babad40 100644 --- a/router/xlink_transport/xlink.go +++ b/router/xlink_transport/xlink.go @@ -20,8 +20,18 @@ import ( "github.com/openziti/channel" "github.com/openziti/fabric/router/xgress" "github.com/openziti/foundation/identity/identity" + "github.com/openziti/foundation/util/concurrenz" ) +type impl struct { + id *identity.TokenId + ch channel.Channel + routerId string + routerVersion string + linkType string + closeNotified concurrenz.AtomicBoolean +} + func (self *impl) Id() *identity.TokenId { return self.id } @@ -42,12 +52,25 @@ func (self *impl) Close() error { return self.ch.Close() } +func (self *impl) CloseNotified() error { + self.closeNotified.Set(true) + return self.Close() +} + func (self *impl) DestinationId() string { return self.routerId } -type impl struct { - id *identity.TokenId - ch channel.Channel - routerId string +func (self *impl) DestVersion() string { + return self.routerVersion +} + +func (self *impl) LinkType() string { + return self.linkType +} + +func (self *impl) HandleCloseNotification(f func()) { + if self.closeNotified.CompareAndSwap(false, true) { + f() + } } diff --git a/router/xlink_transport/xlink_split.go b/router/xlink_transport/xlink_split.go index 4da6fe888..97937c4f7 100644 --- a/router/xlink_transport/xlink_split.go +++ b/router/xlink_transport/xlink_split.go @@ -20,9 +20,20 @@ import ( "github.com/openziti/channel" "github.com/openziti/fabric/router/xgress" "github.com/openziti/foundation/identity/identity" + "github.com/openziti/foundation/util/concurrenz" "github.com/pkg/errors" ) +type splitImpl struct { + id *identity.TokenId + payloadCh channel.Channel + ackCh channel.Channel + routerId string + routerVersion string + linkType string + closeNotified concurrenz.AtomicBoolean +} + func (self *splitImpl) Id() *identity.TokenId { return self.id } @@ -39,6 +50,11 @@ func (self *splitImpl) SendAcknowledgement(acknowledgement *xgress.Acknowledgeme return self.ackCh.Send(acknowledgement.Marshall()) } +func (self *splitImpl) CloseNotified() error { + self.closeNotified.Set(true) + return self.Close() +} + func (self *splitImpl) Close() error { var err, err2 error if self.payloadCh != nil { @@ -61,9 +77,16 @@ func (self *splitImpl) DestinationId() string { return self.routerId } -type splitImpl struct { - id *identity.TokenId - payloadCh channel.Channel - ackCh channel.Channel - routerId string +func (self *splitImpl) DestVersion() string { + return self.routerVersion +} + +func (self *splitImpl) LinkType() string { + return self.linkType +} + +func (self *splitImpl) HandleCloseNotification(f func()) { + if self.closeNotified.CompareAndSwap(false, true) { + f() + } } diff --git a/specs/swagger.yml b/specs/swagger.yml index ca4617c44..b3bfc0286 100644 --- a/specs/swagger.yml +++ b/specs/swagger.yml @@ -1110,10 +1110,22 @@ definitions: maximum: 65535 noTraversal: type: boolean - listenerAddress: - type: string + listenerAddresses: + type: array + items: + $ref: '#/definitions/routerListener' versionInfo: $ref: '#/definitions/versionInfo' + routerListener: + type: object + required: + - address + - type + properties: + address: + type: string + type: + type: string routerCreate: type: object required: @@ -1357,6 +1369,7 @@ definitions: type: object required: - id + - type - sourceRouter - destRouter - state @@ -1368,6 +1381,8 @@ definitions: properties: id: type: string + type: + type: string sourceRouter: $ref: '#/definitions/entityRef' destRouter: @@ -1479,7 +1494,11 @@ definitions: name: type: string value: - type: string + type: + - object + - string + - number + - boolean inspectResponse: type: object required: diff --git a/tests/context.go b/tests/context.go index e7d667898..43a3d1567 100644 --- a/tests/context.go +++ b/tests/context.go @@ -59,7 +59,7 @@ func init() { SetTrimPrefix("github.com/openziti/"). StartingToday() - pfxlog.GlobalInit(logrus.DebugLevel, logOptions) + pfxlog.GlobalInit(logrus.InfoLevel, logOptions) pfxlog.SetFormatter(pfxlog.NewFormatter(logOptions)) _ = os.Setenv("ZITI_TRACE_ENABLED", "false") @@ -199,16 +199,17 @@ func (ctx *TestContext) StartServerFor(test string, clean bool) { ctx.requireRestPort(time.Second * 5) } -func (ctx *TestContext) startRouter(index uint8) { +func (ctx *TestContext) startRouter(index uint8) *router.Router { config, err := router.LoadConfig(fmt.Sprintf(RouterConfFile, index)) ctx.Req.NoError(err) r := router.Create(config, NewVersionProviderTest()) ctx.Req.NoError(r.Start()) ctx.routers = append(ctx.routers, r) + return r } -func (ctx *TestContext) shutdownRouter() { +func (ctx *TestContext) shutdownRouters() { for _, r := range ctx.routers { ctx.Req.NoError(r.Shutdown()) } @@ -243,9 +244,28 @@ func (ctx *TestContext) waitForPort(address string, duration time.Duration) erro } } +func (ctx *TestContext) waitForPortClose(address string, duration time.Duration) error { + now := time.Now() + endTime := now.Add(duration) + maxWait := duration + for { + conn, err := net.DialTimeout("tcp", address, maxWait) + if err != nil { + return nil + } + _ = conn.Close() + now = time.Now() + if !now.Before(endTime) { + return err + } + maxWait = endTime.Sub(now) + time.Sleep(10 * time.Millisecond) + } +} + func (ctx *TestContext) Teardown() { pfxlog.Logger().Info("tearing down test context") - ctx.shutdownRouter() + ctx.shutdownRouters() if ctx.fabricController != nil { ctx.fabricController.Shutdown() ctx.fabricController = nil diff --git a/tests/control.go b/tests/control.go new file mode 100644 index 000000000..1b6fe3bdf --- /dev/null +++ b/tests/control.go @@ -0,0 +1,21 @@ +package tests + +import ( + "github.com/openziti/channel" + "github.com/openziti/fabric/controller" +) + +func (ctx *TestContext) NewControlChannelListener() channel.UnderlayListener { + config, err := controller.LoadConfig(ControllerConfFile) + ctx.Req.NoError(err) + + versionHeader, err := VersionProviderTest{}.EncoderDecoder().Encode(VersionProviderTest{}.AsVersionInfo()) + ctx.Req.NoError(err) + headers := map[int32][]byte{ + channel.HelloVersionHeader: versionHeader, + } + + ctrlListener := channel.NewClassicListener(config.Id, config.Ctrl.Listener, config.Ctrl.Options.ConnectOptions, headers) + ctx.Req.NoError(ctrlListener.Listen()) + return ctrlListener +} diff --git a/tests/link_test.go b/tests/link_test.go index 187148d26..c0d9447a9 100644 --- a/tests/link_test.go +++ b/tests/link_test.go @@ -3,10 +3,16 @@ package tests import ( + "github.com/golang/protobuf/proto" + "github.com/google/uuid" "github.com/openziti/channel" + "github.com/openziti/channel/protobufs" + "github.com/openziti/fabric/pb/ctrl_pb" + "github.com/openziti/fabric/router" "github.com/openziti/fabric/router/xgress" "github.com/openziti/fabric/router/xlink" "github.com/openziti/fabric/router/xlink_transport" + "github.com/openziti/fabric/tests/testutil" "github.com/openziti/foundation/identity/identity" "github.com/openziti/foundation/transport" "github.com/sirupsen/logrus" @@ -32,11 +38,11 @@ func (self *testXlinkAcceptor) getLink() xlink.Xlink { type testBindHandlerFactory struct{} -func (t testBindHandlerFactory) BindChannel(binding channel.Binding) error { +func (t testBindHandlerFactory) BindChannel(channel.Binding) error { return nil } -func (t testBindHandlerFactory) NewBindHandler(xlink xlink.Xlink, latency bool, listenerSide bool) channel.BindHandler { +func (t testBindHandlerFactory) NewBindHandler(xlink.Xlink, bool, bool) channel.BindHandler { return t } @@ -59,11 +65,16 @@ func Test_LinkWithValidCertFromUnknownChain(t *testing.T) { tcfg := transport.Configuration{ "split": false, } - factory := xlink_transport.NewFactory(xla, testBindHandlerFactory{}, tcfg) + factory := xlink_transport.NewFactory(xla, testBindHandlerFactory{}, tcfg, router.NewLinkRegistry()) dialer, err := factory.CreateDialer(badId, nil, tcfg) ctx.Req.NoError(err) - linkId := badId.ShallowCloneWithNewToken("testLinkId") - err = dialer.Dial("tls:127.0.0.1:6004", linkId, "router1") + dial := &ctrl_pb.Dial{ + LinkId: "testLinkId", + Address: "tls:127.0.0.1:6004", + RouterId: "002", + LinkType: "default", + } + _, err = dialer.Dial(dial) ctx.Req.Error(err) ctx.Req.ErrorIs(err, io.EOF) } @@ -88,11 +99,16 @@ func Test_UnrequestedLinkFromValidRouter(t *testing.T) { tcfg := transport.Configuration{ "split": false, } - factory := xlink_transport.NewFactory(xla, testBindHandlerFactory{}, tcfg) + factory := xlink_transport.NewFactory(xla, testBindHandlerFactory{}, tcfg, router.NewLinkRegistry()) dialer, err := factory.CreateDialer(router2Id, nil, tcfg) ctx.Req.NoError(err) - linkId := router2Id.ShallowCloneWithNewToken("testLinkId") - err = dialer.Dial("tls:127.0.0.1:6004", linkId, "router1") + dial := &ctrl_pb.Dial{ + LinkId: "testLinkId", + Address: "tls:127.0.0.1:6004", + RouterId: "002", + LinkType: "default", + } + _, err = dialer.Dial(dial) if err != nil { ctx.Req.ErrorIs(err, io.EOF, "unexpected error: %v", err) } else { @@ -111,3 +127,282 @@ func Test_UnrequestedLinkFromValidRouter(t *testing.T) { ctx.Req.EqualError(err, "channel closed", "unexpected error: %v", err) } } + +func Test_DuplicateLinks(t *testing.T) { + ctx := NewTestContext(t) + defer ctx.Teardown() + ctx.StartServer() + mgmtClient := ctx.createMgmtClient() + mgmtClient.EnrollRouter("001", "router-1", "testdata/router/001-client.cert.pem") + mgmtClient.EnrollRouter("002", "router-2", "testdata/router/002-client.cert.pem") + ctx.Teardown() + + ctrlListener := ctx.NewControlChannelListener() + ctx.startRouter(1) + + acceptControl := func(id string) (channel.Channel, *testutil.MessageCollector) { + return testutil.AcceptControl(id, ctrlListener, ctx.Req) + } + + router1cc, msgc1 := acceptControl("router-1") + + ctx.startRouter(2) + + router2cc, _ := acceptControl("router-2") + + dial1 := &ctrl_pb.Dial{ + LinkId: uuid.NewString(), + RouterId: "002", + Address: "tls:localhost:6005", + LinkType: "tls", + } + + // send initial link request. This should result in a new link + err := protobufs.MarshalTyped(dial1).WithTimeout(time.Second).SendAndWaitForWire(router1cc) + ctx.Req.NoError(err) + + msg, err := msgc1.Next(time.Second) + ctx.Req.NoError(err) + ctx.Req.Equal(int32(ctrl_pb.ContentType_LinkConnectedType), msg.ContentType) + + dial2 := &ctrl_pb.Dial{ + LinkId: uuid.NewString(), + RouterId: "002", + Address: "tls:localhost:6005", + LinkType: "tls", + } + + // send another link request. This should result in the router letting us know about the current link + err = protobufs.MarshalTyped(dial2).WithTimeout(time.Second).SendAndWaitForWire(router1cc) + ctx.Req.NoError(err) + + msg, err = msgc1.Next(time.Second) + ctx.Req.NoError(err) + ctx.Req.Equal(int32(ctrl_pb.ContentType_RouterLinksType), msg.ContentType) + rl := &ctrl_pb.RouterLinks{} + ctx.Req.NoError(proto.Unmarshal(msg.Body, rl)) + ctx.Req.Equal(1, len(rl.Links)) + ctx.Req.Equal(dial1.LinkId, rl.Links[0].Id) + ctx.Req.Equal("002", rl.Links[0].DestRouterId) + + // disconnect and reconnect the router + _ = router1cc.Close() + router1cc, msgc1 = acceptControl("router-1") + + // The router should let us know about existing links + msg, err = msgc1.Next(time.Second) + ctx.Req.NoError(err) + ctx.Req.Equal(int32(ctrl_pb.ContentType_RouterLinksType), msg.ContentType) + rl = &ctrl_pb.RouterLinks{} + ctx.Req.NoError(proto.Unmarshal(msg.Body, rl)) + ctx.Req.Equal(1, len(rl.Links)) + ctx.Req.Equal(dial1.LinkId, rl.Links[0].Id) + ctx.Req.Equal("002", rl.Links[0].DestRouterId) + + // Try one more time, should get another router link notification + err = protobufs.MarshalTyped(dial2).WithTimeout(time.Second).SendAndWaitForWire(router1cc) + ctx.Req.NoError(err) + + msg, err = msgc1.Next(time.Second) + ctx.Req.NoError(err) + ctx.Req.Equal(int32(ctrl_pb.ContentType_RouterLinksType), msg.ContentType) + rl = &ctrl_pb.RouterLinks{} + ctx.Req.NoError(proto.Unmarshal(msg.Body, rl)) + ctx.Req.Equal(1, len(rl.Links)) + ctx.Req.Equal(dial1.LinkId, rl.Links[0].Id) + ctx.Req.Equal("002", rl.Links[0].DestRouterId) + + ctx.Teardown() + _ = router1cc.Close() + _ = router2cc.Close() + _ = ctrlListener.Close() +} + +func Test_DuplicateLinkWithLinkCloseListener(t *testing.T) { + ctx := NewTestContext(t) + defer ctx.Teardown() + ctx.StartServer() + mgmtClient := ctx.createMgmtClient() + mgmtClient.EnrollRouter("001", "router-1", "testdata/router/001-client.cert.pem") + mgmtClient.EnrollRouter("002", "router-2", "testdata/router/002-client.cert.pem") + ctx.Teardown() + + ctrlListener := ctx.NewControlChannelListener() + ctx.startRouter(1) + + acceptControl := func(id string) (channel.Channel, *testutil.MessageCollector) { + return testutil.AcceptControl(id, ctrlListener, ctx.Req) + } + + router1cc, msgc1 := acceptControl("router-1") + + router2 := ctx.startRouter(2) + + router2cc, _ := acceptControl("router-2") + + dial1 := &ctrl_pb.Dial{ + LinkId: uuid.NewString(), + RouterId: "002", + Address: "tls:localhost:6005", + LinkType: "tls", + } + + // send initial link request. This should result in a new link + err := protobufs.MarshalTyped(dial1).WithTimeout(time.Second).SendAndWaitForWire(router1cc) + ctx.Req.NoError(err) + + msg, err := msgc1.Next(time.Second) + ctx.Req.NoError(err) + ctx.Req.Equal(int32(ctrl_pb.ContentType_LinkConnectedType), msg.ContentType) + + dial2 := &ctrl_pb.Dial{ + LinkId: uuid.NewString(), + RouterId: "002", + Address: "tls:localhost:6005", + LinkType: "tls", + } + + // send another link request. This should result in the router letting us know about the current link + err = protobufs.MarshalTyped(dial2).WithTimeout(time.Second).SendAndWaitForWire(router1cc) + ctx.Req.NoError(err) + + msg, err = msgc1.Next(time.Second) + ctx.Req.NoError(err) + ctx.Req.Equal(int32(ctrl_pb.ContentType_RouterLinksType), msg.ContentType) + rl := &ctrl_pb.RouterLinks{} + ctx.Req.NoError(proto.Unmarshal(msg.Body, rl)) + ctx.Req.Equal(1, len(rl.Links)) + ctx.Req.Equal(dial1.LinkId, rl.Links[0].Id) + ctx.Req.Equal("002", rl.Links[0].DestRouterId) + + // The router should also send us a fault response for the dialed link + msg, err = msgc1.Next(5 * time.Second) + ctx.Req.NoError(err) + ctx.Req.Equal(int32(ctrl_pb.ContentType_FaultType), msg.ContentType) + fault := &ctrl_pb.Fault{} + ctx.Req.NoError(proto.Unmarshal(msg.Body, fault)) + ctx.Req.Equal(ctrl_pb.FaultSubject_LinkFault, fault.Subject) + ctx.Req.Equal(dial2.LinkId, fault.Id) + + // disconnect and reconnect the router + ctx.Req.NoError(router2.Shutdown()) + ctx.Req.NoError(ctx.waitForPortClose("localhost:6005", 2*time.Second)) + router2 = ctx.startRouter(2) + + router2cc, _ = acceptControl("router-2") + + // The router should tell us about the faulty link + msg, err = msgc1.Next(5 * time.Second) + ctx.Req.NoError(err) + ctx.Req.Equal(int32(ctrl_pb.ContentType_FaultType), msg.ContentType) + fault = &ctrl_pb.Fault{} + ctx.Req.NoError(proto.Unmarshal(msg.Body, fault)) + ctx.Req.Equal(ctrl_pb.FaultSubject_LinkFault, fault.Subject) + ctx.Req.Equal(dial1.LinkId, fault.Id) + + // Try one more time, this time it should result in a new connection + err = protobufs.MarshalTyped(dial2).WithTimeout(time.Second).SendAndWaitForWire(router1cc) + ctx.Req.NoError(err) + + msg, err = msgc1.Next(time.Second) + ctx.Req.NoError(err) + ctx.Req.Equal(int32(ctrl_pb.ContentType_LinkConnectedType), msg.ContentType) + linkConnected := &ctrl_pb.LinkConnected{} + ctx.Req.NoError(proto.Unmarshal(msg.Body, linkConnected)) + ctx.Req.Equal(dial2.LinkId, linkConnected.Id) + + ctx.Teardown() + _ = router1cc.Close() + _ = router2cc.Close() + _ = ctrlListener.Close() +} + +func Test_DuplicateLinkWithLinkCloseDialer(t *testing.T) { + ctx := NewTestContext(t) + defer ctx.Teardown() + ctx.StartServer() + mgmtClient := ctx.createMgmtClient() + mgmtClient.EnrollRouter("001", "router-1", "testdata/router/001-client.cert.pem") + mgmtClient.EnrollRouter("002", "router-2", "testdata/router/002-client.cert.pem") + ctx.Teardown() + + ctrlListener := ctx.NewControlChannelListener() + router1 := ctx.startRouter(1) + + acceptControl := func(id string) (channel.Channel, *testutil.MessageCollector) { + return testutil.AcceptControl(id, ctrlListener, ctx.Req) + } + + router1cc, msgc1 := acceptControl("router-1") + + ctx.startRouter(2) + + router2cc, msgc2 := acceptControl("router-2") + + dial1 := &ctrl_pb.Dial{ + LinkId: uuid.NewString(), + RouterId: "002", + Address: "tls:localhost:6005", + LinkType: "tls", + } + + // send initial link request. This should result in a new link + err := protobufs.MarshalTyped(dial1).WithTimeout(time.Second).SendAndWaitForWire(router1cc) + ctx.Req.NoError(err) + + msg, err := msgc1.Next(time.Second) + ctx.Req.NoError(err) + ctx.Req.Equal(int32(ctrl_pb.ContentType_LinkConnectedType), msg.ContentType) + + dial2 := &ctrl_pb.Dial{ + LinkId: uuid.NewString(), + RouterId: "002", + Address: "tls:localhost:6005", + LinkType: "tls", + } + + // send another link request. This should result in the router letting us know about the current link + err = protobufs.MarshalTyped(dial2).WithTimeout(time.Second).SendAndWaitForWire(router1cc) + ctx.Req.NoError(err) + + msg, err = msgc1.Next(time.Second) + ctx.Req.NoError(err) + ctx.Req.Equal(int32(ctrl_pb.ContentType_RouterLinksType), msg.ContentType) + rl := &ctrl_pb.RouterLinks{} + ctx.Req.NoError(proto.Unmarshal(msg.Body, rl)) + ctx.Req.Equal(1, len(rl.Links)) + ctx.Req.Equal(dial1.LinkId, rl.Links[0].Id) + ctx.Req.Equal("002", rl.Links[0].DestRouterId) + + // disconnect and reconnect the router + ctx.Req.NoError(router1.Shutdown()) + ctx.Req.NoError(ctx.waitForPortClose("localhost:6004", 2*time.Second)) + router1 = ctx.startRouter(1) + + router1cc, msgc1 = acceptControl("router-1") + + // The router should tell us about the faulty link + msg, err = msgc2.Next(5 * time.Second) + ctx.Req.NoError(err) + ctx.Req.Equal(int32(ctrl_pb.ContentType_FaultType), msg.ContentType) + fault := &ctrl_pb.Fault{} + ctx.Req.NoError(proto.Unmarshal(msg.Body, fault)) + ctx.Req.Equal(ctrl_pb.FaultSubject_LinkFault, fault.Subject) + ctx.Req.Equal(dial1.LinkId, fault.Id) + + // Try one more time, this time it should result in a new connection + err = protobufs.MarshalTyped(dial2).WithTimeout(time.Second).SendAndWaitForWire(router1cc) + ctx.Req.NoError(err) + + msg, err = msgc1.Next(time.Second) + ctx.Req.NoError(err) + ctx.Req.Equal(int32(ctrl_pb.ContentType_LinkConnectedType), msg.ContentType) + linkConnected := &ctrl_pb.LinkConnected{} + ctx.Req.NoError(proto.Unmarshal(msg.Body, linkConnected)) + ctx.Req.Equal(dial2.LinkId, linkConnected.Id) + + ctx.Teardown() + _ = router1cc.Close() + _ = router2cc.Close() + _ = ctrlListener.Close() +} diff --git a/tests/testdata/config/router-1.yml b/tests/testdata/config/router-1.yml index 1a22647c7..420e7c289 100644 --- a/tests/testdata/config/router-1.yml +++ b/tests/testdata/config/router-1.yml @@ -46,6 +46,8 @@ dialers: - binding: transport link: + dialers: + - binding: transport listeners: - binding: transport bind: tls:127.0.0.1:6004 diff --git a/tests/testdata/config/router-2.yml b/tests/testdata/config/router-2.yml new file mode 100644 index 000000000..f2083deb6 --- /dev/null +++ b/tests/testdata/config/router-2.yml @@ -0,0 +1,54 @@ +v: 3 + +# (required) The certificates, ca chains, and private key used to represent this routers identity +# +# If the 'edge" section is enabled and enrollment is run, the files in this section will be written to with the results. +# The "key" value can be provided through any of the support private key engines (file, etc.). +# +# If the 'key' value is a file and that file exists; it is expected to a pre-generated private key. It will be used for +# the enrollment process and it will back the certificates generated during enrollment. If the file does not exist, it +# will be generated. +# +# The 'cert', 'server_cert', and 'ca' files will be overwritten with the results of the enrollment process. +# +# When running the ziti-router the files/values in this section will be read and used. The 'cert' value will be used +# as the client certificate to connect to the controller. The 'server_cert' is used for all servers/listeners. The +# 'ca' value is used to populate an internally Trusted CA pool. +identity: + cert: testdata/router/002-client.cert.pem + server_cert: testdata/router/002-server.cert.pem + key: testdata/router/002.key.pem + ca: testdata/ca/intermediate/certs/ca-chain.cert.pem + +ctrl: + endpoint: tls:127.0.0.1:6363 + +csr: + country: US + province: NC + locality: Charlotte + organization: NetFoundry + organizationalUnit: Ziti + sans: + dns: + - "localhost" + - "test-network" + - "test-network.localhost" + - "ziti-dev-ingress01" + email: + - "admin@example.com" + ip: + - "127.0.0.1" + uri: + - "ziti://ziti-dev-router01/made/up/example" + +dialers: + - binding: transport + +link: + dialers: + - binding: transport + listeners: + - binding: transport + bind: tls:127.0.0.1:6005 + advertise: tls:127.0.0.1:6005 diff --git a/tests/testutil/control.go b/tests/testutil/control.go new file mode 100644 index 000000000..df3df0e8f --- /dev/null +++ b/tests/testutil/control.go @@ -0,0 +1,25 @@ +package testutil + +import ( + "github.com/openziti/channel" + "github.com/openziti/fabric/controller/handler_common" + "github.com/openziti/fabric/pb/ctrl_pb" + "github.com/stretchr/testify/require" + "time" +) + +func AcceptControl(id string, uf channel.UnderlayFactory, assertions *require.Assertions) (channel.Channel, *MessageCollector) { + msgc := NewMessageCollector(id) + bindHandler := func(binding channel.Binding) error { + binding.AddReceiveHandler(channel.AnyContentType, msgc) + binding.AddReceiveHandlerF(int32(ctrl_pb.ContentType_VerifyLinkType), func(msg *channel.Message, ch channel.Channel) { + handler_common.SendSuccess(msg, ch, "link success") + }) + return nil + } + + timeoutUF := NewTimeoutUnderlayFactory(uf, 2*time.Second) + ch, err := channel.NewChannel(id, timeoutUF, channel.BindHandlerF(bindHandler), channel.DefaultOptions()) + assertions.NoError(err) + return ch, msgc +} diff --git a/tests/testutil/message_collector.go b/tests/testutil/message_collector.go new file mode 100644 index 000000000..f8812c8d0 --- /dev/null +++ b/tests/testutil/message_collector.go @@ -0,0 +1,59 @@ +package testutil + +import ( + "fmt" + "github.com/openziti/channel" + "github.com/openziti/fabric/pb/ctrl_pb" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" + "time" +) + +func NewMessageCollector(id string) *MessageCollector { + return &MessageCollector{ + id: id, + msgs: make(chan *channel.Message, 16), + decoders: []channel.TraceMessageDecoder{channel.Decoder{}, ctrl_pb.Decoder{}}, + } +} + +type MessageCollector struct { + id string + msgs chan *channel.Message + decoders []channel.TraceMessageDecoder +} + +func (self *MessageCollector) HandleReceive(m *channel.Message, ch channel.Channel) { + select { + case self.msgs <- m: + decoded := fmt.Sprintf("ContentType: %v", m.ContentType) + for _, decoder := range self.decoders { + if val, ok := decoder.Decode(m); ok { + decoded += ", decoded=" + string(val) + break + } + } + logrus.Infof("%v: received %v", self.id, decoded) + case <-time.After(time.Second * 5): + logrus.Error("timed out trying to queue message, closing channel") + _ = ch.Close() + } +} + +func (self *MessageCollector) Next(timeout time.Duration) (*channel.Message, error) { + select { + case msg := <-self.msgs: + return msg, nil + case <-time.After(timeout): + return nil, errors.New("timed out") + } +} + +func (self *MessageCollector) NoMessages(timeout time.Duration, req require.Assertions) { + select { + case msg := <-self.msgs: + req.Nil(msg) + case <-time.After(timeout): + } +} diff --git a/tests/testutil/timeout_underlay.go b/tests/testutil/timeout_underlay.go new file mode 100644 index 000000000..c9f81c5af --- /dev/null +++ b/tests/testutil/timeout_underlay.go @@ -0,0 +1,42 @@ +package testutil + +import ( + "github.com/openziti/channel" + "github.com/openziti/foundation/transport" + "github.com/pkg/errors" + "time" +) + +func NewTimeoutUnderlayFactory(factory channel.UnderlayFactory, timeout time.Duration) *UnderlayFactoryWrapper { + return &UnderlayFactoryWrapper{ + timeout: timeout, + wrapped: factory, + } +} + +type UnderlayFactoryWrapper struct { + timeout time.Duration + wrapped channel.UnderlayFactory +} + +func (self *UnderlayFactoryWrapper) Create(timeout time.Duration, tcfg transport.Configuration) (channel.Underlay, error) { + underlayC := make(chan channel.Underlay, 1) + errC := make(chan error, 1) + go func() { + u, err := self.wrapped.Create(timeout, tcfg) + if err != nil { + errC <- err + } else { + underlayC <- u + } + }() + + select { + case underlay := <-underlayC: + return underlay, nil + case err := <-errC: + return nil, err + case <-time.After(self.timeout): + return nil, errors.New("timed out") + } +} diff --git a/trace/controller.go b/trace/controller.go index 2b99f474a..a7b746e97 100644 --- a/trace/controller.go +++ b/trace/controller.go @@ -187,19 +187,35 @@ type controllerImpl struct { } func (controller *controllerImpl) EnableTracing(sourceType SourceType, matcher SourceMatcher, resultChan chan<- ToggleApplyResult) { - controller.events <- &enableSourcesEvent{sourceType, matcher, resultChan} + select { + case controller.events <- &enableSourcesEvent{sourceType, matcher, resultChan}: + case <-controller.closeNotify: + return + } } func (controller *controllerImpl) DisableTracing(sourceType SourceType, matcher SourceMatcher, resultChan chan<- ToggleApplyResult) { - controller.events <- &disableSourcesEvent{sourceType, matcher, resultChan} + select { + case controller.events <- &disableSourcesEvent{sourceType, matcher, resultChan}: + case <-controller.closeNotify: + return + } } func (controller *controllerImpl) AddSource(source Source) { - controller.events <- &sourceAddedEvent{source} + select { + case controller.events <- &sourceAddedEvent{source}: + case <-controller.closeNotify: + return + } } func (controller *controllerImpl) RemoveSource(source Source) { - controller.events <- &sourceRemovedEvent{source} + select { + case controller.events <- &sourceRemovedEvent{source}: + case <-controller.closeNotify: + return + } } func (controller *controllerImpl) run() {