Merge pull request #1338 from openziti/edge-terminator-fixes

Edge terminator fixes

common/pb/edge_ctrl_pb/edge_ctrl.proto

@@ -28,6 +28,8 @@ enum ContentType {
   ValidateSessionsRequestType = 20213;
   HealthEventType = 20214;
   SigningCertAddedType = 20215;
+  CreateTerminatorV2RequestType = 20216;
+  CreateTerminatorV2ResponseType = 20217;
 
   EnrollType = 20300;
   EnrollmentCertsResponseType = 20301;
@@ -66,7 +68,7 @@ enum Header {
 message ServerHello {
   string version = 1;
   map<string, string> data = 2;
-  map<string,bytes> byteData = 3;
+  map<string, bytes> byteData = 3;
 }
 
 message Address {
@@ -148,7 +150,7 @@ message PostureSet {
 message PostureCheck {
   string id = 1;
   string type = 2;
-  map<string,bytes> data = 3;
+  map<string, bytes> data = 3;
 }
 
 message Session {
@@ -224,6 +226,29 @@ message CreateTerminatorRequest {
   bytes instanceSecret = 7;
 }
 
+message CreateTerminatorV2Request {
+  string address = 1;
+  string sessionToken = 2;
+  repeated string fingerprints = 3;
+  map<uint32, bytes> peerData = 4;
+  uint32 cost = 5;
+  TerminatorPrecedence precedence = 6;
+  string instanceId = 7;
+  bytes instanceSecret = 8;
+}
+
+enum CreateTerminatorResult {
+  Success = 0;
+  FailedIdConflict = 1;
+  FailedOther = 2;
+}
+
+message CreateTerminatorV2Response {
+  string terminatorId = 1;
+  CreateTerminatorResult result = 2;
+  string msg = 3;
+}
+
 message RemoveTerminatorRequest {
   string sessionToken = 1;
   repeated string fingerprints = 2;

common/pb/edge_ctrl_pb/messages.go

@@ -30,9 +30,6 @@ func (request *CreateTerminatorRequest) GetContentType() int32 {
 	return int32(ContentType_CreateTerminatorRequestType)
 }
 
-func (request *Error) GetContentType() int32 {
-	return int32(ContentType_ErrorType)
-}
 func (request *CreateTerminatorRequest) GetXtPrecedence() xt.Precedence {
 	if request.GetPrecedence() == TerminatorPrecedence_Failed {
 		return xt.Precedences.Failed
@@ -43,6 +40,28 @@ func (request *CreateTerminatorRequest) GetXtPrecedence() xt.Precedence {
 	return xt.Precedences.Default
 }
 
+func (request *CreateTerminatorV2Request) GetContentType() int32 {
+	return int32(ContentType_CreateTerminatorV2RequestType)
+}
+
+func (request *CreateTerminatorV2Request) GetXtPrecedence() xt.Precedence {
+	if request.GetPrecedence() == TerminatorPrecedence_Failed {
+		return xt.Precedences.Failed
+	}
+	if request.GetPrecedence() == TerminatorPrecedence_Required {
+		return xt.Precedences.Required
+	}
+	return xt.Precedences.Default
+}
+
+func (request *CreateTerminatorV2Response) GetContentType() int32 {
+	return int32(ContentType_CreateTerminatorV2ResponseType)
+}
+
+func (request *Error) GetContentType() int32 {
+	return int32(ContentType_ErrorType)
+}
+
 func (request *UpdateTerminatorRequest) GetContentType() int32 {
 	return int32(ContentType_UpdateTerminatorRequestType)
 }

controller/handler_edge_ctrl/create_terminator_v2.go

@@ -0,0 +1,219 @@
+/*
+	Copyright NetFoundry Inc.
+
+	Licensed under the Apache License, Version 2.0 (the "License");
+	you may not use this file except in compliance with the License.
+	You may obtain a copy of the License at
+
+	https://www.apache.org/licenses/LICENSE-2.0
+
+	Unless required by applicable law or agreed to in writing, software
+	distributed under the License is distributed on an "AS IS" BASIS,
+	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+	See the License for the specific language governing permissions and
+	limitations under the License.
+*/
+
+package handler_edge_ctrl
+
+import (
+	"fmt"
+	"github.com/michaelquigley/pfxlog"
+	"github.com/openziti/channel/v2"
+	"github.com/openziti/channel/v2/protobufs"
+	"github.com/openziti/fabric/controller/models"
+	"github.com/openziti/fabric/controller/network"
+	"github.com/openziti/ziti/common"
+	"github.com/openziti/ziti/common/pb/edge_ctrl_pb"
+	"github.com/openziti/ziti/controller/env"
+	"github.com/openziti/ziti/controller/model"
+	"github.com/openziti/ziti/controller/persistence"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	"google.golang.org/protobuf/proto"
+	"math"
+)
+
+type createTerminatorV2Handler struct {
+	baseRequestHandler
+}
+
+func NewCreateTerminatorV2Handler(appEnv *env.AppEnv, ch channel.Channel) channel.TypedReceiveHandler {
+	return &createTerminatorV2Handler{
+		baseRequestHandler{
+			ch:     ch,
+			appEnv: appEnv,
+		},
+	}
+}
+
+func (self *createTerminatorV2Handler) ContentType() int32 {
+	return int32(edge_ctrl_pb.ContentType_CreateTerminatorV2RequestType)
+}
+
+func (self *createTerminatorV2Handler) Label() string {
+	return "create.terminator"
+}
+
+func (self *createTerminatorV2Handler) HandleReceive(msg *channel.Message, ch channel.Channel) {
+	req := &edge_ctrl_pb.CreateTerminatorV2Request{}
+	if err := proto.Unmarshal(msg.Body, req); err != nil {
+		pfxlog.ContextLogger(ch.Label()).WithError(err).Error("could not unmarshal CreateTerminatorV2Request")
+		return
+	}
+
+	ctx := &CreateTerminatorV2RequestContext{
+		baseSessionRequestContext: baseSessionRequestContext{handler: self, msg: msg},
+		req:                       req,
+	}
+
+	go self.CreateTerminatorV2(ctx)
+}
+
+func (self *createTerminatorV2Handler) CreateTerminatorV2(ctx *CreateTerminatorV2RequestContext) {
+	logger := pfxlog.ContextLogger(self.ch.Label()).
+		WithField("routerId", self.ch.Id()).
+		WithField("token", ctx.req.SessionToken).
+		WithField("terminatorId", ctx.req.Address)
+
+	if !ctx.loadRouter() {
+		return
+	}
+	ctx.loadSession(ctx.req.SessionToken)
+	ctx.checkSessionType(persistence.SessionTypeBind)
+	ctx.checkSessionFingerprints(ctx.req.Fingerprints)
+	ctx.verifyEdgeRouterAccess()
+	ctx.loadService()
+
+	if ctx.err != nil {
+		self.returnError(ctx, edge_ctrl_pb.CreateTerminatorResult_FailedOther, ctx.err, logger)
+		return
+	}
+
+	logger = logger.WithField("serviceId", ctx.service.Id).WithField("service", ctx.service.Name)
+
+	if ctx.req.Cost > math.MaxUint16 {
+		ctx.err = invalidCost(fmt.Sprintf("invalid cost %v. cost must be between 0 and %v inclusive", ctx.req.Cost, math.MaxUint16))
+		self.returnError(ctx, edge_ctrl_pb.CreateTerminatorResult_FailedOther, ctx.err, logger)
+		return
+	}
+
+	terminator, _ := self.getNetwork().Terminators.Read(ctx.req.Address)
+	if terminator != nil {
+		if ctx.err = ctx.validateExistingTerminator(terminator, logger); ctx.err != nil {
+			self.returnError(ctx, edge_ctrl_pb.CreateTerminatorResult_FailedIdConflict, ctx.err, logger)
+			return
+		}
+	} else {
+		terminator = &network.Terminator{
+			BaseEntity: models.BaseEntity{
+				Id:       ctx.req.Address,
+				IsSystem: true,
+			},
+			Service:        ctx.session.ServiceId,
+			Router:         ctx.sourceRouter.Id,
+			Binding:        common.EdgeBinding,
+			Address:        ctx.req.Address,
+			InstanceId:     ctx.req.InstanceId,
+			InstanceSecret: ctx.req.InstanceSecret,
+			PeerData:       ctx.req.PeerData,
+			Precedence:     ctx.req.GetXtPrecedence(),
+			Cost:           uint16(ctx.req.Cost),
+			HostId:         ctx.session.IdentityId,
+		}
+
+		cmd := &model.CreateEdgeTerminatorCmd{
+			Env:     self.appEnv,
+			Entity:  terminator,
+			Context: ctx.newChangeContext(),
+		}
+
+		if err := self.appEnv.GetHostController().GetNetwork().Managers.Command.Dispatch(cmd); err != nil {
+			// terminator might have been created while we were trying to create.
+			if terminator, _ = self.getNetwork().Terminators.Read(ctx.req.Address); terminator != nil {
+				if validateError := ctx.validateExistingTerminator(terminator, logger); validateError != nil {
+					self.returnError(ctx, edge_ctrl_pb.CreateTerminatorResult_FailedIdConflict, validateError, logger)
+					return
+				}
+			} else {
+				self.returnError(ctx, edge_ctrl_pb.CreateTerminatorResult_FailedOther, err, logger)
+				return
+			}
+		} else {
+			logger.WithField("terminator", terminator.Id).Info("created terminator")
+		}
+	}
+
+	response := &edge_ctrl_pb.CreateTerminatorV2Response{
+		TerminatorId: terminator.Id,
+		Result:       edge_ctrl_pb.CreateTerminatorResult_Success,
+	}
+
+	body, err := proto.Marshal(response)
+	if err != nil {
+		logger.WithError(err).Error("failed to marshal CreateTunnelTerminatorResponse")
+		return
+	}
+
+	responseMsg := channel.NewMessage(response.GetContentType(), body)
+	responseMsg.ReplyTo(ctx.msg)
+	if err = self.ch.Send(responseMsg); err != nil {
+		logger.WithError(err).Error("failed to send CreateTunnelTerminatorResponse")
+	}
+
+	logger.Info("completed create tunnel terminator operation")
+}
+
+func (self *createTerminatorV2Handler) returnError(ctx *CreateTerminatorV2RequestContext, resultType edge_ctrl_pb.CreateTerminatorResult, err error, logger *logrus.Entry) {
+	response := &edge_ctrl_pb.CreateTerminatorV2Response{
+		TerminatorId: ctx.req.Address,
+		Result:       resultType,
+		Msg:          err.Error(),
+	}
+
+	if sendErr := protobufs.MarshalTyped(response).ReplyTo(ctx.msg).Send(self.ch); sendErr != nil {
+		logger.WithError(err).WithField("sendError", sendErr).Error("failed to send error response")
+	} else {
+		logger.WithError(err).Error("responded with error")
+	}
+}
+
+type CreateTerminatorV2RequestContext struct {
+	baseSessionRequestContext
+	req *edge_ctrl_pb.CreateTerminatorV2Request
+}
+
+func (self *CreateTerminatorV2RequestContext) GetSessionToken() string {
+	return self.req.SessionToken
+}
+
+func (self *CreateTerminatorV2RequestContext) validateExistingTerminator(terminator *network.Terminator, log *logrus.Entry) controllerError {
+	if terminator.Binding != common.EdgeBinding {
+		log.WithField("binding", common.EdgeBinding).
+			WithField("conflictingBinding", terminator.Binding).
+			Error("selected terminator address conflicts with a terminator for a different binding")
+		return internalError(errors.New("selected id conflicts with terminator for different binding"))
+	}
+
+	if terminator.Service != self.session.ServiceId {
+		log.WithField("conflictingService", terminator.Service).
+			Error("selected terminator address conflicts with a terminator for a different service")
+		return internalError(errors.New("selected id conflicts with terminator for different service"))
+	}
+
+	if terminator.Router != self.sourceRouter.Id {
+		log.WithField("conflictingRouter", terminator.Router).
+			Error("selected terminator address conflicts with a terminator for a different router")
+		return internalError(errors.New("selected id conflicts with terminator for different router"))
+	}
+
+	if terminator.HostId != self.session.IdentityId {
+		log.WithField("identityId", self.session.IdentityId).
+			WithField("conflictingIdentity", terminator.HostId).
+			Error("selected terminator address conflicts with a terminator for a different identity")
+		return internalError(errors.New("selected id conflicts with terminator for different identity"))
+	}
+
+	log.Info("terminator already exists")
+	return nil
+}

controller/handler_edge_ctrl/create_tunnel_terminator.go

@@ -26,6 +26,7 @@ import (
 	"github.com/openziti/ziti/controller/persistence"
 	"github.com/openziti/fabric/controller/models"
 	"github.com/openziti/fabric/controller/network"
+	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"google.golang.org/protobuf/proto"
 	"math"
@@ -105,7 +106,10 @@ func (self *createTunnelTerminatorHandler) CreateTerminator(ctx *CreateTunnelTer
 
 	terminator, _ := self.getNetwork().Terminators.Read(ctx.req.Address)
 	if terminator != nil {
-		logger.Info("terminator already exists")
+		if err := ctx.validateExistingTerminator(terminator, logger); err != nil {
+			self.returnError(ctx, err)
+			return
+		}
 	} else {
 		terminator = &network.Terminator{
 			BaseEntity: models.BaseEntity{
@@ -125,12 +129,13 @@ func (self *createTunnelTerminatorHandler) CreateTerminator(ctx *CreateTunnelTer
 		}
 
 		n := self.appEnv.GetHostController().GetNetwork()
-		err := n.Terminators.Create(terminator, ctx.newTunnelChangeContext())
-		if err != nil {
+		if err := n.Terminators.Create(terminator, ctx.newTunnelChangeContext()); err != nil {
 			// terminator might have been created while we were trying to create.
-			terminator, _ = self.getNetwork().Terminators.Read(ctx.req.Address)
-			if terminator != nil {
-				logger.Info("terminator already exists")
+			if terminator, _ = self.getNetwork().Terminators.Read(ctx.req.Address); terminator != nil {
+				if validateError := ctx.validateExistingTerminator(terminator, logger); validateError != nil {
+					self.returnError(ctx, validateError)
+					return
+				}
 			} else {
 				self.returnError(ctx, internalError(err))
 				return
@@ -174,3 +179,34 @@ type CreateTunnelTerminatorRequestContext struct {
 	baseTunnelRequestContext
 	req *edge_ctrl_pb.CreateTunnelTerminatorRequest
 }
+
+func (self *CreateTunnelTerminatorRequestContext) validateExistingTerminator(terminator *network.Terminator, log *logrus.Entry) controllerError {
+	if terminator.Binding != common.TunnelBinding {
+		log.WithField("binding", common.TunnelBinding).
+			WithField("conflictingBinding", terminator.Binding).
+			Error("selected terminator address conflicts with a terminator for a different binding")
+		return internalError(errors.New("selected id conflicts with terminator for different binding"))
+	}
+
+	if terminator.Service != self.session.ServiceId {
+		log.WithField("conflictingService", terminator.Service).
+			Error("selected terminator address conflicts with a terminator for a different service")
+		return internalError(errors.New("selected id conflicts with terminator for different service"))
+	}
+
+	if terminator.Router != self.sourceRouter.Id {
+		log.WithField("conflictingRouter", terminator.Router).
+			Error("selected terminator address conflicts with a terminator for a different router")
+		return internalError(errors.New("selected id conflicts with terminator for different router"))
+	}
+
+	if terminator.HostId != self.session.IdentityId {
+		log.WithField("identityId", self.session.IdentityId).
+			WithField("conflictingIdentity", terminator.HostId).
+			Error("selected terminator address conflicts with a terminator for a different identity")
+		return internalError(errors.New("selected id conflicts with terminator for different identity"))
+	}
+
+	log.Info("terminator already exists")
+	return nil
+}

controller/model/create_terminator_cmd.go

@@ -2,14 +2,14 @@ package model
 
 import (
 	"github.com/michaelquigley/pfxlog"
-	"github.com/openziti/ziti/common"
-	"github.com/openziti/ziti/common/pb/edge_cmd_pb"
-	"github.com/openziti/ziti/controller/persistence"
+	"github.com/openziti/fabric/common/pb/cmd_pb"
 	"github.com/openziti/fabric/controller/change"
 	"github.com/openziti/fabric/controller/command"
 	"github.com/openziti/fabric/controller/network"
-	"github.com/openziti/fabric/common/pb/cmd_pb"
 	"github.com/openziti/storage/boltz"
+	"github.com/openziti/ziti/common"
+	"github.com/openziti/ziti/common/pb/edge_cmd_pb"
+	"github.com/openziti/ziti/controller/persistence"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"go.etcd.io/bbolt"
@@ -34,10 +34,6 @@ func (self *CreateEdgeTerminatorCmd) Apply(ctx boltz.MutateContext) error {
 
 func (self *CreateEdgeTerminatorCmd) validateTerminatorIdentity(ctx boltz.MutateContext, terminator *network.Terminator) error {
 	tx := ctx.Tx()
-	session, err := self.getTerminatorSession(tx, terminator, "")
-	if err != nil {
-		return err
-	}
 
 	if terminator.GetInstanceId() == "" {
 		return nil
@@ -49,21 +45,15 @@ func (self *CreateEdgeTerminatorCmd) validateTerminatorIdentity(ctx boltz.Mutate
 	}
 
 	for _, otherTerminator := range identityTerminators {
-		otherSession, err := self.getTerminatorSession(tx, otherTerminator, "sibling ")
-		if err != nil {
-			return err
-		}
-		if otherSession != nil {
-			if otherSession.ApiSession.IdentityId != session.ApiSession.IdentityId {
-				pfxlog.Logger().WithFields(logrus.Fields{
-					"terminatorId":       terminator.GetId(),
-					"siblingId":          otherTerminator.GetId(),
-					"instanceId":         terminator.InstanceId,
-					"terminatorIdentity": session.ApiSession.IdentityId,
-					"existingIdentity":   otherSession.ApiSession.IdentityId,
-				}).Warn("validation of terminator failed, shared identity belongs to different identity")
-				return errors.Errorf("sibling terminator %v with shared identity %v belongs to different identity", otherTerminator.GetId(), terminator.GetInstanceId())
-			}
+		if otherTerminator.HostId != terminator.HostId {
+			pfxlog.Logger().WithFields(logrus.Fields{
+				"terminatorId":       terminator.GetId(),
+				"siblingId":          otherTerminator.GetId(),
+				"instanceId":         terminator.InstanceId,
+				"terminatorIdentity": terminator.HostId,
+				"existingIdentity":   otherTerminator.HostId,
+			}).Warn("validation of terminator failed, shared identity belongs to different identity")
+			return errors.Errorf("sibling terminator %v with shared identity %v belongs to different identity", otherTerminator.GetId(), terminator.GetInstanceId())
 		}
 	}
 

controller/server/controller.go

@@ -154,6 +154,7 @@ func (c *Controller) GetCtrlHandlers(binding channel.Binding) []channel.TypedRec
 		handler_edge_ctrl.NewSessionHeartbeatHandler(c.AppEnv),
 		handler_edge_ctrl.NewCreateCircuitHandler(c.AppEnv, ch),
 		handler_edge_ctrl.NewCreateTerminatorHandler(c.AppEnv, ch),
+		handler_edge_ctrl.NewCreateTerminatorV2Handler(c.AppEnv, ch),
 		handler_edge_ctrl.NewUpdateTerminatorHandler(c.AppEnv, ch),
 		handler_edge_ctrl.NewRemoveTerminatorHandler(c.AppEnv, ch),
 		handler_edge_ctrl.NewValidateSessionsHandler(c.AppEnv, ch),

go.mod

@@ -11,6 +11,7 @@ require (
 	github.com/cenkalti/backoff/v4 v4.2.1
 	github.com/coreos/go-iptables v0.6.0
 	github.com/dgryski/dgoogauth v0.0.0-20190221195224-5a805980a5f3
+	github.com/dineshappavoo/basex v0.0.0-20170425072625-481a6f6dc663
 	github.com/fatih/color v1.15.0
 	github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa
 	github.com/go-acme/lego/v4 v4.2.0
@@ -18,7 +19,7 @@ require (
 	github.com/go-openapi/loads v0.21.2
 	github.com/go-openapi/runtime v0.26.0
 	github.com/go-openapi/strfmt v0.21.7
-	github.com/go-resty/resty/v2 v2.7.0
+	github.com/go-resty/resty/v2 v2.8.0
 	github.com/golang-jwt/jwt/v5 v5.0.0
 	github.com/google/go-cmp v0.5.9
 	github.com/google/gopacket v1.1.19
@@ -37,7 +38,7 @@ require (
 	github.com/openziti/agent v1.0.15
 	github.com/openziti/channel/v2 v2.0.99
 	github.com/openziti/edge-api v0.25.36
-	github.com/openziti/fabric v0.24.30
+	github.com/openziti/fabric v0.24.36
 	github.com/openziti/foundation/v2 v2.0.32
 	github.com/openziti/identity v1.0.63
 	github.com/openziti/jwks v1.0.3
@@ -45,7 +46,7 @@ require (
 	github.com/openziti/runzmd v1.0.31
 	github.com/openziti/sdk-golang v0.20.112
 	github.com/openziti/secretstream v0.1.12
-	github.com/openziti/storage v0.2.15
+	github.com/openziti/storage v0.2.18
 	github.com/openziti/transport/v2 v2.0.107
 	github.com/openziti/x509-claims v1.0.3
 	github.com/openziti/xweb/v2 v2.1.0
@@ -84,7 +85,7 @@ require (
 	github.com/MichaelMure/go-term-text v0.3.1 // indirect
 	github.com/alecthomas/chroma v0.10.0 // indirect
 	github.com/andybalholm/brotli v1.0.5 // indirect
-	github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20211106181442-e4c1a74c66bd // indirect
+	github.com/antlr4-go/antlr/v4 v4.13.0 // indirect
 	github.com/armon/go-metrics v0.4.1 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
 	github.com/biogo/store v0.0.0-20200525035639-8c94ae1e7c9c // indirect
@@ -92,7 +93,6 @@ require (
 	github.com/c-bata/go-prompt v0.2.6 // indirect
 	github.com/creack/pty v1.1.11 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/dineshappavoo/basex v0.0.0-20170425072625-481a6f6dc663 // indirect
 	github.com/disintegration/imaging v1.6.2 // indirect
 	github.com/dlclark/regexp2 v1.10.0 // indirect
 	github.com/docker/go-units v0.5.0 // indirect

go.sum

@@ -100,8 +100,8 @@ github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/
 github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
-github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20211106181442-e4c1a74c66bd h1:fjJY1LimH0wVCvOHLX35SCX/MbWomAglET1H2kvz7xc=
-github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20211106181442-e4c1a74c66bd/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY=
+github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI=
+github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g=
 github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
@@ -299,8 +299,8 @@ github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+
 github.com/go-playground/validator/v10 v10.2.0 h1:KgJ0snyC2R9VXYN2rneOtQcw5aHQB1Vv0sFl1UcHBOY=
 github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI=
 github.com/go-resty/resty/v2 v2.1.1-0.20191201195748-d7b97669fe48/go.mod h1:dZGr0i9PLlaaTD4H/hoZIDjQ+r6xq8mgbRzHZf7f2J8=
-github.com/go-resty/resty/v2 v2.7.0 h1:me+K9p3uhSmXtrBZ4k9jcEAfJmuC8IivWHwaLZwPrFY=
-github.com/go-resty/resty/v2 v2.7.0/go.mod h1:9PWDzw47qPphMRFfhsyk0NnSgvluHcljSMVIq3w7q0I=
+github.com/go-resty/resty/v2 v2.8.0 h1:J29d0JFWwSWrDCysnOK/YjsPMLQTx0TvgJEHVGvf2L8=
+github.com/go-resty/resty/v2 v2.8.0/go.mod h1:UCui0cMHekLrSntoMyofdSTaPpinlRHFtPpizuyDW2w=
 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0=
@@ -714,8 +714,8 @@ github.com/openziti/dilithium v0.3.3 h1:PLgQ6PMNLSTzCFbX/h98cmudgz/cU6TmjdSv5NAP
 github.com/openziti/dilithium v0.3.3/go.mod h1:vsCjI2AU/hon9e+dLhUFbCNGesJDj2ASgkySOcpmvjo=
 github.com/openziti/edge-api v0.25.36 h1:CjQTmXyYWjUX1Yc1FO0xkeATwd9A1KPu6CvNpstNMrY=
 github.com/openziti/edge-api v0.25.36/go.mod h1:OMedQFZPoBER7E9nkyECML/Pt2C2uajdbm/BBP1wOuY=
-github.com/openziti/fabric v0.24.30 h1:Ll2+11rL0A/ZNkTaZNdzs51KPA/xvd+oukyk9rjXY/Q=
-github.com/openziti/fabric v0.24.30/go.mod h1:BqlSpPukrkWA9LZSwZH32BGisKu59uJYnIzQ8bvXnX8=
+github.com/openziti/fabric v0.24.36 h1:KAlMvFjeCeTfuZDKbH55M03eKpIiv/Zx4Tt+qdRydDE=
+github.com/openziti/fabric v0.24.36/go.mod h1:ZglDYZABDfl0rWI4eXo4AYMc9G18XSlDNombiGNzs7Q=
 github.com/openziti/foundation/v2 v2.0.32 h1:I4CS7UCMjxWrS2JqzQvPfgalJM9Z9PKi3wtn0b7ULmM=
 github.com/openziti/foundation/v2 v2.0.32/go.mod h1:KCQatGmkVDIZnfSLrohKwmQi7kg7vnECpNTkpuLNguc=
 github.com/openziti/identity v1.0.63 h1:yJwVAcPpc+e7zhi2fb9lgAnqtR7cO5cdxuk2Y9Z6i0I=
@@ -730,8 +730,8 @@ github.com/openziti/sdk-golang v0.20.112 h1:oc6fNVg9poyRMdPQK2L84RGvOk3fKs3X+dzY
 github.com/openziti/sdk-golang v0.20.112/go.mod h1:/ioQ2qRfkwpSse3NCu37eXyJvQLgcK2RdenH4ajmhPw=
 github.com/openziti/secretstream v0.1.12 h1:N78CHxtqWzSyNFOsYtYRWNNTfX1ZDAPkFgzHobpodZU=
 github.com/openziti/secretstream v0.1.12/go.mod h1:gHMH1REH0r4VlmCtuWx8biU7j5ZfOivFjz9mLgwq7mk=
-github.com/openziti/storage v0.2.15 h1:mxEM+NuxZN3qPgXw4dc40FRLeMD4p+W1fspnt1zTGDM=
-github.com/openziti/storage v0.2.15/go.mod h1:BZcM3mHa13re5IlA8UjBCfjPCnN6H0lScg9U2lR1/cY=
+github.com/openziti/storage v0.2.18 h1:a5G9bYJ+nMNU00WvDlWyhMlrlhPLsnxPkyStHKE6KMY=
+github.com/openziti/storage v0.2.18/go.mod h1:IC8TdgMTTQJRZIwRjT2WeDs+vBF8R/cyiqA70NTzY/g=
 github.com/openziti/transport/v2 v2.0.107 h1:Azw/joD+6c1wgsaipXXYgzIn0eb212ymaF6EDckti3k=
 github.com/openziti/transport/v2 v2.0.107/go.mod h1:8WTSYMyGfLurWJYg58PBIp+abKLT6rTkFf8OVqK4SqI=
 github.com/openziti/x509-claims v1.0.3 h1:HNdQ8Nf1agB3lBs1gahcO6zfkeS4S5xoQ2/PkY4HRX0=
@@ -1141,11 +1141,11 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
 golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM=
-golang.org/x/net v0.0.0-20211029224645-99673261e6eb/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
 golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8=
 golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -1276,6 +1276,7 @@ golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9sn
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
 golang.org/x/term v0.12.0 h1:/ZfYdc3zq+q02Rv9vGqTeSItdzZTSNDmfTi0mBAuidU=
 golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -1290,6 +1291,7 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
 golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
 golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=

router/xgress_edge/dialer.go

@@ -23,8 +23,8 @@ import (
 
 	"github.com/michaelquigley/pfxlog"
 	"github.com/openziti/channel/v2"
-	"github.com/openziti/fabric/controller/xt"
 	"github.com/openziti/fabric/common/logcontext"
+	"github.com/openziti/fabric/controller/xt"
 	"github.com/openziti/fabric/router/xgress"
 	"github.com/openziti/sdk-golang/ziti/edge"
 	"github.com/pkg/errors"
@@ -36,19 +36,9 @@ type dialer struct {
 }
 
 func (dialer *dialer) IsTerminatorValid(id string, destination string) bool {
-	destParts := strings.Split(destination, ":")
-	if len(destParts) != 2 {
-		return false
-	}
-
-	if destParts[0] != "hosted" {
-		return false
-	}
-
-	token := destParts[1]
-
+	terminatorAddress := strings.TrimPrefix(destination, "hosted:")
 	pfxlog.Logger().Debug("looking up hosted service conn")
-	terminator, found := dialer.factory.hostedServices.Get(token)
+	terminator, found := dialer.factory.hostedServices.Get(terminatorAddress)
 	return found && terminator.terminatorId.Load() == id
 }
 
@@ -61,29 +51,20 @@ func newDialer(factory *Factory, options *Options) xgress.Dialer {
 }
 
 func (dialer *dialer) Dial(params xgress.DialParams) (xt.PeerData, error) {
-	destination := params.GetDestination()
+	terminatorAddress := params.GetDestination()
 	circuitId := params.GetCircuitId()
 	log := pfxlog.ChannelLogger(logcontext.EstablishPath).Wire(params.GetLogContext()).
 		WithField("binding", "edge").
-		WithField("destination", destination)
+		WithField("terminatorAddress", terminatorAddress)
 
-	destParts := strings.Split(destination, ":")
-	if len(destParts) != 2 {
-		return nil, fmt.Errorf("destination '%v' format is incorrect", destination)
-	}
+	terminatorAddress = strings.TrimPrefix(terminatorAddress, "hosted:")
 
-	if destParts[0] != "hosted" {
-		return nil, fmt.Errorf("unsupported destination type: '%v'", destParts[0])
-	}
-
-	token := destParts[1]
-
-	log.Debugf("looking up hosted service conn for token %v", token)
-	listenConn, found := dialer.factory.hostedServices.Get(token)
+	log.Debugf("looking up hosted service conn for address %v", terminatorAddress)
+	terminator, found := dialer.factory.hostedServices.Get(terminatorAddress)
 	if !found {
-		return nil, xgress.InvalidTerminatorError{InnerError: fmt.Errorf("host for token '%v' not found", token)}
+		return nil, xgress.InvalidTerminatorError{InnerError: fmt.Errorf("host for terminator address '%v' not found", terminatorAddress)}
 	}
-	log = log.WithField("bindConnId", listenConn.Id())
+	log = log.WithField("bindConnId", terminator.MsgChannel.Id())
 
 	callerId := ""
 	if circuitId.Data != nil {
@@ -93,7 +74,7 @@ func (dialer *dialer) Dial(params xgress.DialParams) (xt.PeerData, error) {
 	}
 
 	log.Debug("dialing sdk client hosting service")
-	dialRequest := edge.NewDialMsg(listenConn.Id(), token, callerId)
+	dialRequest := edge.NewDialMsg(terminator.Id(), terminator.token, callerId)
 	if pk, ok := circuitId.Data[edge.PublicKeyHeader]; ok {
 		dialRequest.Headers[edge.PublicKeyHeader] = pk
 	}
@@ -103,15 +84,15 @@ func (dialer *dialer) Dial(params xgress.DialParams) (xt.PeerData, error) {
 		dialRequest.Headers[edge.AppDataHeader] = appData
 	}
 
-	if listenConn.assignIds {
-		connId := listenConn.nextDialConnId()
+	if terminator.assignIds {
+		connId := terminator.nextDialConnId()
 		log = log.WithField("connId", connId)
 		log.Debugf("router assigned connId %v for dial", connId)
 		dialRequest.PutUint32Header(edge.RouterProvidedConnId, connId)
 
-		conn, err := listenConn.newConnection(connId)
+		conn, err := terminator.newConnection(connId)
 		if err != nil {
-			return nil, errors.Wrapf(err, "failed to create edge xgress conn for token %v", token)
+			return nil, errors.Wrapf(err, "failed to create edge xgress conn for terminator address %v", terminatorAddress)
 		}
 
 		// On the terminator, which this is, this only starts the txer, which pulls data from the link
@@ -128,7 +109,7 @@ func (dialer *dialer) Dial(params xgress.DialParams) (xt.PeerData, error) {
 		if timeToDeadline > 0 && timeToDeadline < to {
 			to = timeToDeadline
 		}
-		reply, err := dialRequest.WithPriority(channel.Highest).WithTimeout(to).SendForReply(listenConn.Channel)
+		reply, err := dialRequest.WithPriority(channel.Highest).WithTimeout(to).SendForReply(terminator.Channel)
 		if err != nil {
 			conn.close(false, err.Error())
 			x.Close()
@@ -143,7 +124,7 @@ func (dialer *dialer) Dial(params xgress.DialParams) (xt.PeerData, error) {
 		}
 
 		if !result.Success {
-			msg := fmt.Sprintf("failed to establish connection with token %v. error: (%v)", token, result.Message)
+			msg := fmt.Sprintf("failed to establish connection with terminator address %v. error: (%v)", terminatorAddress, result.Message)
 			conn.close(false, msg)
 			x.Close()
 			return nil, fmt.Errorf(msg)
@@ -153,7 +134,7 @@ func (dialer *dialer) Dial(params xgress.DialParams) (xt.PeerData, error) {
 		return nil, nil
 	} else {
 		log.Debug("router not assigning connId for dial")
-		reply, err := dialRequest.WithPriority(channel.Highest).WithTimeout(5 * time.Second).SendForReply(listenConn.Channel)
+		reply, err := dialRequest.WithPriority(channel.Highest).WithTimeout(5 * time.Second).SendForReply(terminator.Channel)
 		if err != nil {
 			return nil, err
 		}
@@ -164,19 +145,19 @@ func (dialer *dialer) Dial(params xgress.DialParams) (xt.PeerData, error) {
 		}
 
 		if !result.Success {
-			return nil, fmt.Errorf("failed to establish connection with token %v. error: (%v)", token, result.Message)
+			return nil, fmt.Errorf("failed to establish connection with terminator address %v. error: (%v)", terminatorAddress, result.Message)
 		}
 
-		conn, err := listenConn.newConnection(result.NewConnId)
+		conn, err := terminator.newConnection(result.NewConnId)
 		if err != nil {
 			startFail := edge.NewStateConnectedMsg(result.ConnId)
 			startFail.ReplyTo(reply)
 
-			if sendErr := listenConn.SendState(startFail); sendErr != nil {
+			if sendErr := terminator.SendState(startFail); sendErr != nil {
 				log.Debug("failed to send state disconnected")
 			}
 
-			return nil, errors.Wrapf(err, "failed to create edge xgress conn for token %v", token)
+			return nil, errors.Wrapf(err, "failed to create edge xgress conn for terminator address %v", terminatorAddress)
 		}
 
 		x := xgress.NewXgress(circuitId.Token, params.GetCtrlId(), params.GetAddress(), conn, xgress.Terminator, &dialer.options.Options, params.GetCircuitTags())
@@ -186,6 +167,6 @@ func (dialer *dialer) Dial(params xgress.DialParams) (xt.PeerData, error) {
 
 		start := edge.NewStateConnectedMsg(result.ConnId)
 		start.ReplyTo(reply)
-		return nil, listenConn.SendState(start)
+		return nil, terminator.SendState(start)
 	}
 }

router/xgress_edge/fabric.go

@@ -42,9 +42,11 @@ type edgeTerminator struct {
 	edge.MsgChannel
 	edgeClientConn *edgeClientConn
 	token          string
+	instance       string
 	terminatorId   concurrenz.AtomicValue[string]
 	assignIds      bool
 	onClose        func()
+	v2             bool
 }
 
 func (self *edgeTerminator) nextDialConnId() uint32 {
@@ -70,22 +72,43 @@ func (self *edgeTerminator) close(notify bool, reason string) {
 		}
 	}
 
-	if self.terminatorId.Load() != "" {
-		logger.Info("removing terminator on controller")
-		ctrlCh := self.edgeClientConn.listener.factory.ctrls.AnyCtrlChannel()
-		if ctrlCh == nil {
-			logger.Error("no controller available, unable to remove terminator")
-		} else if err := self.edgeClientConn.removeTerminator(ctrlCh, self); err != nil {
-			logger.WithError(err).Error("failed to remove terminator")
-		} else {
-			logger.Info("Successfully removed terminator on channel close")
+	if self.v2 {
+		if terminatorId := self.terminatorId.Load(); terminatorId != "" {
+			if self.terminatorId.CompareAndSwap(terminatorId, "") {
+				logger.Info("removing terminator on controller")
+				ctrlCh := self.edgeClientConn.listener.factory.ctrls.AnyCtrlChannel()
+				if ctrlCh == nil {
+					logger.Error("no controller available, unable to remove terminator")
+				} else if err := self.edgeClientConn.removeTerminator(ctrlCh, self.token, terminatorId); err != nil {
+					logger.WithError(err).Error("failed to remove terminator")
+				} else {
+					logger.Info("Successfully removed terminator on channel close")
+				}
+			} else {
+				logger.Warn("edge terminator closing, but no terminator id set, so can't remove on controller")
+			}
+
+			logger.Debug("removing terminator on router")
+			self.edgeClientConn.listener.factory.hostedServices.Delete(self.token)
 		}
 	} else {
-		logger.Warn("edge terminator closing, but no terminator id set, so can't remove on controller")
-	}
+		if terminatorId := self.terminatorId.Load(); terminatorId != "" {
+			logger.Info("removing terminator on controller")
+			ctrlCh := self.edgeClientConn.listener.factory.ctrls.AnyCtrlChannel()
+			if ctrlCh == nil {
+				logger.Error("no controller available, unable to remove terminator")
+			} else if err := self.edgeClientConn.removeTerminator(ctrlCh, self.token, terminatorId); err != nil {
+				logger.WithError(err).Error("failed to remove terminator")
+			} else {
+				logger.Info("Successfully removed terminator on channel close")
+			}
+		} else {
+			logger.Warn("edge terminator closing, but no terminator id set, so can't remove on controller")
+		}
 
-	logger.Debug("removing terminator on router")
-	self.edgeClientConn.listener.factory.hostedServices.Delete(self.token)
+		logger.Debug("removing terminator on router")
+		self.edgeClientConn.listener.factory.hostedServices.Delete(self.token)
+	}
 
 	if self.onClose != nil {
 		self.onClose()

router/xgress_edge/factory.go

@@ -129,7 +129,7 @@ func (factory *Factory) LoadConfig(configMap map[interface{}]interface{}) error
 // NewFactory constructs a new Edge Xgress Factory instance
 func NewFactory(routerConfig *router.Config, versionProvider versions.VersionProvider, stateManager fabric.StateManager, metricsRegistry metrics.Registry) *Factory {
 	factory := &Factory{
-		hostedServices:  &hostedServiceRegistry{},
+		hostedServices:  NewHostedServicesRegistry(),
 		stateManager:    stateManager,
 		versionProvider: versionProvider,
 		routerConfig:    routerConfig,

router/xgress_edge/hosted.go

@@ -17,11 +17,21 @@
 package xgress_edge
 
 import (
+	"github.com/michaelquigley/pfxlog"
+	cmap "github.com/orcaman/concurrent-map/v2"
 	"sync"
 )
 
+func NewHostedServicesRegistry() *hostedServiceRegistry {
+	return &hostedServiceRegistry{
+		services: sync.Map{},
+		ids:      cmap.New[string](),
+	}
+}
+
 type hostedServiceRegistry struct {
 	services sync.Map
+	ids      cmap.ConcurrentMap[string, string]
 }
 
 func (registry *hostedServiceRegistry) Put(hostId string, conn *edgeTerminator) {
@@ -51,3 +61,33 @@ func (registry *hostedServiceRegistry) cleanupServices(proxy *edgeClientConn) {
 		return true
 	})
 }
+
+func (registry *hostedServiceRegistry) unbindSession(sessionToken string, proxy *edgeClientConn) bool {
+	atLeastOneRemoved := false
+	registry.services.Range(func(key, value interface{}) bool {
+		terminator := value.(*edgeTerminator)
+		if terminator.token == sessionToken && terminator.edgeClientConn == proxy {
+			terminator.close(true, "unbind successful") // don't notify, sdk asked us to unbind
+			registry.services.Delete(key)
+			pfxlog.Logger().WithField("routerId", terminator.edgeClientConn.listener.id.Token).
+				WithField("sessionToken", sessionToken).
+				WithField("terminatorId", terminator.terminatorId.Load()).
+				Info("terminator removed")
+			atLeastOneRemoved = true
+		}
+		return true
+	})
+	return atLeastOneRemoved
+}
+
+func (registry *hostedServiceRegistry) getRelatedTerminators(sessionToken string, proxy *edgeClientConn) []*edgeTerminator {
+	var result []*edgeTerminator
+	registry.services.Range(func(key, value interface{}) bool {
+		terminator := value.(*edgeTerminator)
+		if terminator.token == sessionToken && terminator.edgeClientConn == proxy {
+			result = append(result, terminator)
+		}
+		return true
+	})
+	return result
+}

router/xgress_edge/listener.go

@@ -19,21 +19,27 @@ package xgress_edge
 import (
 	"encoding/binary"
 	"fmt"
+	"github.com/dineshappavoo/basex"
+	"github.com/openziti/fabric/common/capabilities"
+	fabricMetrics "github.com/openziti/fabric/common/metrics"
+	"github.com/openziti/fabric/common/pb/ctrl_pb"
+	"github.com/openziti/fabric/controller/idgen"
 	"github.com/openziti/ziti/common/cert"
 	"github.com/openziti/ziti/common/pb/edge_ctrl_pb"
-	fabricMetrics "github.com/openziti/fabric/common/metrics"
 	"github.com/pkg/errors"
+	"golang.org/x/crypto/sha3"
 	"google.golang.org/protobuf/proto"
+	"math/big"
 	"time"
 
 	"github.com/michaelquigley/pfxlog"
 	"github.com/openziti/channel/v2"
 	"github.com/openziti/channel/v2/protobufs"
-	"github.com/openziti/ziti/router/xgress_common"
 	"github.com/openziti/fabric/router/xgress"
 	"github.com/openziti/identity"
 	"github.com/openziti/sdk-golang/ziti/edge"
 	"github.com/openziti/transport/v2"
+	"github.com/openziti/ziti/router/xgress_common"
 )
 
 type listener struct {
@@ -183,10 +189,38 @@ func (self *edgeClientConn) processConnect(req *channel.Message, ch channel.Chan
 }
 
 func (self *edgeClientConn) processBind(req *channel.Message, ch channel.Channel) {
+	ctrlCh := self.listener.factory.ctrls.AnyCtrlChannel()
+	if ctrlCh == nil {
+		errStr := "no controller available, cannot create terminator"
+		pfxlog.ContextLogger(ch.Label()).
+			WithField("sessionToken", string(req.Body)).
+			WithFields(edge.GetLoggerFields(req)).
+			WithField("routerId", self.listener.id.Token).
+			Error(errStr)
+		self.sendStateClosedReply(errStr, req)
+		return
+	}
+
+	supportsCreateTerminatorV2 := false
+	headers := ctrlCh.Underlay().Headers()
+	if val, found := headers[int32(ctrl_pb.ContentType_CapabilitiesHeader)]; found {
+		capabilitiesMask := &big.Int{}
+		capabilitiesMask.SetBytes(val)
+		supportsCreateTerminatorV2 = capabilitiesMask.Bit(capabilities.ControllerCreateTerminatorV2) == 1
+	}
+
+	if supportsCreateTerminatorV2 {
+		self.processBindV2(req, ch, ctrlCh)
+	} else {
+		self.processBindV1(req, ch, ctrlCh)
+	}
+}
+
+func (self *edgeClientConn) processBindV1(req *channel.Message, ch channel.Channel, ctrlCh channel.Channel) {
 	token := string(req.Body)
 
 	log := pfxlog.ContextLogger(ch.Label()).
-		WithField("sessionId", token).
+		WithField("sessionToken", token).
 		WithFields(edge.GetLoggerFields(req)).
 		WithField("routerId", self.listener.id.Token)
 
@@ -196,14 +230,6 @@ func (self *edgeClientConn) processBind(req *channel.Message, ch channel.Channel
 		return
 	}
 
-	ctrlCh := self.listener.factory.ctrls.AnyCtrlChannel()
-	if ctrlCh == nil {
-		errStr := "no controller available, cannot create terminator"
-		log.Error(errStr)
-		self.sendStateClosedReply(errStr, req)
-		return
-	}
-
 	log.Debug("binding service")
 
 	hostData := make(map[uint32][]byte)
@@ -288,45 +314,171 @@ func (self *edgeClientConn) processBind(req *channel.Message, ch channel.Channel
 	log.Info("created terminator")
 }
 
-func (self *edgeClientConn) processUnbind(req *channel.Message, ch channel.Channel) {
-	token := string(req.Body)
-	log := pfxlog.ContextLogger(ch.Label()).
-		WithField("sessionId", token).
-		WithFields(edge.GetLoggerFields(req))
+func (self *edgeClientConn) getTerminatorId(sessionToken, terminatorInstance string) string {
+	hasher := sha3.NewShake256()
+	// Shake.Write is documented to only fail if written to after being read
+	_, _ = hasher.Write([]byte(self.listener.id.Token))
+	_, _ = hasher.Write([]byte(sessionToken))
+	_, _ = hasher.Write([]byte(terminatorInstance))
 
-	ctrlCh := self.listener.factory.ctrls.AnyCtrlChannel()
-	if ctrlCh == nil {
-		errStr := "no controller available, cannot remove terminator"
-		log.Error(errStr)
-		self.sendStateClosedReply(errStr, req)
+	buf := make([]byte, 16)
+	// Shake.Read is documented not to fail
+	_, _ = hasher.Read(buf)
+
+	v := &big.Int{}
+	v.SetBytes(buf)
+	result, _ := basex.EncodeInt(v)
+	return result
+}
+
+func (self *edgeClientConn) processBindV2(req *channel.Message, ch channel.Channel, ctrlCh channel.Channel) {
+	token := string(req.Body)
+
+	log := pfxlog.ContextLogger(ch.Label()).
+		WithField("sessionToken", token).
+		WithFields(edge.GetLoggerFields(req)).
+		WithField("routerId", self.listener.id.Token)
+
+	connId, found := req.GetUint32Header(edge.ConnIdHeader)
+	if !found {
+		pfxlog.Logger().Errorf("connId not set. unable to process bind message")
 		return
 	}
 
-	terminator, ok := self.listener.factory.hostedServices.Get(token)
-	if ok {
-		log = log.WithField("routerId", self.listener.id.Token).
-			WithField("terminatorId", terminator.terminatorId.Load())
+	terminatorInstance, _ := req.GetStringHeader(edge.TerminatorIdentityHeader)
 
-		defer self.listener.factory.hostedServices.Delete(token)
+	assignIds, _ := req.GetBoolHeader(edge.RouterProvidedConnId)
+	log.Debugf("client requested router provided connection ids: %v", assignIds)
 
-		log.Debug("removing terminator")
-		if err := self.removeTerminator(ctrlCh, terminator); err != nil {
-			log.WithError(err).Error("error while removing terminator")
-			self.sendStateClosedReply(err.Error(), req)
+	terminatorId := self.getTerminatorId(token, terminatorInstance)
+
+	messageSink := &edgeTerminator{
+		MsgChannel:     *edge.NewEdgeMsgChannel(self.ch, connId),
+		edgeClientConn: self,
+		token:          token,
+		instance:       terminatorInstance,
+		assignIds:      assignIds,
+		v2:             true,
+	}
+
+	log = log.WithField("bindConnId", messageSink.MsgChannel.Id())
+
+	createTerminatorOnCtrl := true
+
+	terminator, found := self.listener.factory.hostedServices.Get(terminatorId)
+	// we unfortunately can't re-use existing edgeTerminators, as a new bind request will have a new connId.
+	if found {
+		// if we somehow got a conflict, just use a new terminator id
+		if terminator.token != token || terminator.instance != terminatorInstance {
+			terminatorId = idgen.NewUUIDString()
+		} else if terminator.terminatorId.CompareAndSwap(terminatorId, "") {
+			// there was an existing terminator, and we got to it before it was torn down. We can just replace it
+			// and carry on
+			self.listener.factory.hostedServices.Put(terminatorId, messageSink)
+			createTerminatorOnCtrl = false
+			log.WithField("terminatorId", terminatorId).
+				WithField("oldBindConnId", terminator.MsgChannel.Id()).
+				Info("duplicate create terminator request detected, taking over terminator")
 		} else {
-			self.sendStateClosedReply("unbind successful", req)
-			log.Info("removed terminator")
+			terminatorId = idgen.NewUUIDString()
 		}
-	} else {
+	}
+
+	messageSink.terminatorId.Store(terminatorId)
+
+	log = log.WithField("terminatorId", terminatorId)
+	log.Debug("binding service")
+
+	hostData := make(map[uint32][]byte)
+	pubKey, hasKey := req.Headers[edge.PublicKeyHeader]
+	if hasKey {
+		hostData[edge.PublicKeyHeader] = pubKey
+	}
+
+	cost := uint16(0)
+	if costBytes, hasCost := req.Headers[edge.CostHeader]; hasCost {
+		cost = binary.LittleEndian.Uint16(costBytes)
+	}
+
+	precedence := edge_ctrl_pb.TerminatorPrecedence_Default
+	if precedenceData, hasPrecedence := req.Headers[edge.PrecedenceHeader]; hasPrecedence && len(precedenceData) > 0 {
+		edgePrecedence := edge.Precedence(precedenceData[0])
+		if edgePrecedence == edge.PrecedenceRequired {
+			precedence = edge_ctrl_pb.TerminatorPrecedence_Required
+		} else if edgePrecedence == edge.PrecedenceFailed {
+			precedence = edge_ctrl_pb.TerminatorPrecedence_Failed
+		}
+	}
+
+	log.Debug("establishing listener")
+
+	// need to remove session remove listener on close
+	messageSink.onClose = self.listener.factory.stateManager.AddEdgeSessionRemovedListener(token, func(token string) {
+		messageSink.close(true, "session ended")
+	})
+
+	self.listener.factory.hostedServices.Put(terminatorId, messageSink)
+
+	var terminatorInstanceSecret []byte
+	if terminatorInstance != "" {
+		terminatorInstanceSecret = req.Headers[edge.TerminatorIdentitySecretHeader]
+	}
+
+	request := &edge_ctrl_pb.CreateTerminatorV2Request{
+		Address:        terminatorId,
+		SessionToken:   token,
+		Fingerprints:   self.fingerprints.Prints(),
+		PeerData:       hostData,
+		Cost:           uint32(cost),
+		Precedence:     precedence,
+		InstanceId:     terminatorInstance,
+		InstanceSecret: terminatorInstanceSecret,
+	}
+
+	if createTerminatorOnCtrl {
+		timeout := self.listener.factory.ctrls.DefaultRequestTimeout()
+		responseMsg, err := protobufs.MarshalTyped(request).WithTimeout(timeout).SendForReply(ctrlCh)
+		resp := &edge_ctrl_pb.CreateTerminatorV2Response{}
+		err = protobufs.TypedResponse(resp).Unmarshall(responseMsg, err)
+		if err == nil && resp.Result != edge_ctrl_pb.CreateTerminatorResult_Success {
+			err = errors.Errorf("terminator create failed: %s", resp.Msg)
+		}
+
+		if err != nil {
+			log.WithError(err).Warn("error creating terminator")
+			messageSink.close(false, "") // don't notify here, as we're notifying next line with a response
+			self.sendStateClosedReply(err.Error(), req)
+			return
+		}
+	}
+
+	if messageSink.MsgChannel.IsClosed() {
+		log.Warn("edge channel closed while setting up terminator. cleaning up terminator now")
+		messageSink.close(false, "edge channel closed")
+		return
+	}
+
+	log.Debug("registered listener for terminator")
+	log.Debug("returning connection state CONNECTED to client")
+	self.sendStateConnectedReply(req, nil)
+
+	log.Info("created terminator")
+}
+
+func (self *edgeClientConn) processUnbind(req *channel.Message, _ channel.Channel) {
+	token := string(req.Body)
+	atLeastOneTerminatorRemoved := self.listener.factory.hostedServices.unbindSession(token, self)
+
+	if !atLeastOneTerminatorRemoved {
 		self.sendStateClosedReply(fmt.Sprintf("no terminator found for token '%s'", token), req)
 	}
 }
 
-func (self *edgeClientConn) removeTerminator(ctrlCh channel.Channel, terminator *edgeTerminator) error {
+func (self *edgeClientConn) removeTerminator(ctrlCh channel.Channel, token, terminatorId string) error {
 	request := &edge_ctrl_pb.RemoveTerminatorRequest{
-		SessionToken: terminator.token,
+		SessionToken: token,
 		Fingerprints: self.fingerprints.Prints(),
-		TerminatorId: terminator.terminatorId.Load(),
+		TerminatorId: terminatorId,
 	}
 
 	timeout := self.listener.factory.ctrls.DefaultRequestTimeout()
@@ -336,11 +488,11 @@ func (self *edgeClientConn) removeTerminator(ctrlCh channel.Channel, terminator
 
 func (self *edgeClientConn) processUpdateBind(req *channel.Message, ch channel.Channel) {
 	token := string(req.Body)
-	log := pfxlog.ContextLogger(ch.Label()).WithField("sessionId", token).WithFields(edge.GetLoggerFields(req))
 
-	terminator, ok := self.listener.factory.hostedServices.Get(token)
+	log := pfxlog.ContextLogger(ch.Label()).WithField("sessionToken", token).WithFields(edge.GetLoggerFields(req))
+	terminators := self.listener.factory.hostedServices.getRelatedTerminators(token, self)
 
-	if !ok {
+	if len(terminators) == 0 {
 		log.Error("failed to update bind, no listener found")
 		return
 	}
@@ -351,42 +503,44 @@ func (self *edgeClientConn) processUpdateBind(req *channel.Message, ch channel.C
 		return
 	}
 
-	request := &edge_ctrl_pb.UpdateTerminatorRequest{
-		SessionToken: token,
-		Fingerprints: self.fingerprints.Prints(),
-		TerminatorId: terminator.terminatorId.Load(),
-	}
-
-	if costVal, hasCost := req.GetUint16Header(edge.CostHeader); hasCost {
-		request.UpdateCost = true
-		request.Cost = uint32(costVal)
-	}
-
-	if precedenceData, hasPrecedence := req.Headers[edge.PrecedenceHeader]; hasPrecedence && len(precedenceData) > 0 {
-		edgePrecedence := edge.Precedence(precedenceData[0])
-		request.Precedence = edge_ctrl_pb.TerminatorPrecedence_Default
-		request.UpdatePrecedence = true
-		if edgePrecedence == edge.PrecedenceRequired {
-			request.Precedence = edge_ctrl_pb.TerminatorPrecedence_Required
-		} else if edgePrecedence == edge.PrecedenceFailed {
-			request.Precedence = edge_ctrl_pb.TerminatorPrecedence_Failed
+	for _, terminator := range terminators {
+		request := &edge_ctrl_pb.UpdateTerminatorRequest{
+			SessionToken: token,
+			Fingerprints: self.fingerprints.Prints(),
+			TerminatorId: terminator.terminatorId.Load(),
 		}
-	}
 
-	log = log.WithField("terminator", terminator.terminatorId.Load()).
-		WithField("precedence", request.Precedence).
-		WithField("cost", request.Cost).
-		WithField("updatingPrecedence", request.UpdatePrecedence).
-		WithField("updatingCost", request.UpdateCost)
+		if costVal, hasCost := req.GetUint16Header(edge.CostHeader); hasCost {
+			request.UpdateCost = true
+			request.Cost = uint32(costVal)
+		}
 
-	log.Debug("updating terminator")
+		if precedenceData, hasPrecedence := req.Headers[edge.PrecedenceHeader]; hasPrecedence && len(precedenceData) > 0 {
+			edgePrecedence := edge.Precedence(precedenceData[0])
+			request.Precedence = edge_ctrl_pb.TerminatorPrecedence_Default
+			request.UpdatePrecedence = true
+			if edgePrecedence == edge.PrecedenceRequired {
+				request.Precedence = edge_ctrl_pb.TerminatorPrecedence_Required
+			} else if edgePrecedence == edge.PrecedenceFailed {
+				request.Precedence = edge_ctrl_pb.TerminatorPrecedence_Failed
+			}
+		}
 
-	timeout := self.listener.factory.ctrls.DefaultRequestTimeout()
-	responseMsg, err := protobufs.MarshalTyped(request).WithTimeout(timeout).SendForReply(ctrlCh)
-	if err := xgress_common.CheckForFailureResult(responseMsg, err, edge_ctrl_pb.ContentType_UpdateTerminatorResponseType); err != nil {
-		log.WithError(err).Error("terminator update failed")
-	} else {
-		log.Debug("terminator updated successfully")
+		log = log.WithField("terminator", terminator.terminatorId.Load()).
+			WithField("precedence", request.Precedence).
+			WithField("cost", request.Cost).
+			WithField("updatingPrecedence", request.UpdatePrecedence).
+			WithField("updatingCost", request.UpdateCost)
+
+		log.Debug("updating terminator")
+
+		timeout := self.listener.factory.ctrls.DefaultRequestTimeout()
+		responseMsg, err := protobufs.MarshalTyped(request).WithTimeout(timeout).SendForReply(ctrlCh)
+		if err := xgress_common.CheckForFailureResult(responseMsg, err, edge_ctrl_pb.ContentType_UpdateTerminatorResponseType); err != nil {
+			log.WithError(err).Error("terminator update failed")
+		} else {
+			log.Debug("terminator updated successfully")
+		}
 	}
 }
 

zititest/go.mod

@@ -13,11 +13,11 @@ require (
 	github.com/openziti/agent v1.0.15
 	github.com/openziti/channel/v2 v2.0.99
 	github.com/openziti/fablab v0.5.16
-	github.com/openziti/fabric v0.24.30
+	github.com/openziti/fabric v0.24.36
 	github.com/openziti/foundation/v2 v2.0.32
 	github.com/openziti/identity v1.0.63
 	github.com/openziti/sdk-golang v0.20.112
-	github.com/openziti/storage v0.2.15
+	github.com/openziti/storage v0.2.18
 	github.com/openziti/transport/v2 v2.0.107
 	github.com/openziti/ziti v0.28.3
 	github.com/pkg/errors v0.9.1
@@ -38,7 +38,7 @@ require (
 	github.com/MichaelMure/go-term-text v0.3.1 // indirect
 	github.com/alecthomas/chroma v0.10.0 // indirect
 	github.com/andybalholm/brotli v1.0.5 // indirect
-	github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20211106181442-e4c1a74c66bd // indirect
+	github.com/antlr4-go/antlr/v4 v4.13.0 // indirect
 	github.com/armon/go-metrics v0.4.1 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
 	github.com/aws/aws-sdk-go v1.45.13 // indirect
@@ -75,7 +75,7 @@ require (
 	github.com/go-openapi/strfmt v0.21.7 // indirect
 	github.com/go-openapi/swag v0.22.4 // indirect
 	github.com/go-openapi/validate v0.22.1 // indirect
-	github.com/go-resty/resty/v2 v2.7.0 // indirect
+	github.com/go-resty/resty/v2 v2.8.0 // indirect
 	github.com/golang-jwt/jwt/v5 v5.0.0 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/gomarkdown/markdown v0.0.0-20230916125811-7478c230c7cd // indirect

zititest/go.sum

@@ -100,8 +100,8 @@ github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/
 github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
-github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20211106181442-e4c1a74c66bd h1:fjJY1LimH0wVCvOHLX35SCX/MbWomAglET1H2kvz7xc=
-github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20211106181442-e4c1a74c66bd/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY=
+github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI=
+github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g=
 github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
@@ -299,8 +299,8 @@ github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+
 github.com/go-playground/validator/v10 v10.2.0 h1:KgJ0snyC2R9VXYN2rneOtQcw5aHQB1Vv0sFl1UcHBOY=
 github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI=
 github.com/go-resty/resty/v2 v2.1.1-0.20191201195748-d7b97669fe48/go.mod h1:dZGr0i9PLlaaTD4H/hoZIDjQ+r6xq8mgbRzHZf7f2J8=
-github.com/go-resty/resty/v2 v2.7.0 h1:me+K9p3uhSmXtrBZ4k9jcEAfJmuC8IivWHwaLZwPrFY=
-github.com/go-resty/resty/v2 v2.7.0/go.mod h1:9PWDzw47qPphMRFfhsyk0NnSgvluHcljSMVIq3w7q0I=
+github.com/go-resty/resty/v2 v2.8.0 h1:J29d0JFWwSWrDCysnOK/YjsPMLQTx0TvgJEHVGvf2L8=
+github.com/go-resty/resty/v2 v2.8.0/go.mod h1:UCui0cMHekLrSntoMyofdSTaPpinlRHFtPpizuyDW2w=
 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0=
@@ -726,8 +726,8 @@ github.com/openziti/edge-api v0.25.36 h1:CjQTmXyYWjUX1Yc1FO0xkeATwd9A1KPu6CvNpst
 github.com/openziti/edge-api v0.25.36/go.mod h1:OMedQFZPoBER7E9nkyECML/Pt2C2uajdbm/BBP1wOuY=
 github.com/openziti/fablab v0.5.16 h1:+tqqDR5Sl4qLT3fSC+Dv1AGZuOeeX87h9LNKa1KJS5Y=
 github.com/openziti/fablab v0.5.16/go.mod h1:KiU/NxThNtH9U803Ep0WcSrbov3moZso2oRgIpQq0S0=
-github.com/openziti/fabric v0.24.30 h1:Ll2+11rL0A/ZNkTaZNdzs51KPA/xvd+oukyk9rjXY/Q=
-github.com/openziti/fabric v0.24.30/go.mod h1:BqlSpPukrkWA9LZSwZH32BGisKu59uJYnIzQ8bvXnX8=
+github.com/openziti/fabric v0.24.36 h1:KAlMvFjeCeTfuZDKbH55M03eKpIiv/Zx4Tt+qdRydDE=
+github.com/openziti/fabric v0.24.36/go.mod h1:ZglDYZABDfl0rWI4eXo4AYMc9G18XSlDNombiGNzs7Q=
 github.com/openziti/foundation/v2 v2.0.32 h1:I4CS7UCMjxWrS2JqzQvPfgalJM9Z9PKi3wtn0b7ULmM=
 github.com/openziti/foundation/v2 v2.0.32/go.mod h1:KCQatGmkVDIZnfSLrohKwmQi7kg7vnECpNTkpuLNguc=
 github.com/openziti/identity v1.0.63 h1:yJwVAcPpc+e7zhi2fb9lgAnqtR7cO5cdxuk2Y9Z6i0I=
@@ -742,8 +742,8 @@ github.com/openziti/sdk-golang v0.20.112 h1:oc6fNVg9poyRMdPQK2L84RGvOk3fKs3X+dzY
 github.com/openziti/sdk-golang v0.20.112/go.mod h1:/ioQ2qRfkwpSse3NCu37eXyJvQLgcK2RdenH4ajmhPw=
 github.com/openziti/secretstream v0.1.12 h1:N78CHxtqWzSyNFOsYtYRWNNTfX1ZDAPkFgzHobpodZU=
 github.com/openziti/secretstream v0.1.12/go.mod h1:gHMH1REH0r4VlmCtuWx8biU7j5ZfOivFjz9mLgwq7mk=
-github.com/openziti/storage v0.2.15 h1:mxEM+NuxZN3qPgXw4dc40FRLeMD4p+W1fspnt1zTGDM=
-github.com/openziti/storage v0.2.15/go.mod h1:BZcM3mHa13re5IlA8UjBCfjPCnN6H0lScg9U2lR1/cY=
+github.com/openziti/storage v0.2.18 h1:a5G9bYJ+nMNU00WvDlWyhMlrlhPLsnxPkyStHKE6KMY=
+github.com/openziti/storage v0.2.18/go.mod h1:IC8TdgMTTQJRZIwRjT2WeDs+vBF8R/cyiqA70NTzY/g=
 github.com/openziti/transport/v2 v2.0.107 h1:Azw/joD+6c1wgsaipXXYgzIn0eb212ymaF6EDckti3k=
 github.com/openziti/transport/v2 v2.0.107/go.mod h1:8WTSYMyGfLurWJYg58PBIp+abKLT6rTkFf8OVqK4SqI=
 github.com/openziti/x509-claims v1.0.3 h1:HNdQ8Nf1agB3lBs1gahcO6zfkeS4S5xoQ2/PkY4HRX0=
@@ -1155,11 +1155,11 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
 golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM=
-golang.org/x/net v0.0.0-20211029224645-99673261e6eb/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
 golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8=
 golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -1290,6 +1290,7 @@ golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9sn
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
 golang.org/x/term v0.12.0 h1:/ZfYdc3zq+q02Rv9vGqTeSItdzZTSNDmfTi0mBAuidU=
 golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -1304,6 +1305,7 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
 golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
 golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=