Files
ziti/controller/handler_edge_ctrl/create_tunnel_terminator.go
Paul Lorenz 210f6d50ed Stop selecting terminators whose host is gone. Fixes #4529
- adds an UNUSABLE_TERMINATOR route error code and UnusableTerminatorError for a terminator the router
  still has registered but can't dial
- reports an edge terminator that is being removed, or whose SDK channel has closed, as unusable
  before opening a connection on it, instead of failing in the msg mux with a generic error
- dials the replacement when a new bind takes over an edge terminator while the dialer is looking it up
- doesn't take over an edge terminator the dialer would report unusable, and orders the takeover with
  the dialer's recheck, so an id reported unusable isn't reused
- reports an ER/T terminator that is closing or being removed as unusable
- skips terminators reported invalid or unusable in path selection for up to a minute, and clears
  the mark when the terminator is created, deleted, or re-established by a create for an existing id
- keeps deleting terminators reported invalid, and logs the delete failure that was dropped before
- adds the ROUTER_ERR_UNUSABLE_TERMINATOR circuit failure cause
- passes a RouteSenderEnv to the route sender in place of its separate dependencies
2026-10-19 18:50:48 -04:00

214 lines
7.1 KiB
Go

/*
Copyright NetFoundry Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package handler_edge_ctrl
import (
"fmt"
"math"
"time"
"github.com/michaelquigley/pfxlog"
"github.com/openziti/channel/v5"
"github.com/openziti/ziti/v2/common"
"github.com/openziti/ziti/v2/common/pb/edge_ctrl_pb"
"github.com/openziti/ziti/v2/controller/db"
"github.com/openziti/ziti/v2/controller/env"
"github.com/openziti/ziti/v2/controller/model"
"github.com/openziti/ziti/v2/controller/models"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"google.golang.org/protobuf/proto"
)
type createTunnelTerminatorHandler struct {
baseRequestHandler
*TunnelState
}
func NewCreateTunnelTerminatorHandler(appEnv *env.AppEnv, ch channel.Channel, tunnelState *TunnelState) channel.ContentTypeReceiver {
return &createTunnelTerminatorHandler{
baseRequestHandler: baseRequestHandler{ch: ch, appEnv: appEnv},
TunnelState: tunnelState,
}
}
func (self *createTunnelTerminatorHandler) getTunnelState() *TunnelState {
return self.TunnelState
}
func (self *createTunnelTerminatorHandler) ContentType() int32 {
return int32(edge_ctrl_pb.ContentType_CreateTunnelTerminatorRequestType)
}
func (self *createTunnelTerminatorHandler) Label() string {
return "tunnel.create.terminator"
}
func (self *createTunnelTerminatorHandler) HandleReceive(msg *channel.Message, ch channel.Channel) {
startTime := time.Now()
req := &edge_ctrl_pb.CreateTunnelTerminatorRequest{}
if err := proto.Unmarshal(msg.Body, req); err != nil {
pfxlog.ContextLogger(ch.Label()).WithError(err).Error("could not unmarshal CreateTerminatorRequest")
return
}
ctx := &CreateTunnelTerminatorRequestContext{
baseTunnelRequestContext: baseTunnelRequestContext{
baseSessionRequestContext: baseSessionRequestContext{handler: self, msg: msg, env: self.appEnv},
},
req: req,
}
go self.CreateTerminator(ctx, startTime)
}
func (self *createTunnelTerminatorHandler) CreateTerminator(ctx *CreateTunnelTerminatorRequestContext, startTime time.Time) {
logger := logrus.
WithField("routerId", self.ch.Id()).
WithField("terminatorId", ctx.req.Address)
if !ctx.loadRouter() {
return
}
ctx.loadIdentity()
newApiSession := ctx.ensureApiSession(nil)
ctx.loadServiceForName(ctx.req.ServiceName)
ctx.ensureSessionForService(ctx.req.SessionId, db.SessionTypeBind)
ctx.verifyRouterEdgeRouterAccess()
if ctx.err != nil {
self.logResult(ctx, ctx.err)
return
}
logger = logger.WithField("serviceId", ctx.service.Id).WithField("service", ctx.service.Name)
if ctx.req.Cost > math.MaxUint16 {
self.returnError(ctx, invalidCost(fmt.Sprintf("invalid cost %v. cost must be between 0 and %v inclusive", ctx.req.Cost, math.MaxUint16)))
return
}
terminator, _ := self.getNetwork().Terminator.Read(ctx.req.Address)
if terminator != nil {
if err := ctx.validateExistingTerminator(terminator, logger); err != nil {
self.returnError(ctx, err)
return
}
} else {
terminator = &model.Terminator{
BaseEntity: models.BaseEntity{
Id: ctx.req.Address,
IsSystem: true,
},
Service: ctx.session.ServiceId,
Router: ctx.sourceRouter.Id,
Binding: common.TunnelBinding,
Address: ctx.req.Address,
InstanceId: ctx.req.InstanceId,
InstanceSecret: ctx.req.InstanceSecret,
PeerData: ctx.req.PeerData,
Precedence: ctx.req.GetXtPrecedence(),
Cost: uint16(ctx.req.Cost),
HostId: ctx.session.IdentityId,
SourceCtrl: self.appEnv.GetId(),
}
if err := self.appEnv.Managers.Terminator.Create(terminator, ctx.newTunnelChangeContext()); err != nil {
// terminator might have been created while we were trying to create.
if terminator, _ = self.getNetwork().Terminator.Read(ctx.req.Address); terminator != nil {
if validateError := ctx.validateExistingTerminator(terminator, logger); validateError != nil {
self.returnError(ctx, validateError)
return
}
} else {
self.returnError(ctx, internalError(err))
return
}
} else {
logger.Info("created terminator")
}
}
// A create for an existing terminator writes nothing, so no store event clears an unusable mark.
self.getNetwork().TerminatorEstablished(ctx.req.Address)
response := &edge_ctrl_pb.CreateTunnelTerminatorResponse{
Session: ctx.getCreateSessionResponse(),
TerminatorId: ctx.req.Address,
StartTime: ctx.req.StartTime,
}
if newApiSession {
var err error
response.ApiSession, err = ctx.getCreateApiSessionResponse()
if err != nil {
self.returnError(ctx, internalError(err))
return
}
}
body, err := proto.Marshal(response)
if err != nil {
logger.WithError(err).Error("failed to marshal CreateTunnelTerminatorResponse")
return
}
responseMsg := channel.NewMessage(response.GetContentType(), body)
responseMsg.ReplyTo(ctx.msg)
if err = self.ch.Send(responseMsg); err != nil {
logger.WithError(err).Error("failed to send CreateTunnelTerminatorResponse")
}
logger.WithField("elapsedTime", time.Since(startTime)).Info("completed create tunnel terminator operation")
}
type CreateTunnelTerminatorRequestContext struct {
baseTunnelRequestContext
req *edge_ctrl_pb.CreateTunnelTerminatorRequest
}
func (self *CreateTunnelTerminatorRequestContext) validateExistingTerminator(terminator *model.Terminator, log *logrus.Entry) controllerError {
if terminator.Binding != common.TunnelBinding {
log.WithField("binding", common.TunnelBinding).
WithField("conflictingBinding", terminator.Binding).
Error("selected terminator address conflicts with a terminator for a different binding")
return internalError(errors.New("selected id conflicts with terminator for different binding"))
}
if terminator.Service != self.session.ServiceId {
log.WithField("conflictingService", terminator.Service).
Error("selected terminator address conflicts with a terminator for a different service")
return internalError(errors.New("selected id conflicts with terminator for different service"))
}
if terminator.Router != self.sourceRouter.Id {
log.WithField("conflictingRouter", terminator.Router).
Error("selected terminator address conflicts with a terminator for a different router")
return internalError(errors.New("selected id conflicts with terminator for different router"))
}
if terminator.HostId != self.session.IdentityId {
log.WithField("identityId", self.session.IdentityId).
WithField("conflictingIdentity", terminator.HostId).
Error("selected terminator address conflicts with a terminator for a different identity")
return internalError(errors.New("selected id conflicts with terminator for different identity"))
}
log.Info("terminator already exists")
return nil
}