Files
ziti/controller/handler_edge_ctrl/create_terminator_v2.go
Paul Lorenz 210f6d50ed Stop selecting terminators whose host is gone. Fixes #4529
- adds an UNUSABLE_TERMINATOR route error code and UnusableTerminatorError for a terminator the router
  still has registered but can't dial
- reports an edge terminator that is being removed, or whose SDK channel has closed, as unusable
  before opening a connection on it, instead of failing in the msg mux with a generic error
- dials the replacement when a new bind takes over an edge terminator while the dialer is looking it up
- doesn't take over an edge terminator the dialer would report unusable, and orders the takeover with
  the dialer's recheck, so an id reported unusable isn't reused
- reports an ER/T terminator that is closing or being removed as unusable
- skips terminators reported invalid or unusable in path selection for up to a minute, and clears
  the mark when the terminator is created, deleted, or re-established by a create for an existing id
- keeps deleting terminators reported invalid, and logs the delete failure that was dropped before
- adds the ROUTER_ERR_UNUSABLE_TERMINATOR circuit failure cause
- passes a RouteSenderEnv to the route sender in place of its separate dependencies
2026-10-19 18:50:48 -04:00

225 lines
7.3 KiB
Go

/*
Copyright NetFoundry Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package handler_edge_ctrl
import (
"fmt"
"math"
"time"
"github.com/michaelquigley/pfxlog"
"github.com/openziti/channel/v5"
"github.com/openziti/channel/v5/protobufs"
"github.com/openziti/ziti/v2/common"
"github.com/openziti/ziti/v2/common/pb/edge_ctrl_pb"
"github.com/openziti/ziti/v2/controller/command"
"github.com/openziti/ziti/v2/controller/db"
"github.com/openziti/ziti/v2/controller/env"
"github.com/openziti/ziti/v2/controller/fields"
"github.com/openziti/ziti/v2/controller/model"
"github.com/openziti/ziti/v2/controller/models"
"github.com/sirupsen/logrus"
"google.golang.org/protobuf/proto"
)
type createTerminatorV2Handler struct {
baseRequestHandler
}
func NewCreateTerminatorV2Handler(appEnv *env.AppEnv, ch channel.Channel) channel.ContentTypeReceiver {
return &createTerminatorV2Handler{
baseRequestHandler{
ch: ch,
appEnv: appEnv,
},
}
}
func (self *createTerminatorV2Handler) ContentType() int32 {
return int32(edge_ctrl_pb.ContentType_CreateTerminatorV2RequestType)
}
func (self *createTerminatorV2Handler) Label() string {
return "create.terminator"
}
func (self *createTerminatorV2Handler) HandleReceive(msg *channel.Message, ch channel.Channel) {
req := &edge_ctrl_pb.CreateTerminatorV2Request{}
if err := proto.Unmarshal(msg.Body, req); err != nil {
pfxlog.ContextLogger(ch.Label()).WithError(err).Error("could not unmarshal CreateTerminatorV2Request")
return
}
ctx := &CreateTerminatorV2RequestContext{
baseSessionRequestContext: baseSessionRequestContext{handler: self, msg: msg, env: self.appEnv},
req: req,
}
go self.CreateTerminatorV2(ctx)
}
func (self *createTerminatorV2Handler) CreateTerminatorV2(ctx *CreateTerminatorV2RequestContext) {
start := time.Now()
logger := pfxlog.ContextLogger(self.ch.Label()).
WithField("routerId", self.ch.Id()).
WithField("terminatorId", ctx.req.Address)
if !ctx.loadRouter() {
return
}
ctx.verifyTerminatorId(ctx.req.Address)
ctx.loadSession(ctx.req.SessionToken, ctx.req.ApiSessionToken)
ctx.checkSessionType(db.SessionTypeBind)
ctx.verifyIdentityEdgeRouterAccess()
ctx.loadService()
if ctx.err != nil {
self.returnError(ctx, ctx.err, logger)
return
}
logger = logger.WithField("serviceId", ctx.service.Id).WithField("service", ctx.service.Name)
if ctx.req.Cost > math.MaxUint16 {
ctx.err = invalidCost(fmt.Sprintf("invalid cost %v. cost must be between 0 and %v inclusive", ctx.req.Cost, math.MaxUint16))
self.returnError(ctx, ctx.err, logger)
return
}
terminator, _ := self.getNetwork().Terminator.Read(ctx.req.Address)
if terminator != nil {
if ctx.err = ctx.validateExistingTerminator(terminator, ctx.session.IdentityId, common.EdgeBinding, logger); ctx.err != nil {
self.returnError(ctx, ctx.err, logger)
return
}
// if the precedence or cost has changed, update the terminator
if terminator.Precedence != ctx.req.GetXtPrecedence() || terminator.Cost != uint16(ctx.req.Cost) {
terminator.Precedence = ctx.req.GetXtPrecedence()
terminator.Cost = uint16(ctx.req.Cost)
err := self.appEnv.GetManagers().Terminator.Update(terminator, fields.UpdatedFieldsMap{
db.FieldTerminatorPrecedence: struct{}{},
db.FieldTerminatorCost: struct{}{},
}, ctx.newChangeContext())
if err != nil {
// A rate-limited or leaderless dispatch is transient; reply busy so the router requeues
// promptly instead of treating it as a hard failure.
if command.WasRateLimited(err) || command.WasLeaderless(err) {
self.returnError(ctx, busyError(err), logger)
return
}
self.returnError(ctx, internalError(err), logger)
return
}
}
} else {
terminator = &model.Terminator{
BaseEntity: models.BaseEntity{
Id: ctx.req.Address,
IsSystem: true,
},
Service: ctx.session.ServiceId,
Router: ctx.sourceRouter.Id,
Binding: common.EdgeBinding,
Address: ctx.req.Address,
InstanceId: ctx.req.InstanceId,
InstanceSecret: ctx.req.InstanceSecret,
PeerData: ctx.req.PeerData,
Precedence: ctx.req.GetXtPrecedence(),
Cost: uint16(ctx.req.Cost),
HostId: ctx.session.IdentityId,
SourceCtrl: self.appEnv.GetId(),
}
cmd := &model.CreateEdgeTerminatorCmd{
Env: self.appEnv,
Entity: terminator,
Context: ctx.newChangeContext(),
}
createStart := time.Now()
if err := self.appEnv.GetHostController().GetNetwork().Managers.Dispatcher.Dispatch(cmd); err != nil {
// terminator might have been created while we were trying to create.
if terminator, _ = self.getNetwork().Terminator.Read(ctx.req.Address); terminator != nil {
if validateError := ctx.validateExistingTerminator(terminator, ctx.session.IdentityId, common.EdgeBinding, logger); validateError != nil {
self.returnError(ctx, validateError, logger)
return
}
} else {
if command.WasRateLimited(err) || command.WasLeaderless(err) {
self.returnError(ctx, busyError(err), logger)
return
}
self.returnError(ctx, internalError(err), logger)
return
}
} else {
logger.WithField("terminator", terminator.Id).
WithField("createTime", time.Since(createStart)).
Info("created terminator")
}
}
// A create for an existing terminator writes nothing, so no store event clears an unusable mark.
self.getNetwork().TerminatorEstablished(ctx.req.Address)
response := &edge_ctrl_pb.CreateTerminatorV2Response{
TerminatorId: terminator.Id,
Result: edge_ctrl_pb.CreateTerminatorResult_Success,
}
body, err := proto.Marshal(response)
if err != nil {
logger.WithError(err).Error("failed to marshal CreateTunnelTerminatorResponse")
return
}
responseMsg := channel.NewMessage(response.GetContentType(), body)
responseMsg.ReplyTo(ctx.msg)
if err = self.ch.Send(responseMsg); err != nil {
logger.WithError(err).Error("failed to send CreateTunnelTerminatorResponse")
}
logger.WithField("elapsed", time.Since(start)).Info("completed create terminator v2 operation")
}
func (self *createTerminatorV2Handler) returnError(ctx *CreateTerminatorV2RequestContext, err controllerError, logger *logrus.Entry) {
response := &edge_ctrl_pb.CreateTerminatorV2Response{
TerminatorId: ctx.req.Address,
Result: retryHintToResult(err.GetRetryHint()),
Msg: err.Error(),
ErrorCode: err.ErrorCode(),
RetryHint: uint32(err.GetRetryHint()),
}
if sendErr := protobufs.MarshalTyped(response).ReplyTo(ctx.msg).Send(self.ch); sendErr != nil {
logger.WithError(err).WithField("sendError", sendErr).Error("failed to send error response")
} else {
logger.WithError(err).Error("responded with error")
}
}
type CreateTerminatorV2RequestContext struct {
baseSessionRequestContext
req *edge_ctrl_pb.CreateTerminatorV2Request
}
func (self *CreateTerminatorV2RequestContext) GetSessionToken() string {
return self.req.SessionToken
}