Files
ziti/controller/handler_edge_ctrl/create_terminator_v2.go
Paul Lorenz d838e209ac Signal retry on leaderless terminator operations. Fixes #4160
- adds command.WasLeaderless to classify cluster-has-no-leader dispatch errors as retriable
- replies busy instead of dropping or hard-failing terminator creates when the cluster is briefly leaderless, so the router backs off and requeues promptly rather than waiting for its multi-minute recovery scan
- removes the racy up-front leaderless pre-check in the sdk create handler in favor of classifying the actual dispatch result
- applies the same retriable classification to the ert tunnel create and batch remove terminator handlers
2026-07-24 11:43:33 -04:00

222 lines
7.2 KiB
Go

/*
Copyright NetFoundry Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package handler_edge_ctrl
import (
"fmt"
"math"
"time"
"github.com/michaelquigley/pfxlog"
"github.com/openziti/channel/v5"
"github.com/openziti/channel/v5/protobufs"
"github.com/openziti/ziti/v2/common"
"github.com/openziti/ziti/v2/common/pb/edge_ctrl_pb"
"github.com/openziti/ziti/v2/controller/command"
"github.com/openziti/ziti/v2/controller/db"
"github.com/openziti/ziti/v2/controller/env"
"github.com/openziti/ziti/v2/controller/fields"
"github.com/openziti/ziti/v2/controller/model"
"github.com/openziti/ziti/v2/controller/models"
"github.com/sirupsen/logrus"
"google.golang.org/protobuf/proto"
)
type createTerminatorV2Handler struct {
baseRequestHandler
}
func NewCreateTerminatorV2Handler(appEnv *env.AppEnv, ch channel.Channel) channel.ContentTypeReceiver {
return &createTerminatorV2Handler{
baseRequestHandler{
ch: ch,
appEnv: appEnv,
},
}
}
func (self *createTerminatorV2Handler) ContentType() int32 {
return int32(edge_ctrl_pb.ContentType_CreateTerminatorV2RequestType)
}
func (self *createTerminatorV2Handler) Label() string {
return "create.terminator"
}
func (self *createTerminatorV2Handler) HandleReceive(msg *channel.Message, ch channel.Channel) {
req := &edge_ctrl_pb.CreateTerminatorV2Request{}
if err := proto.Unmarshal(msg.Body, req); err != nil {
pfxlog.ContextLogger(ch.Label()).WithError(err).Error("could not unmarshal CreateTerminatorV2Request")
return
}
ctx := &CreateTerminatorV2RequestContext{
baseSessionRequestContext: baseSessionRequestContext{handler: self, msg: msg, env: self.appEnv},
req: req,
}
go self.CreateTerminatorV2(ctx)
}
func (self *createTerminatorV2Handler) CreateTerminatorV2(ctx *CreateTerminatorV2RequestContext) {
start := time.Now()
logger := pfxlog.ContextLogger(self.ch.Label()).
WithField("routerId", self.ch.Id()).
WithField("terminatorId", ctx.req.Address)
if !ctx.loadRouter() {
return
}
ctx.verifyTerminatorId(ctx.req.Address)
ctx.loadSession(ctx.req.SessionToken, ctx.req.ApiSessionToken)
ctx.checkSessionType(db.SessionTypeBind)
ctx.verifyIdentityEdgeRouterAccess()
ctx.loadService()
if ctx.err != nil {
self.returnError(ctx, ctx.err, logger)
return
}
logger = logger.WithField("serviceId", ctx.service.Id).WithField("service", ctx.service.Name)
if ctx.req.Cost > math.MaxUint16 {
ctx.err = invalidCost(fmt.Sprintf("invalid cost %v. cost must be between 0 and %v inclusive", ctx.req.Cost, math.MaxUint16))
self.returnError(ctx, ctx.err, logger)
return
}
terminator, _ := self.getNetwork().Terminator.Read(ctx.req.Address)
if terminator != nil {
if ctx.err = ctx.validateExistingTerminator(terminator, ctx.session.IdentityId, common.EdgeBinding, logger); ctx.err != nil {
self.returnError(ctx, ctx.err, logger)
return
}
// if the precedence or cost has changed, update the terminator
if terminator.Precedence != ctx.req.GetXtPrecedence() || terminator.Cost != uint16(ctx.req.Cost) {
terminator.Precedence = ctx.req.GetXtPrecedence()
terminator.Cost = uint16(ctx.req.Cost)
err := self.appEnv.GetManagers().Terminator.Update(terminator, fields.UpdatedFieldsMap{
db.FieldTerminatorPrecedence: struct{}{},
db.FieldTerminatorCost: struct{}{},
}, ctx.newChangeContext())
if err != nil {
// A rate-limited or leaderless dispatch is transient; reply busy so the router requeues
// promptly instead of treating it as a hard failure.
if command.WasRateLimited(err) || command.WasLeaderless(err) {
self.returnError(ctx, busyError(err), logger)
return
}
self.returnError(ctx, internalError(err), logger)
return
}
}
} else {
terminator = &model.Terminator{
BaseEntity: models.BaseEntity{
Id: ctx.req.Address,
IsSystem: true,
},
Service: ctx.session.ServiceId,
Router: ctx.sourceRouter.Id,
Binding: common.EdgeBinding,
Address: ctx.req.Address,
InstanceId: ctx.req.InstanceId,
InstanceSecret: ctx.req.InstanceSecret,
PeerData: ctx.req.PeerData,
Precedence: ctx.req.GetXtPrecedence(),
Cost: uint16(ctx.req.Cost),
HostId: ctx.session.IdentityId,
SourceCtrl: self.appEnv.GetId(),
}
cmd := &model.CreateEdgeTerminatorCmd{
Env: self.appEnv,
Entity: terminator,
Context: ctx.newChangeContext(),
}
createStart := time.Now()
if err := self.appEnv.GetHostController().GetNetwork().Managers.Dispatcher.Dispatch(cmd); err != nil {
// terminator might have been created while we were trying to create.
if terminator, _ = self.getNetwork().Terminator.Read(ctx.req.Address); terminator != nil {
if validateError := ctx.validateExistingTerminator(terminator, ctx.session.IdentityId, common.EdgeBinding, logger); validateError != nil {
self.returnError(ctx, validateError, logger)
return
}
} else {
if command.WasRateLimited(err) || command.WasLeaderless(err) {
self.returnError(ctx, busyError(err), logger)
return
}
self.returnError(ctx, internalError(err), logger)
return
}
} else {
logger.WithField("terminator", terminator.Id).
WithField("createTime", time.Since(createStart)).
Info("created terminator")
}
}
response := &edge_ctrl_pb.CreateTerminatorV2Response{
TerminatorId: terminator.Id,
Result: edge_ctrl_pb.CreateTerminatorResult_Success,
}
body, err := proto.Marshal(response)
if err != nil {
logger.WithError(err).Error("failed to marshal CreateTunnelTerminatorResponse")
return
}
responseMsg := channel.NewMessage(response.GetContentType(), body)
responseMsg.ReplyTo(ctx.msg)
if err = self.ch.Send(responseMsg); err != nil {
logger.WithError(err).Error("failed to send CreateTunnelTerminatorResponse")
}
logger.WithField("elapsed", time.Since(start)).Info("completed create terminator v2 operation")
}
func (self *createTerminatorV2Handler) returnError(ctx *CreateTerminatorV2RequestContext, err controllerError, logger *logrus.Entry) {
response := &edge_ctrl_pb.CreateTerminatorV2Response{
TerminatorId: ctx.req.Address,
Result: retryHintToResult(err.GetRetryHint()),
Msg: err.Error(),
ErrorCode: err.ErrorCode(),
RetryHint: uint32(err.GetRetryHint()),
}
if sendErr := protobufs.MarshalTyped(response).ReplyTo(ctx.msg).Send(self.ch); sendErr != nil {
logger.WithError(err).WithField("sendError", sendErr).Error("failed to send error response")
} else {
logger.WithError(err).Error("responded with error")
}
}
type CreateTerminatorV2RequestContext struct {
baseSessionRequestContext
req *edge_ctrl_pb.CreateTerminatorV2Request
}
func (self *CreateTerminatorV2RequestContext) GetSessionToken() string {
return self.req.SessionToken
}