mirror of
https://github.com/openziti/ziti.git
synced 2026-09-10 08:45:41 +00:00
d838e209ac
- adds command.WasLeaderless to classify cluster-has-no-leader dispatch errors as retriable - replies busy instead of dropping or hard-failing terminator creates when the cluster is briefly leaderless, so the router backs off and requeues promptly rather than waiting for its multi-minute recovery scan - removes the racy up-front leaderless pre-check in the sdk create handler in favor of classifying the actual dispatch result - applies the same retriable classification to the ert tunnel create and batch remove terminator handlers
222 lines
7.2 KiB
Go
222 lines
7.2 KiB
Go
/*
|
|
Copyright NetFoundry Inc.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
https://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package handler_edge_ctrl
|
|
|
|
import (
|
|
"fmt"
|
|
"math"
|
|
"time"
|
|
|
|
"github.com/michaelquigley/pfxlog"
|
|
"github.com/openziti/channel/v5"
|
|
"github.com/openziti/channel/v5/protobufs"
|
|
"github.com/openziti/ziti/v2/common"
|
|
"github.com/openziti/ziti/v2/common/pb/edge_ctrl_pb"
|
|
"github.com/openziti/ziti/v2/controller/command"
|
|
"github.com/openziti/ziti/v2/controller/db"
|
|
"github.com/openziti/ziti/v2/controller/env"
|
|
"github.com/openziti/ziti/v2/controller/fields"
|
|
"github.com/openziti/ziti/v2/controller/model"
|
|
"github.com/openziti/ziti/v2/controller/models"
|
|
"github.com/sirupsen/logrus"
|
|
"google.golang.org/protobuf/proto"
|
|
)
|
|
|
|
type createTerminatorV2Handler struct {
|
|
baseRequestHandler
|
|
}
|
|
|
|
func NewCreateTerminatorV2Handler(appEnv *env.AppEnv, ch channel.Channel) channel.ContentTypeReceiver {
|
|
return &createTerminatorV2Handler{
|
|
baseRequestHandler{
|
|
ch: ch,
|
|
appEnv: appEnv,
|
|
},
|
|
}
|
|
}
|
|
|
|
func (self *createTerminatorV2Handler) ContentType() int32 {
|
|
return int32(edge_ctrl_pb.ContentType_CreateTerminatorV2RequestType)
|
|
}
|
|
|
|
func (self *createTerminatorV2Handler) Label() string {
|
|
return "create.terminator"
|
|
}
|
|
|
|
func (self *createTerminatorV2Handler) HandleReceive(msg *channel.Message, ch channel.Channel) {
|
|
req := &edge_ctrl_pb.CreateTerminatorV2Request{}
|
|
if err := proto.Unmarshal(msg.Body, req); err != nil {
|
|
pfxlog.ContextLogger(ch.Label()).WithError(err).Error("could not unmarshal CreateTerminatorV2Request")
|
|
return
|
|
}
|
|
|
|
ctx := &CreateTerminatorV2RequestContext{
|
|
baseSessionRequestContext: baseSessionRequestContext{handler: self, msg: msg, env: self.appEnv},
|
|
req: req,
|
|
}
|
|
|
|
go self.CreateTerminatorV2(ctx)
|
|
}
|
|
|
|
func (self *createTerminatorV2Handler) CreateTerminatorV2(ctx *CreateTerminatorV2RequestContext) {
|
|
start := time.Now()
|
|
logger := pfxlog.ContextLogger(self.ch.Label()).
|
|
WithField("routerId", self.ch.Id()).
|
|
WithField("terminatorId", ctx.req.Address)
|
|
|
|
if !ctx.loadRouter() {
|
|
return
|
|
}
|
|
ctx.verifyTerminatorId(ctx.req.Address)
|
|
ctx.loadSession(ctx.req.SessionToken, ctx.req.ApiSessionToken)
|
|
ctx.checkSessionType(db.SessionTypeBind)
|
|
ctx.verifyIdentityEdgeRouterAccess()
|
|
ctx.loadService()
|
|
|
|
if ctx.err != nil {
|
|
self.returnError(ctx, ctx.err, logger)
|
|
return
|
|
}
|
|
|
|
logger = logger.WithField("serviceId", ctx.service.Id).WithField("service", ctx.service.Name)
|
|
|
|
if ctx.req.Cost > math.MaxUint16 {
|
|
ctx.err = invalidCost(fmt.Sprintf("invalid cost %v. cost must be between 0 and %v inclusive", ctx.req.Cost, math.MaxUint16))
|
|
self.returnError(ctx, ctx.err, logger)
|
|
return
|
|
}
|
|
|
|
terminator, _ := self.getNetwork().Terminator.Read(ctx.req.Address)
|
|
if terminator != nil {
|
|
if ctx.err = ctx.validateExistingTerminator(terminator, ctx.session.IdentityId, common.EdgeBinding, logger); ctx.err != nil {
|
|
self.returnError(ctx, ctx.err, logger)
|
|
return
|
|
}
|
|
|
|
// if the precedence or cost has changed, update the terminator
|
|
if terminator.Precedence != ctx.req.GetXtPrecedence() || terminator.Cost != uint16(ctx.req.Cost) {
|
|
terminator.Precedence = ctx.req.GetXtPrecedence()
|
|
terminator.Cost = uint16(ctx.req.Cost)
|
|
err := self.appEnv.GetManagers().Terminator.Update(terminator, fields.UpdatedFieldsMap{
|
|
db.FieldTerminatorPrecedence: struct{}{},
|
|
db.FieldTerminatorCost: struct{}{},
|
|
}, ctx.newChangeContext())
|
|
|
|
if err != nil {
|
|
// A rate-limited or leaderless dispatch is transient; reply busy so the router requeues
|
|
// promptly instead of treating it as a hard failure.
|
|
if command.WasRateLimited(err) || command.WasLeaderless(err) {
|
|
self.returnError(ctx, busyError(err), logger)
|
|
return
|
|
}
|
|
self.returnError(ctx, internalError(err), logger)
|
|
return
|
|
}
|
|
}
|
|
} else {
|
|
terminator = &model.Terminator{
|
|
BaseEntity: models.BaseEntity{
|
|
Id: ctx.req.Address,
|
|
IsSystem: true,
|
|
},
|
|
Service: ctx.session.ServiceId,
|
|
Router: ctx.sourceRouter.Id,
|
|
Binding: common.EdgeBinding,
|
|
Address: ctx.req.Address,
|
|
InstanceId: ctx.req.InstanceId,
|
|
InstanceSecret: ctx.req.InstanceSecret,
|
|
PeerData: ctx.req.PeerData,
|
|
Precedence: ctx.req.GetXtPrecedence(),
|
|
Cost: uint16(ctx.req.Cost),
|
|
HostId: ctx.session.IdentityId,
|
|
SourceCtrl: self.appEnv.GetId(),
|
|
}
|
|
|
|
cmd := &model.CreateEdgeTerminatorCmd{
|
|
Env: self.appEnv,
|
|
Entity: terminator,
|
|
Context: ctx.newChangeContext(),
|
|
}
|
|
|
|
createStart := time.Now()
|
|
if err := self.appEnv.GetHostController().GetNetwork().Managers.Dispatcher.Dispatch(cmd); err != nil {
|
|
// terminator might have been created while we were trying to create.
|
|
if terminator, _ = self.getNetwork().Terminator.Read(ctx.req.Address); terminator != nil {
|
|
if validateError := ctx.validateExistingTerminator(terminator, ctx.session.IdentityId, common.EdgeBinding, logger); validateError != nil {
|
|
self.returnError(ctx, validateError, logger)
|
|
return
|
|
}
|
|
} else {
|
|
if command.WasRateLimited(err) || command.WasLeaderless(err) {
|
|
self.returnError(ctx, busyError(err), logger)
|
|
return
|
|
}
|
|
self.returnError(ctx, internalError(err), logger)
|
|
return
|
|
}
|
|
} else {
|
|
logger.WithField("terminator", terminator.Id).
|
|
WithField("createTime", time.Since(createStart)).
|
|
Info("created terminator")
|
|
}
|
|
}
|
|
|
|
response := &edge_ctrl_pb.CreateTerminatorV2Response{
|
|
TerminatorId: terminator.Id,
|
|
Result: edge_ctrl_pb.CreateTerminatorResult_Success,
|
|
}
|
|
|
|
body, err := proto.Marshal(response)
|
|
if err != nil {
|
|
logger.WithError(err).Error("failed to marshal CreateTunnelTerminatorResponse")
|
|
return
|
|
}
|
|
|
|
responseMsg := channel.NewMessage(response.GetContentType(), body)
|
|
responseMsg.ReplyTo(ctx.msg)
|
|
if err = self.ch.Send(responseMsg); err != nil {
|
|
logger.WithError(err).Error("failed to send CreateTunnelTerminatorResponse")
|
|
}
|
|
|
|
logger.WithField("elapsed", time.Since(start)).Info("completed create terminator v2 operation")
|
|
}
|
|
|
|
func (self *createTerminatorV2Handler) returnError(ctx *CreateTerminatorV2RequestContext, err controllerError, logger *logrus.Entry) {
|
|
response := &edge_ctrl_pb.CreateTerminatorV2Response{
|
|
TerminatorId: ctx.req.Address,
|
|
Result: retryHintToResult(err.GetRetryHint()),
|
|
Msg: err.Error(),
|
|
ErrorCode: err.ErrorCode(),
|
|
RetryHint: uint32(err.GetRetryHint()),
|
|
}
|
|
|
|
if sendErr := protobufs.MarshalTyped(response).ReplyTo(ctx.msg).Send(self.ch); sendErr != nil {
|
|
logger.WithError(err).WithField("sendError", sendErr).Error("failed to send error response")
|
|
} else {
|
|
logger.WithError(err).Error("responded with error")
|
|
}
|
|
}
|
|
|
|
type CreateTerminatorV2RequestContext struct {
|
|
baseSessionRequestContext
|
|
req *edge_ctrl_pb.CreateTerminatorV2Request
|
|
}
|
|
|
|
func (self *CreateTerminatorV2RequestContext) GetSessionToken() string {
|
|
return self.req.SessionToken
|
|
}
|