Files
ziti/controller/command/command.go
T
Paul Lorenz 444999bc5b Break the SDK terminator removal retry storm. Fixes #4156
- adds RemoveTerminatorsV2, an asynchronous control-channel removal, so a router
  no longer holds a request slot open waiting for a synchronous reply; gated on a
  new capability so routers fall back to the synchronous path against controllers
  that do not advertise it
- paces establish and remove retries with a per-terminator exponential backoff
  after a controller rejection, so a rejected operation is not re-sent on the next
  event-loop tick, which is what let a rate-limited removal sustain itself
- routes v2 removes to the leader and skips the ordered delete for a terminator the
  router confirms it created that no longer exists, so a storm of no-op retries
  cannot consume the command rate limiter
- gates that skip on the leader having applied everything committed when it took
  leadership, since a leader reports itself elected before those applies finish and
  would otherwise read a committed create as absent and drop its delete
- guards the leader term and every write to the caught-up flag with a dedicated
  lock, so a barrier completing alongside a leadership change cannot record a result
  for a term that has already ended
- scopes v2 removals to the requesting router, so a router can only remove
  terminators it owns on the asynchronous path as well as the synchronous one
- tracks in-flight removal batches on the router, resolving or requeueing them when
  the response arrives; lost responses self-heal via rate-limiter expiry and the
  retry scan
- waits for an outstanding operation before sending a terminator delete, and checks
  that before dequeuing, so a delete neither races a create the controller has
  committed but not applied nor falls out of the queue until the retry scan finds it
  minutes later
- matches a create response to the attempt that produced it, by control channel and
  reply sequence, since attempts overlap once one overruns the establishment timeout
  and every attempt carries the same terminator id; only the outstanding attempt's
  response may confirm the create or report the terminator idle
- records the attempt as the sender assigns its sequence, so a response cannot be
  processed before the request it answers is known, and withdraws an attempt whose
  send never queued
- reports a removal that succeeds past the establishment timeout as congestion
  rather than success, matching how the establish path classifies its own latency,
  since both resolve into the same limiter
- extracts the transient-error test both remove handlers share, so the v2 path
  treats a leaderless cluster as busy like v1 does instead of as a permanent
  failure, and the two cannot drift again
- applies the same delete sequencing and response correlation to the router-hosted
  (ER/T) registry, which cleared the same in-flight marker unguarded and dropped a
  deferred delete from its queue rather than leaving it there
- bounds the v2 response send with a timeout, so a router that stops draining its
  send queue cannot accumulate blocked responder goroutines on the controller
2026-09-24 13:24:51 -04:00

178 lines
5.1 KiB
Go

/*
Copyright NetFoundry Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package command
import (
"reflect"
"sync"
"github.com/michaelquigley/pfxlog"
"github.com/openziti/channel/v5"
"github.com/openziti/foundation/v2/debugz"
"github.com/openziti/foundation/v2/rate"
"github.com/openziti/ziti/v2/common/pb/ctrl_pb"
"github.com/openziti/ziti/v2/controller/change"
"github.com/openziti/ziti/v2/controller/storage/boltz"
"github.com/sirupsen/logrus"
)
// Command instances represent actions to be taken by the fabric controller. They are serializable,
// so they can be shipped from one controller for RAFT coordination
type Command interface {
// Apply runs the commands
Apply(ctx boltz.MutateContext) error
// GetChangeContext returns the change context associated with the command
GetChangeContext() *change.Context
// Encode returns a serialized representation of the command
Encode() ([]byte, error)
}
// CriticalCommand marks commands that establish base state (e.g. a snapshot restore). A failed apply
// of one must halt the node rather than log-and-advance the raft index, which would leave the node
// caught up on index but missing data. Ordinary commands are logged and skipped on failure.
type CriticalCommand interface {
Command
IsCriticalCommand()
}
// Validatable instances can be validated. Command instances which implement Validable will be validated
// before Command.Apply is called
type Validatable interface {
Validate() error
}
// Dispatcher instances will take a command and either send it to the leader to be applied, or if the current
// system is the leader, apply it locally
type Dispatcher interface {
Dispatch(command Command) error
IsLeaderOrLeaderless() bool
IsLeaderless() bool
IsLeader() bool
// IsLeaderAndCaughtUp reports whether this node is the leader and has applied everything that
// was committed when it took leadership. Required rather than IsLeader when a local read decides
// whether a replicated mutation is needed.
IsLeaderAndCaughtUp() bool
GetPeers() map[string]channel.Channel
GetRateLimiter() rate.RateLimiter
Bootstrap() error
CtrlAddresses() (uint64, []string, []*ctrl_pb.CtrlDetail)
// GetDecoders returns the command decoder registry used to decode commands dispatched
// through this dispatcher. Each dispatcher owns its own registry so multiple controllers
// in one process don't decode each other's commands.
GetDecoders() Decoders
}
// LocalDispatcher should be used when running a non-clustered system
type LocalDispatcher struct {
EncodeDecodeCommands bool
Limiter rate.RateLimiter
decodersInit sync.Once
decoders Decoders
}
func (self *LocalDispatcher) GetDecoders() Decoders {
self.decodersInit.Do(func() {
self.decoders = NewDecoders()
})
return self.decoders
}
func (self *LocalDispatcher) Bootstrap() error {
return nil
}
func (self *LocalDispatcher) IsLeader() bool {
return true
}
// IsLeaderAndCaughtUp is always true without raft: there is no log for the local view to trail.
func (self *LocalDispatcher) IsLeaderAndCaughtUp() bool {
return true
}
func (self *LocalDispatcher) IsLeaderOrLeaderless() bool {
return true
}
func (self *LocalDispatcher) IsLeaderless() bool {
return false
}
func (self *LocalDispatcher) GetPeers() map[string]channel.Channel {
return nil
}
func (self *LocalDispatcher) GetRateLimiter() rate.RateLimiter {
return self.Limiter
}
func (self *LocalDispatcher) CtrlAddresses() (uint64, []string, []*ctrl_pb.CtrlDetail) {
return 0, nil, nil
}
func (self *LocalDispatcher) Dispatch(command Command) error {
defer func() {
if p := recover(); p != nil {
pfxlog.Logger().
WithField(logrus.ErrorKey, p).
WithField("cmdType", reflect.TypeOf(command)).
Error("error while dispatching command of type")
debugz.DumpLocalStack()
panic(p)
}
}()
changeCtx := command.GetChangeContext()
if changeCtx == nil {
changeCtx = change.New().SetSourceType("unattributed").SetChangeAuthorType(change.AuthorTypeUnattributed)
}
if self.EncodeDecodeCommands {
bytes, err := command.Encode()
if err != nil {
return err
}
cmd, err := self.GetDecoders().Decode(bytes)
if err != nil {
return err
}
command = cmd
}
return self.Limiter.RunRateLimited(func() error {
ctx := changeCtx.NewMutateContext()
return command.Apply(ctx)
})
}
// Decoder instances know how to decode encoded commands
type Decoder interface {
Decode(commandType int32, data []byte) (Command, error)
}
// DecoderF is a function version of the Decoder interface
type DecoderF func(commandType int32, data []byte) (Command, error)
func (self DecoderF) Decode(commandType int32, data []byte) (Command, error) {
return self(commandType, data)
}