Files
ziti/controller/command/command.go
T
Paul Lorenz 11e049a452 Halt on critical-command apply failure instead of advancing. For #4104
- adds a CriticalCommand marker interface for commands that establish base
  state, and marks SyncSnapshotCommand (a full snapshot restore) as critical
- makes BoltDbFsm.Apply halt when a critical command fails to apply, rather
  than logging the error and persisting the advanced raft index; the failed
  apply's in-tx index update is rolled back and left unpersisted, so raft
  replays and retries the command on restart instead of the node running
  caught-up-on-index but empty-on-data
- attaches the command type to all apply log lines so a failure is
  self-contained
2026-07-17 18:07:50 -04:00

152 lines
4.2 KiB
Go

/*
Copyright NetFoundry Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package command
import (
"reflect"
"github.com/michaelquigley/pfxlog"
"github.com/openziti/channel/v5"
"github.com/openziti/foundation/v2/debugz"
"github.com/openziti/foundation/v2/rate"
"github.com/openziti/ziti/v2/common/pb/ctrl_pb"
"github.com/openziti/ziti/v2/controller/change"
"github.com/openziti/ziti/v2/controller/storage/boltz"
"github.com/sirupsen/logrus"
)
// Command instances represent actions to be taken by the fabric controller. They are serializable,
// so they can be shipped from one controller for RAFT coordination
type Command interface {
// Apply runs the commands
Apply(ctx boltz.MutateContext) error
// GetChangeContext returns the change context associated with the command
GetChangeContext() *change.Context
// Encode returns a serialized representation of the command
Encode() ([]byte, error)
}
// CriticalCommand marks commands that establish base state (e.g. a snapshot restore). A failed apply
// of one must halt the node rather than log-and-advance the raft index, which would leave the node
// caught up on index but missing data. Ordinary commands are logged and skipped on failure.
type CriticalCommand interface {
Command
IsCriticalCommand()
}
// Validatable instances can be validated. Command instances which implement Validable will be validated
// before Command.Apply is called
type Validatable interface {
Validate() error
}
// Dispatcher instances will take a command and either send it to the leader to be applied, or if the current
// system is the leader, apply it locally
type Dispatcher interface {
Dispatch(command Command) error
IsLeaderOrLeaderless() bool
IsLeaderless() bool
IsLeader() bool
GetPeers() map[string]channel.Channel
GetRateLimiter() rate.RateLimiter
Bootstrap() error
CtrlAddresses() (uint64, []string, []*ctrl_pb.CtrlDetail)
}
// LocalDispatcher should be used when running a non-clustered system
type LocalDispatcher struct {
EncodeDecodeCommands bool
Limiter rate.RateLimiter
}
func (self *LocalDispatcher) Bootstrap() error {
return nil
}
func (self *LocalDispatcher) IsLeader() bool {
return true
}
func (self *LocalDispatcher) IsLeaderOrLeaderless() bool {
return true
}
func (self *LocalDispatcher) IsLeaderless() bool {
return false
}
func (self *LocalDispatcher) GetPeers() map[string]channel.Channel {
return nil
}
func (self *LocalDispatcher) GetRateLimiter() rate.RateLimiter {
return self.Limiter
}
func (self *LocalDispatcher) CtrlAddresses() (uint64, []string, []*ctrl_pb.CtrlDetail) {
return 0, nil, nil
}
func (self *LocalDispatcher) Dispatch(command Command) error {
defer func() {
if p := recover(); p != nil {
pfxlog.Logger().
WithField(logrus.ErrorKey, p).
WithField("cmdType", reflect.TypeOf(command)).
Error("error while dispatching command of type")
debugz.DumpLocalStack()
panic(p)
}
}()
changeCtx := command.GetChangeContext()
if changeCtx == nil {
changeCtx = change.New().SetSourceType("unattributed").SetChangeAuthorType(change.AuthorTypeUnattributed)
}
if self.EncodeDecodeCommands {
bytes, err := command.Encode()
if err != nil {
return err
}
cmd, err := GetDefaultDecoders().Decode(bytes)
if err != nil {
return err
}
command = cmd
}
return self.Limiter.RunRateLimited(func() error {
ctx := changeCtx.NewMutateContext()
return command.Apply(ctx)
})
}
// Decoder instances know how to decode encoded commands
type Decoder interface {
Decode(commandType int32, data []byte) (Command, error)
}
// DecoderF is a function version of the Decoder interface
type DecoderF func(commandType int32, data []byte) (Command, error)
func (self DecoderF) Decode(commandType int32, data []byte) (Command, error) {
return self(commandType, data)
}