Files
ziti/controller/model/router_model.go
Paul Lorenz a4285af743 Deliver link listeners to every controller until written. Fixes #4456
- tracks the link listener set owed to each controller in the link registry, marked on a listener change and after each reconnect
- sends the listener set from the registry loop with wait-for-wire once the controller's reconnect link announcement is written, retrying every tick until the write succeeds, replacing the fire-and-forget send to controllers connected at the time of the change
- fences listener completions with the reconnect generation so a send that reached an earlier connection does not count for a later one
- keeps sync state for controllers absent from a controller snapshot while they still owe the listener set
- stamps the listener set sent to controllers with the config generation read in the same snapshot, matching the hello
- makes the controller skip the peer fan-out for an UpdateLinkListeners message whose set matches the one it replaced, still recording the message generation
- makes Router.SetLinkListeners return the set it held, read under the same lock as the update
2026-09-29 15:23:45 -04:00

212 lines
7.7 KiB
Go

/*
Copyright NetFoundry Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package model
import (
"fmt"
"sync"
"sync/atomic"
"time"
"github.com/openziti/foundation/v2/errorz"
"github.com/openziti/foundation/v2/versions"
"github.com/openziti/ziti/v2/common/capabilities"
"github.com/openziti/ziti/v2/common/ctrlchan"
"github.com/openziti/ziti/v2/common/pb/ctrl_pb"
"github.com/openziti/ziti/v2/controller/db"
"github.com/openziti/ziti/v2/controller/models"
"github.com/openziti/ziti/v2/controller/storage/boltz"
"go.etcd.io/bbolt"
)
type Router struct {
models.BaseEntity
Name string
Fingerprint *string
// listeners is the router's currently-advertised link listener set.
// Hello carries the initial snapshot; UpdateLinkListeners pushes
// mid-session changes from the router. Guarded by mu, so it is
// unexported: reach it through SetLinkListeners / GetLinkListeners.
listeners []*ctrl_pb.Listener
// listenerGeneration is the generation of the recorded listener set, scoped
// to the current control channel session. Zero means nothing recorded yet.
listenerGeneration uint64
// mu guards the fields a connected Router mutates mid-session, which
// today is just listeners. Fields added later should share it rather
// than take their own lock; contention is irrelevant at this scale.
mu sync.RWMutex
Control ctrlchan.CtrlChannel
Connected atomic.Bool
ConnectTime time.Time
// VersionInfo is reported in the router's hello and is not persisted, so it is only populated on the
// instance built for a control-channel connection. It is nil on an instance loaded from the database.
// Read it from GetConnected rather than from whatever instance is to hand.
VersionInfo *versions.VersionInfo
routerLinks RouterLinks
Cost uint16
NoTraversal bool
Disabled bool
Capabilities *capabilities.RouterCapabilityMask
Interfaces []*Interface
CtrlChanListeners map[string][]string
Configs []string
}
func (entity *Router) GetLinks() []*Link {
return entity.routerLinks.GetLinks()
}
func (entity *Router) toBoltEntityForUpdate(tx *bbolt.Tx, env Env, checker boltz.FieldChecker) (*db.Router, error) {
if err := validateRouterConfigs(tx, env, entity.Configs, checker); err != nil {
return nil, err
}
return entity.toBoltEntity(), nil
}
func (entity *Router) toBoltEntityForCreate(tx *bbolt.Tx, env Env) (*db.Router, error) {
if err := validateRouterConfigs(tx, env, entity.Configs, nil); err != nil {
return nil, err
}
return entity.toBoltEntity(), nil
}
func (entity *Router) toBoltEntity() *db.Router {
return &db.Router{
BaseExtEntity: *boltz.NewExtEntity(entity.Id, entity.Tags),
Name: entity.Name,
Fingerprint: entity.Fingerprint,
Cost: entity.Cost,
NoTraversal: entity.NoTraversal,
Disabled: entity.Disabled,
CtrlChanListeners: entity.CtrlChanListeners,
Interfaces: InterfacesToBolt(entity.Interfaces),
Configs: entity.Configs,
}
}
// validateRouterConfigs validates that all configs target routers and have unique config types.
// If checker is non-nil and the configs field is not being updated, validation is skipped.
func validateRouterConfigs(tx *bbolt.Tx, env Env, configs []string, checker boltz.FieldChecker) error {
if checker != nil && !checker.IsUpdated(db.EntityTypeConfigs) {
return nil
}
if len(configs) == 0 {
return nil
}
typeMap := map[string]*db.Config{}
configStore := env.GetStores().Config
configTypeStore := env.GetStores().ConfigType
for _, id := range configs {
config, err := configStore.LoadById(tx, id)
if err != nil {
return err
}
configType, err := configTypeStore.LoadById(tx, config.TypeId)
if err != nil {
if boltz.IsErrNotFoundErr(err) {
msg := fmt.Sprintf("config %v references config type %v which does not exist",
config.Name, config.TypeId)
return errorz.NewFieldError(msg, "configs", configs)
}
return err
}
if configType.Target != db.ConfigTypeTargetRouter {
msg := fmt.Sprintf("config %v has config type %v which does not target routers",
config.Name, configType.Name)
return errorz.NewFieldError(msg, "configs", configs)
}
if conflictConfig, found := typeMap[config.TypeId]; found {
msg := fmt.Sprintf("duplicate configs named %v and %v found for config type %v. Only one config of a given type is allowed per router",
conflictConfig.Name, config.Name, configType.Name)
return errorz.NewFieldError(msg, "configs", configs)
}
typeMap[config.TypeId] = config
}
return nil
}
func (entity *Router) fillFrom(_ Env, _ *bbolt.Tx, boltRouter *db.Router) error {
entity.Name = boltRouter.Name
entity.Fingerprint = boltRouter.Fingerprint
entity.Cost = boltRouter.Cost
entity.NoTraversal = boltRouter.NoTraversal
entity.Disabled = boltRouter.Disabled
entity.CtrlChanListeners = boltRouter.CtrlChanListeners
entity.Interfaces = InterfacesFromBolt(boltRouter.Interfaces)
entity.Configs = boltRouter.Configs
entity.FillCommon(boltRouter)
return nil
}
func (entity *Router) addLinkListener(addr, linkProtocol string, groups []string) {
entity.mu.Lock()
defer entity.mu.Unlock()
entity.listeners = append(entity.listeners, &ctrl_pb.Listener{
Address: addr,
Protocol: linkProtocol,
Groups: groups,
})
}
// SetLinkListeners records listeners when generation is newer than the one recorded for this control channel
// session, or when the recorded generation is zero, and reports whether it did. It returns the set held before
// the call either way, read under the same lock, so a caller can tell whether the update changed anything.
// Older and equal generations are dropped: a multi-underlay control channel doesn't order updates across
// underlays, and a stale set would be redistributed to peers. Slices are replaced, never mutated, so a returned
// set stays safe to iterate.
func (entity *Router) SetLinkListeners(listeners []*ctrl_pb.Listener, generation uint64) (previous []*ctrl_pb.Listener, recorded bool) {
entity.mu.Lock()
defer entity.mu.Unlock()
previous = entity.listeners
if generation <= entity.listenerGeneration && entity.listenerGeneration != 0 {
return previous, false
}
entity.listeners = listeners
entity.listenerGeneration = generation
return previous, true
}
// ResetLinkListeners clears the listener set and the generation it was recorded
// at, for a newly-connected control channel. The router mints generations per
// process, so they restart when it does; a generation remembered from a
// previous session would reject a restarted router's first updates.
func (entity *Router) ResetLinkListeners() {
entity.mu.Lock()
defer entity.mu.Unlock()
entity.listeners = nil
entity.listenerGeneration = 0
}
// GetLinkListeners returns the current link listener slice under a read
// lock. The returned slice is the live header — safe to iterate because
// SetLinkListeners always replaces the whole slice rather than mutating
// it in place — but callers should not modify it.
func (entity *Router) GetLinkListeners() []*ctrl_pb.Listener {
entity.mu.RLock()
defer entity.mu.RUnlock()
return entity.listeners
}
func (entity *Router) HasCapability(capability capabilities.RouterCapability) bool {
return entity.Capabilities != nil && entity.Capabilities.IsSet(capability)
}