/* Copyright NetFoundry Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package sync_strats import ( "context" "crypto/sha1" "crypto/tls" "encoding/binary" "encoding/json" "fmt" "github.com/lucsky/cuid" "github.com/michaelquigley/pfxlog" "github.com/openziti/channel/v2" "github.com/openziti/foundation/v2/debugz" "github.com/openziti/foundation/v2/genext" nfPem "github.com/openziti/foundation/v2/pem" "github.com/openziti/storage/ast" "github.com/openziti/storage/boltz" "github.com/openziti/ziti/common" "github.com/openziti/ziti/common/build" "github.com/openziti/ziti/common/pb/edge_ctrl_pb" "github.com/openziti/ziti/controller/change" "github.com/openziti/ziti/controller/db" "github.com/openziti/ziti/controller/env" "github.com/openziti/ziti/controller/handler_edge_ctrl" "github.com/openziti/ziti/controller/model" "github.com/openziti/ziti/controller/network" cmap "github.com/orcaman/concurrent-map/v2" "github.com/pkg/errors" "go.etcd.io/bbolt" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/timestamppb" "reflect" "sync" "sync/atomic" "time" ) const ( RouterSyncStrategyInstant env.RouterSyncStrategyType = "instant" ZdbIndexKey string = "index" ZdbKey string = "zdb" ) var _ env.RouterSyncStrategy = (*InstantStrategy)(nil) // InstantStrategyOptions is the options for the instant strategy. // - MaxQueuedRouterConnects - max number of router connected events to buffer // - MaxQueuedClientHellos - max number of client hello messages to buffer // - RouterConnectWorkerCount - max number of workers used to process router connections // - SyncWorkerCount - max number of workers used to send api sessions/session data // - RouterTxBufferSize - max number of messages buffered to be send to a router // - HelloSendTimeout - the max amount of time per worker to wait to send hellos // - SessionChunkSize - the number of sessions to send in each message type InstantStrategyOptions struct { MaxQueuedRouterConnects int32 MaxQueuedClientHellos int32 RouterConnectWorkerCount int32 SyncWorkerCount int32 RouterTxBufferSize int HelloSendTimeout time.Duration SessionChunkSize int } // InstantStrategy assumes that on connect, the router requires and instant // and full set of API Sessions. Send individual create, update, delete events for sessions after synchronization. // // This strategy uses a series of queues and workers to managed synchronization state. The order of events is as follows: // 1. An edge router connects to the controller, triggering RouterConnected() // 2. A RouterSender is created encapsulating the Edge Router, Router, and Sync State // 3. The RouterSender is queued on the routerConnectedQueue channel which buffers up to options.MaxQueuedRouterConnects // 4. The routerConnectedQueue is read and the edge server hello is sent // 5. The controller waits for a client hello to be received via ReceiveClientHello message // 6. The client hello is used to identity the RouterSender associated with the client and is queued on // the receivedClientHelloQueue channel which buffers up to options.MaxQueuedClientHellos // 7. A startSynchronizeWorker will pick up the RouterSender from the receivedClientHelloQueue and being to // send data to the edge router via the RouterSender type InstantStrategy struct { InstantStrategyOptions rtxMap *routerTxMap helloHandler channel.TypedReceiveHandler resyncHandler channel.TypedReceiveHandler ae *env.AppEnv routerConnectedQueue chan *RouterSender receivedClientHelloQueue chan *RouterSender indexProvider IndexProvider stopNotify chan struct{} stopped atomic.Bool *common.RouterDataModel servicePolicyHandler *constraintToIndexedEvents[*db.ServicePolicy] identityHandler *constraintToIndexedEvents[*db.Identity] postureCheckHandler *constraintToIndexedEvents[*db.PostureCheck] serviceHandler *constraintToIndexedEvents[*db.EdgeService] caHandler *constraintToIndexedEvents[*db.Ca] revocationHandler *constraintToIndexedEvents[*db.Revocation] controllerHandler *constraintToIndexedEvents[*db.Controller] changeSetLock sync.Mutex changeSets map[uint64]*edge_ctrl_pb.DataState_ChangeSet } func (strategy *InstantStrategy) AddPublicKey(cert *tls.Certificate) { publicKey := newPublicKey(cert.Certificate[0], edge_ctrl_pb.DataState_PublicKey_X509CertDer, []edge_ctrl_pb.DataState_PublicKey_Usage{edge_ctrl_pb.DataState_PublicKey_ClientX509CertValidation, edge_ctrl_pb.DataState_PublicKey_JWTValidation}) newModel := &edge_ctrl_pb.DataState_Event_PublicKey{PublicKey: publicKey} newEvent := &edge_ctrl_pb.DataState_Event{ Action: edge_ctrl_pb.DataState_Create, Model: newModel, } strategy.HandlePublicKeyEvent(newEvent, newModel) } // Initialize implements RouterDataModelCache func (strategy *InstantStrategy) Initialize(logSize uint64, bufferSize uint) error { strategy.RouterDataModel = common.NewSenderRouterDataModel(logSize, bufferSize) if strategy.ae.HostController.IsRaftEnabled() { strategy.indexProvider = &RaftIndexProvider{ index: strategy.ae.GetHostController().GetRaftIndex(), } } else { strategy.indexProvider = &NonHaIndexProvider{ ae: strategy.ae, } } err := strategy.BuildAll() if err != nil { return err } go strategy.handleRouterModelEvents(strategy.RouterDataModel.NewListener()) //policy create/delete/update strategy.servicePolicyHandler = &constraintToIndexedEvents[*db.ServicePolicy]{ indexProvider: strategy.indexProvider, createHandler: strategy.ServicePolicyCreate, updateHandler: strategy.ServicePolicyUpdate, deleteHandler: strategy.ServicePolicyDelete, } strategy.ae.GetStores().ServicePolicy.AddEntityConstraint(strategy.servicePolicyHandler) //identity create/delete/update strategy.identityHandler = &constraintToIndexedEvents[*db.Identity]{ indexProvider: strategy.indexProvider, createHandler: strategy.IdentityCreate, updateHandler: strategy.IdentityUpdate, deleteHandler: strategy.IdentityDelete, } strategy.ae.GetStores().Identity.AddEntityConstraint(strategy.identityHandler) //posture check create/delete/update strategy.postureCheckHandler = &constraintToIndexedEvents[*db.PostureCheck]{ indexProvider: strategy.indexProvider, createHandler: strategy.PostureCheckCreate, updateHandler: strategy.PostureCheckUpdate, deleteHandler: strategy.PostureCheckDelete, } strategy.ae.GetStores().PostureCheck.AddEntityConstraint(strategy.postureCheckHandler) //service create/delete/update strategy.serviceHandler = &constraintToIndexedEvents[*db.EdgeService]{ indexProvider: strategy.indexProvider, createHandler: strategy.ServiceCreate, updateHandler: strategy.ServiceUpdate, deleteHandler: strategy.ServiceDelete, } strategy.ae.GetStores().EdgeService.AddEntityConstraint(strategy.serviceHandler) //ca create/delete/update strategy.caHandler = &constraintToIndexedEvents[*db.Ca]{ indexProvider: strategy.indexProvider, createHandler: strategy.CaCreate, updateHandler: strategy.CaUpdate, deleteHandler: strategy.CaDelete, } strategy.ae.GetStores().Ca.AddEntityConstraint(strategy.caHandler) //revocation create/delete/update strategy.revocationHandler = &constraintToIndexedEvents[*db.Revocation]{ indexProvider: strategy.indexProvider, createHandler: strategy.RevocationCreate, updateHandler: strategy.RevocationUpdate, deleteHandler: strategy.RevocationDelete, } strategy.ae.GetStores().Revocation.AddEntityConstraint(strategy.revocationHandler) strategy.controllerHandler = &constraintToIndexedEvents[*db.Controller]{ indexProvider: strategy.indexProvider, createHandler: strategy.ControllerCreate, updateHandler: strategy.ControllerUpdate, } strategy.ae.GetDbProvider().GetDb().AddTxCompleteListener(strategy.completeChangeSet) return nil } func NewInstantStrategy(ae *env.AppEnv, options InstantStrategyOptions) *InstantStrategy { if options.MaxQueuedRouterConnects <= 0 { pfxlog.Logger().Panicf("MaxQueuedRouterConnects for InstantStrategy cannot be less than 1, got %d", options.MaxQueuedRouterConnects) } if options.MaxQueuedClientHellos <= 0 { pfxlog.Logger().Panicf("MaxQueuedClientHellos for InstantStrategy cannot be less than 1, got %d", options.MaxQueuedClientHellos) } if options.RouterConnectWorkerCount <= 0 { pfxlog.Logger().Panicf("RouterConnectWorkerCount for InstantStrategy cannot be less than 1, got %d", options.RouterConnectWorkerCount) } if options.SyncWorkerCount <= 0 { pfxlog.Logger().Panicf("SyncWorkerCount for InstantStrategy cannot be less than 1, got %d", options.SyncWorkerCount) } if options.RouterTxBufferSize < 0 { pfxlog.Logger().Panicf("RouterTxBufferSize for InstantStrategy cannot be less than 0, got %d", options.MaxQueuedRouterConnects) } if options.SessionChunkSize <= 0 { pfxlog.Logger().Panicf("SessionChunkSize for InstantStrategy cannot be less than 1, got %d", options.SessionChunkSize) } strategy := &InstantStrategy{ InstantStrategyOptions: options, rtxMap: &routerTxMap{ internalMap: cmap.New[*RouterSender](), }, ae: ae, routerConnectedQueue: make(chan *RouterSender, options.MaxQueuedRouterConnects), receivedClientHelloQueue: make(chan *RouterSender, options.MaxQueuedClientHellos), RouterDataModel: common.NewSenderRouterDataModel(10000, 10000), stopNotify: make(chan struct{}), changeSets: map[uint64]*edge_ctrl_pb.DataState_ChangeSet{}, } err := strategy.Initialize(10000, 1000) if err != nil { pfxlog.Logger().WithError(err).Fatal("could not build initial data model for router synchronization") } strategy.helloHandler = handler_edge_ctrl.NewHelloHandler(ae, strategy.ReceiveClientHello) strategy.resyncHandler = handler_edge_ctrl.NewResyncHandler(ae, strategy.ReceiveResync) for i := int32(0); i < options.RouterConnectWorkerCount; i++ { go strategy.startHandleRouterConnectWorker() } for i := int32(0); i < options.SyncWorkerCount; i++ { go strategy.startSynchronizeWorker() } ae.GetHostController().GetNetwork().AddInspectTarget(strategy.inspect) return strategy } func (strategy *InstantStrategy) GetEdgeRouterState(id string) env.RouterStateValues { return strategy.rtxMap.GetState(id) } func (strategy *InstantStrategy) Type() env.RouterSyncStrategyType { return RouterSyncStrategyInstant } func (strategy *InstantStrategy) Stop() { if strategy.stopped.CompareAndSwap(false, true) { close(strategy.stopNotify) } } func (strategy *InstantStrategy) RouterConnected(edgeRouter *model.EdgeRouter, router *network.Router) { log := pfxlog.Logger().WithField("sync_strategy", strategy.Type()). WithField("routerId", router.Id). WithField("routerName", router.Name). WithField("routerFingerprint", *router.Fingerprint) //connecting router has closed control channel if router.Control.IsClosed() { log.Errorf("connecting router has closed control channel [id: %s], ignoring", router.Id) return } existingRtx := strategy.rtxMap.Get(router.Id) //same channel, do nothing if existingRtx != nil && existingRtx.Router.Control == router.Control { log.Errorf("duplicate router connection detected [id: %s], channels are the same, ignoring", router.Id) return } rtx := newRouterSender(edgeRouter, router, strategy.RouterTxBufferSize) rtx.SetSyncStatus(env.RouterSyncQueued) rtx.SetIsOnline(true) log.WithField("syncStatus", rtx.SyncStatus()).Info("edge router connected, adding to sync routerConnectedQueue") strategy.rtxMap.Add(router.Id, rtx) strategy.routerConnectedQueue <- rtx } func (strategy *InstantStrategy) RouterDisconnected(router *network.Router) { log := pfxlog.Logger().WithField("sync_strategy", strategy.Type()). WithField("routerId", router.Id). WithField("routerName", router.Name). WithField("routerFingerprint", genext.OrDefault(router.Fingerprint)) existingRtx := strategy.rtxMap.Get(router.Id) if existingRtx == nil { log.Infof("edge router [%s] disconnect event, but no rtx found, ignoring", router.Id) return } if existingRtx.Router.Control != router.Control && !existingRtx.Router.Control.IsClosed() { log.Infof("edge router [%s] disconnect event, but channels do not match and existing channel is still open, ignoring", router.Id) return } log.Infof("edge router [%s] disconnect event, router rtx removed", router.Id) existingRtx.SetIsOnline(false) strategy.rtxMap.Remove(router) } func (strategy *InstantStrategy) GetReceiveHandlers() []channel.TypedReceiveHandler { var result []channel.TypedReceiveHandler if strategy.helloHandler != nil { result = append(result, strategy.helloHandler) } if strategy.resyncHandler != nil { result = append(result, strategy.resyncHandler) } return result } func (strategy *InstantStrategy) ApiSessionAdded(apiSession *db.ApiSession) { logger := pfxlog.Logger().WithField("strategy", strategy.Type()) apiSessionProto, err := apiSessionToProto(strategy.ae, apiSession.Token, apiSession.IdentityId, apiSession.Id) if err != nil { logger.WithField("apiSessionId", apiSession.Id). Errorf("error for individual api session added, could not convert to proto: %v", err) return } logger.WithField("apiSessionId", apiSession.Id).WithField("fingerprints", apiSessionProto.CertFingerprints).Debug("adding apiSession") state := &InstantSyncState{ Id: cuid.New(), IsLast: true, Sequence: 0, } strategy.rtxMap.Range(func(rtx *RouterSender) { _ = strategy.sendApiSessionAdded(rtx, false, state, []*edge_ctrl_pb.ApiSession{apiSessionProto}) }) } func (strategy *InstantStrategy) ApiSessionUpdated(apiSession *db.ApiSession, _ *db.ApiSessionCertificate) { logger := pfxlog.Logger().WithField("strategy", strategy.Type()) apiSessionProto, err := apiSessionToProto(strategy.ae, apiSession.Token, apiSession.IdentityId, apiSession.Id) if err != nil { logger.WithField("apiSessionId", apiSession.Id). Errorf("error for individual api session added, could not convert to proto: %v", err) return } logger.WithField("apiSessionId", apiSession.Id).WithField("fingerprints", apiSessionProto.CertFingerprints).Debug("updating apiSession") apiSessionAdded := &edge_ctrl_pb.ApiSessionUpdated{ ApiSessions: []*edge_ctrl_pb.ApiSession{apiSessionProto}, } strategy.rtxMap.Range(func(rtx *RouterSender) { content, _ := proto.Marshal(apiSessionAdded) msg := channel.NewMessage(env.ApiSessionUpdatedType, content) msg.Headers[env.SyncStrategyTypeHeader] = []byte(strategy.Type()) msg.Headers[env.SyncStrategyStateHeader] = nil _ = rtx.Send(msg) }) } func (strategy *InstantStrategy) ApiSessionDeleted(apiSession *db.ApiSession) { sessionRemoved := &edge_ctrl_pb.ApiSessionRemoved{ Tokens: []string{apiSession.Token}, } strategy.rtxMap.Range(func(rtx *RouterSender) { content, _ := proto.Marshal(sessionRemoved) msg := channel.NewMessage(env.ApiSessionRemovedType, content) _ = rtx.Send(msg) }) } func (strategy *InstantStrategy) SessionDeleted(session *db.Session) { sessionRemoved := &edge_ctrl_pb.SessionRemoved{ Tokens: []string{session.Token}, } strategy.rtxMap.Range(func(rtx *RouterSender) { content, _ := proto.Marshal(sessionRemoved) msg := channel.NewMessage(env.SessionRemovedType, content) _ = rtx.Send(msg) }) } func (strategy *InstantStrategy) startHandleRouterConnectWorker() { for { select { case <-strategy.stopNotify: return case rtx := <-strategy.routerConnectedQueue: func() { defer func() { if r := recover(); r != nil { pfxlog.Logger().Errorf("router connect worker panic, worker recovering: %v\n%v", r, debugz.GenerateLocalStack()) rtx.SetSyncStatus(env.RouterSyncError) rtx.logger().Errorf("panic during edge router connection, sync failed") } }() strategy.hello(rtx) }() } } } func (strategy *InstantStrategy) startSynchronizeWorker() { for { select { case <-strategy.stopNotify: return case rtx := <-strategy.receivedClientHelloQueue: func() { defer func() { if r := recover(); r != nil { pfxlog.Logger().Errorf("sync worker panic, worker recovering: %v\n%v", r, debugz.GenerateLocalStack()) rtx.SetSyncStatus(env.RouterSyncError) rtx.logger().Errorf("panic during edge router sync, sync failed") } }() strategy.synchronize(rtx) }() } } } func (strategy *InstantStrategy) hello(rtx *RouterSender) { logger := rtx.logger().WithField("strategy", strategy.Type()) logger.Info("edge router sync starting") if rtx.Router.Control.IsClosed() { rtx.SetSyncStatus(env.RouterSyncDisconnected) logger.WithField("syncStatus", rtx.SyncStatus()).Info("edge router sync aborting, edge router disconnected before sync started") return } rtx.SetSyncStatus(env.RouterSyncHello) logger.WithField("syncStatus", rtx.SyncStatus()).Info("sending edge router hello") strategy.sendHello(rtx) } func (strategy *InstantStrategy) sendHello(rtx *RouterSender) { logger := rtx.logger().WithField("strategy", strategy.Type()) serverVersion := build.GetBuildInfo().Version() serverHello := &edge_ctrl_pb.ServerHello{ Version: serverVersion, Data: map[string]string{ "instant": "true", "routerModel": "true", }, ByteData: map[string][]byte{}, } buf, err := proto.Marshal(serverHello) if err != nil { logger.WithError(err).Error("could not marshal serverHello") return } msg := channel.NewMessage(env.ServerHelloType, buf) msg.PutBoolHeader(int32(edge_ctrl_pb.Header_RouterDataModel), true) if err = msg.WithTimeout(strategy.HelloSendTimeout).Send(rtx.Router.Control); err != nil { if rtx.Router.Control.IsClosed() { rtx.SetSyncStatus(env.RouterSyncDisconnected) rtx.logger().WithError(err).Error("timed out sending serverHello message for edge router, connection closed, giving up") } else { rtx.SetSyncStatus(env.RouterSyncHelloTimeout) rtx.logger().WithError(err).Error("timed out sending serverHello message for edge router, queuing again") go func() { strategy.routerConnectedQueue <- rtx }() } } } func (strategy *InstantStrategy) ReceiveResync(routerId string, _ *edge_ctrl_pb.RequestClientReSync) { rtx := strategy.rtxMap.Get(routerId) if rtx == nil { routerName := "" if router, _ := strategy.ae.Managers.Router.Read(routerId); router != nil { routerName = router.Name } pfxlog.Logger(). WithField("strategy", strategy.Type()). WithField("routerId", routerId). WithField("routerName", routerName). Error("received resync from router that is currently not tracked by the strategy, dropping resync") return } rtx.SetSyncStatus(env.RouterSyncResyncWait) rtx.logger().WithField("strategy", strategy.Type()).Info("received resync from router, queuing") rtx.RouterModelIndex = nil strategy.queueClientHello(rtx) } func (strategy *InstantStrategy) queueClientHello(rtx *RouterSender) { select { case strategy.receivedClientHelloQueue <- rtx: return default: } ticker := time.NewTicker(time.Second) defer ticker.Stop() go func() { for { if ch := rtx.Router.Control; ch == nil || ch.IsClosed() { return } select { case strategy.receivedClientHelloQueue <- rtx: return case <-strategy.stopNotify: return case <-ticker.C: } } }() } func (strategy *InstantStrategy) ReceiveClientHello(routerId string, msg *channel.Message, respHello *edge_ctrl_pb.ClientHello) { rtx := strategy.rtxMap.Get(routerId) if rtx == nil { routerName := "" if router, _ := strategy.ae.Managers.Router.Read(routerId); router != nil { routerName = router.Name } pfxlog.Logger(). WithField("strategy", strategy.Type()). WithField("routerId", routerId). WithField("routerName", routerName). Error("received hello from router that is currently not tracked by the strategy, dropping hello") return } rtx.SetSyncStatus(env.RouterSyncHelloWait) logger := rtx.logger().WithField("strategy", strategy.Type()). WithField("protocols", respHello.Protocols). WithField("protocolPorts", respHello.ProtocolPorts). WithField("listeners", respHello.Listeners). WithField("data", respHello.Data). WithField("version", rtx.Router.VersionInfo.Version). WithField("revision", rtx.Router.VersionInfo.Revision). WithField("buildDate", rtx.Router.VersionInfo.BuildDate). WithField("os", rtx.Router.VersionInfo.OS). WithField("arch", rtx.Router.VersionInfo.Arch) if supported, ok := msg.Headers.GetBoolHeader(int32(edge_ctrl_pb.Header_RouterDataModel)); ok && supported { rtx.SupportsRouterModel = true if index, ok := msg.Headers.GetUint64Header(int32(edge_ctrl_pb.Header_RouterDataModelIndex)); ok { rtx.RouterModelIndex = &index } } protocols := map[string]string{} if len(respHello.Listeners) > 0 { for _, listener := range respHello.Listeners { protocols[listener.Advertise.Protocol] = fmt.Sprintf("%s://%s:%d", listener.Advertise.Protocol, listener.Advertise.Hostname, listener.Advertise.Port) } } else { for idx, protocol := range respHello.Protocols { if len(respHello.ProtocolPorts) > idx { port := respHello.ProtocolPorts[idx] ingressUrl := fmt.Sprintf("%s://%s:%s", protocol, respHello.Hostname, port) protocols[protocol] = ingressUrl } } } rtx.SetHostname(respHello.Hostname) rtx.SetProtocols(protocols) rtx.SetVersionInfo(*rtx.Router.VersionInfo) serverVersion := build.GetBuildInfo().Version() logger.Infof("edge router sent hello with version [%s] to controller with version [%s]", respHello.Version, serverVersion) strategy.queueClientHello(rtx) } func (strategy *InstantStrategy) synchronize(rtx *RouterSender) { defer func() { rtx.logger().WithField("strategy", strategy.Type()).WithField("SupportsRouterModel", rtx.SupportsRouterModel).Infof("exiting synchronization, final status: %s", rtx.SyncStatus()) }() if rtx.Router.Control.IsClosed() { rtx.SetSyncStatus(env.RouterSyncDisconnected) rtx.logger().WithField("strategy", strategy.Type()).WithField("SupportsRouterModel", rtx.SupportsRouterModel).Error("attempting to start synchronization with edge router, but it has disconnected") } rtx.SetSyncStatus(env.RouterSynInProgress) logger := rtx.logger().WithField("strategy", strategy.Type()).WithField("SupportsRouterModel", rtx.SupportsRouterModel) logger.Info("started synchronizing edge router") chunkSize := 100 err := strategy.ae.GetDbProvider().GetDb().View(func(tx *bbolt.Tx) error { var apiSessions []*edge_ctrl_pb.ApiSession state := &InstantSyncState{ Id: cuid.New(), IsLast: true, Sequence: 0, } for cursor := strategy.ae.GetStores().ApiSession.IterateIds(tx, ast.BoolNodeTrue); cursor.IsValid(); cursor.Next() { current := cursor.Current() apiSession, err := strategy.ae.GetStores().ApiSession.LoadById(tx, string(current)) if err != nil { logger.WithError(err).WithField("apiSessionId", string(current)).Errorf("error querying api session [%s]: %v", string(current), err) continue } apiSessionProto, err := apiSessionToProtoWithTx(tx, strategy.ae, apiSession.Token, apiSession.IdentityId, apiSession.Id) if err != nil { logger.WithError(err).WithField("apiSessionId", string(current)).Errorf("error turning apiSession [%s] into proto: %v", string(current), err) continue } apiSessions = append(apiSessions, apiSessionProto) if len(apiSessions) >= chunkSize { state.IsLast = !cursor.IsValid() if err = strategy.sendApiSessionAdded(rtx, true, state, apiSessions); err != nil { return err } state.Sequence = state.Sequence + 1 apiSessions = []*edge_ctrl_pb.ApiSession{} } } if len(apiSessions) > 0 { state.IsLast = true if err := strategy.sendApiSessionAdded(rtx, true, state, apiSessions); err != nil { return err } } return nil }) if err != nil { logger.WithError(err).Error("failure synchronizing api sessions") rtx.SetSyncStatus(env.RouterSyncError) return } if rtx.SupportsRouterModel { replayFrom := rtx.RouterModelIndex if replayFrom != nil { rtx.RouterModelIndex = nil events, ok := strategy.RouterDataModel.ReplayFrom(*replayFrom) if ok { var err error for _, curEvent := range events { err = strategy.sendDataStateChangeSet(rtx, curEvent) if err != nil { pfxlog.Logger().WithError(err). WithField("eventIndex", curEvent.Index). WithField("evenType", reflect.TypeOf(curEvent).String()). WithField("eventIsSynthetic", curEvent.IsSynthetic). Error("could not send data state event") } } var pks []*edge_ctrl_pb.DataState_PublicKey strategy.RouterDataModel.PublicKeys.IterCb(func(_ string, v *edge_ctrl_pb.DataState_PublicKey) { pks = append(pks, v) }) for _, pk := range pks { peerEvent := &edge_ctrl_pb.DataState_Event{ IsSynthetic: true, Action: edge_ctrl_pb.DataState_Create, Model: &edge_ctrl_pb.DataState_Event_PublicKey{ PublicKey: newPublicKey(pk.Data, pk.Format, pk.Usages), }, } changeSet := &edge_ctrl_pb.DataState_ChangeSet{ Changes: []*edge_ctrl_pb.DataState_Event{peerEvent}, } err = strategy.sendDataStateChangeSet(rtx, changeSet) if err != nil { pfxlog.Logger().WithError(err). WithField("evenType", reflect.TypeOf(peerEvent).String()). WithField("eventAction", peerEvent.Action). WithField("eventIsSynthetic", peerEvent.IsSynthetic). Error("could not send data state event for peers") } } } // no error sync is done, if err try full state if err == nil { rtx.SetSyncStatus(env.RouterSyncDone) return } pfxlog.Logger().WithError(err).Error("could not send events for router sync, attempting full state") } //full sync dataState := strategy.RouterDataModel.GetDataState() if dataState == nil { return } dataState.EndIndex = strategy.indexProvider.CurrentIndex() if err := strategy.sendDataState(rtx, dataState); err != nil { logger.WithError(err).Error("failure sending full data state") rtx.SetSyncStatus(env.RouterSyncError) return } } rtx.SetSyncStatus(env.RouterSyncDone) } func (strategy *InstantStrategy) sendApiSessionAdded(rtx *RouterSender, isFullState bool, state *InstantSyncState, apiSessions []*edge_ctrl_pb.ApiSession) error { stateBytes, _ := json.Marshal(state) msgContent := &edge_ctrl_pb.ApiSessionAdded{ IsFullState: isFullState, ApiSessions: apiSessions, } msgContentBytes, _ := proto.Marshal(msgContent) msg := channel.NewMessage(env.ApiSessionAddedType, msgContentBytes) msg.Headers[env.SyncStrategyTypeHeader] = []byte(strategy.Type()) msg.Headers[env.SyncStrategyStateHeader] = stateBytes return rtx.Send(msg) } func (strategy *InstantStrategy) handleRouterModelEvents(eventChannel <-chan *edge_ctrl_pb.DataState_ChangeSet) { for { select { case newEvent := <-eventChannel: strategy.rtxMap.Range(func(rtx *RouterSender) { if !rtx.SupportsRouterModel { return } err := strategy.sendDataStateChangeSet(rtx, newEvent) if err != nil { pfxlog.Logger().WithError(err).WithField("routerId", rtx.Router.Id).Error("error sending data state to router") } }) case <-strategy.ae.HostController.GetCloseNotifyChannel(): return } } } type InstantSyncState struct { Id string `json:"id"` //unique id for the sync attempt IsLast bool `json:"isLast"` // Sequence int `json:"sequence"` //increasing id from 0 per id for the } func (strategy *InstantStrategy) BuildServicePolicies(tx *bbolt.Tx) error { for cursor := strategy.ae.GetStores().ServicePolicy.IterateIds(tx, ast.BoolNodeTrue); cursor.IsValid(); cursor.Next() { currentBytes := cursor.Current() currentId := string(currentBytes) storeModel, err := strategy.ae.GetStores().ServicePolicy.LoadById(tx, currentId) if err != nil { return err } servicePolicy := newServicePolicy(storeModel) newModel := &edge_ctrl_pb.DataState_Event_ServicePolicy{ServicePolicy: servicePolicy} newEvent := &edge_ctrl_pb.DataState_Event{ Action: edge_ctrl_pb.DataState_Create, Model: newModel, } strategy.HandleServicePolicyEvent(newEvent, newModel) result := strategy.ae.GetManagers().ServicePolicy.ListAssociatedIds(tx, storeModel.Id) addServicesEvent := &edge_ctrl_pb.DataState_ServicePolicyChange{ PolicyId: currentId, RelatedEntityIds: result.ServiceIds, RelatedEntityType: edge_ctrl_pb.ServicePolicyRelatedEntityType_RelatedService, Add: true, } strategy.HandleServicePolicyChange(addServicesEvent) addIdentitiesEvent := &edge_ctrl_pb.DataState_ServicePolicyChange{ PolicyId: currentId, RelatedEntityIds: result.IdentityIds, RelatedEntityType: edge_ctrl_pb.ServicePolicyRelatedEntityType_RelatedIdentity, Add: true, } strategy.HandleServicePolicyChange(addIdentitiesEvent) addPostureChecksEvent := &edge_ctrl_pb.DataState_ServicePolicyChange{ PolicyId: currentId, RelatedEntityIds: result.PostureCheckIds, RelatedEntityType: edge_ctrl_pb.ServicePolicyRelatedEntityType_RelatedPostureCheck, Add: true, } strategy.HandleServicePolicyChange(addPostureChecksEvent) } return nil } func (strategy *InstantStrategy) BuildPublicKeys(tx *bbolt.Tx) error { for cursor := strategy.ae.GetStores().Controller.IterateIds(tx, ast.BoolNodeTrue); cursor.IsValid(); cursor.Next() { currentBytes := cursor.Current() currentId := string(currentBytes) storeModel, err := strategy.ae.GetStores().Controller.LoadById(tx, currentId) if err != nil { return err } certs := nfPem.PemStringToCertificates(storeModel.CertPem) newModel := &edge_ctrl_pb.DataState_Event_PublicKey{PublicKey: newPublicKey(certs[0].Raw, edge_ctrl_pb.DataState_PublicKey_X509CertDer, []edge_ctrl_pb.DataState_PublicKey_Usage{edge_ctrl_pb.DataState_PublicKey_JWTValidation, edge_ctrl_pb.DataState_PublicKey_ClientX509CertValidation})} newEvent := &edge_ctrl_pb.DataState_Event{ Action: edge_ctrl_pb.DataState_Create, Model: newModel, } strategy.HandlePublicKeyEvent(newEvent, newModel) } caPEMs := strategy.ae.Config.CaPems() caCerts := nfPem.PemBytesToCertificates(caPEMs) for _, caCert := range caCerts { publicKey := newPublicKey(caCert.Raw, edge_ctrl_pb.DataState_PublicKey_X509CertDer, []edge_ctrl_pb.DataState_PublicKey_Usage{edge_ctrl_pb.DataState_PublicKey_JWTValidation, edge_ctrl_pb.DataState_PublicKey_ClientX509CertValidation}) newModel := &edge_ctrl_pb.DataState_Event_PublicKey{PublicKey: publicKey} newEvent := &edge_ctrl_pb.DataState_Event{ Action: edge_ctrl_pb.DataState_Create, Model: newModel, } strategy.HandlePublicKeyEvent(newEvent, newModel) } for cursor := strategy.ae.GetStores().Ca.IterateIds(tx, ast.BoolNodeTrue); cursor.IsValid(); cursor.Next() { currentBytes := cursor.Current() currentId := string(currentBytes) ca, err := strategy.ae.GetStores().Ca.LoadById(tx, currentId) if err != nil { return err } certs := nfPem.PemStringToCertificates(ca.CertPem) if len(certs) == 0 { continue } publicKey := newPublicKey(certs[0].Raw, edge_ctrl_pb.DataState_PublicKey_X509CertDer, []edge_ctrl_pb.DataState_PublicKey_Usage{edge_ctrl_pb.DataState_PublicKey_ClientX509CertValidation}) newModel := &edge_ctrl_pb.DataState_Event_PublicKey{PublicKey: publicKey} newEvent := &edge_ctrl_pb.DataState_Event{ Action: edge_ctrl_pb.DataState_Create, Model: newModel, } strategy.HandlePublicKeyEvent(newEvent, newModel) } return nil } func (strategy *InstantStrategy) BuildAll() error { err := strategy.ae.GetDbProvider().GetDb().View(func(tx *bbolt.Tx) error { if err := strategy.BuildIdentities(tx); err != nil { return err } if err := strategy.BuildServices(tx); err != nil { return err } if err := strategy.BuildPostureChecks(tx); err != nil { return err } if err := strategy.BuildServicePolicies(tx); err != nil { return err } if err := strategy.BuildPublicKeys(tx); err != nil { return err } strategy.SetCurrentIndex(strategy.indexProvider.CurrentIndex()) return nil }) return err } func (strategy *InstantStrategy) BuildIdentities(tx *bbolt.Tx) error { for cursor := strategy.ae.GetStores().Identity.IterateIds(tx, ast.BoolNodeTrue); cursor.IsValid(); cursor.Next() { currentBytes := cursor.Current() currentId := string(currentBytes) identity, err := newIdentityById(tx, strategy.ae, currentId) if err != nil { return err } newModel := &edge_ctrl_pb.DataState_Event_Identity{Identity: identity} newEvent := &edge_ctrl_pb.DataState_Event{ Action: edge_ctrl_pb.DataState_Create, Model: newModel, } strategy.HandleIdentityEvent(newEvent, newModel) } return nil } func (strategy *InstantStrategy) BuildServices(tx *bbolt.Tx) error { for cursor := strategy.ae.GetStores().EdgeService.IterateIds(tx, ast.BoolNodeTrue); cursor.IsValid(); cursor.Next() { currentBytes := cursor.Current() currentId := string(currentBytes) service, err := newServiceById(tx, strategy.ae, currentId) if err != nil { return err } newModel := &edge_ctrl_pb.DataState_Event_Service{Service: service} newEvent := &edge_ctrl_pb.DataState_Event{ Action: edge_ctrl_pb.DataState_Create, Model: newModel, } strategy.HandleServiceEvent(newEvent, newModel) } return nil } func (strategy *InstantStrategy) BuildPostureChecks(tx *bbolt.Tx) error { for cursor := strategy.ae.GetStores().PostureCheck.IterateIds(tx, ast.BoolNodeTrue); cursor.IsValid(); cursor.Next() { currentBytes := cursor.Current() currentId := string(currentBytes) postureCheck, err := newPostureCheckById(tx, strategy.ae, currentId) if err != nil { return err } newModel := &edge_ctrl_pb.DataState_Event_PostureCheck{PostureCheck: postureCheck} newEvent := &edge_ctrl_pb.DataState_Event{ Action: edge_ctrl_pb.DataState_Create, Model: newModel, } strategy.HandlePostureCheckEvent(newEvent, newModel) } return nil } func newIdentityById(tx *bbolt.Tx, ae *env.AppEnv, id string) (*edge_ctrl_pb.DataState_Identity, error) { identityModel, err := ae.GetStores().Identity.LoadById(tx, id) if err != nil { return nil, err } return newIdentity(identityModel), nil } func newIdentity(identityModel *db.Identity) *edge_ctrl_pb.DataState_Identity { var hostingPrecedences map[string]edge_ctrl_pb.TerminatorPrecedence if identityModel.ServiceHostingPrecedences != nil { hostingPrecedences = map[string]edge_ctrl_pb.TerminatorPrecedence{} for k, v := range identityModel.ServiceHostingPrecedences { hostingPrecedences[k] = edge_ctrl_pb.GetPrecedence(v) } } var hostingCosts map[string]uint32 if identityModel.ServiceHostingCosts != nil { hostingCosts = map[string]uint32{} for k, v := range identityModel.ServiceHostingCosts { hostingCosts[k] = uint32(v) } } var appDataJson []byte if identityModel.AppData != nil { var err error appDataJson, err = json.Marshal(identityModel.AppData) if err != nil { pfxlog.Logger().WithError(err).Error("Failed to marshal app data") } } return &edge_ctrl_pb.DataState_Identity{ Id: identityModel.Id, Name: identityModel.Name, DefaultHostingPrecedence: edge_ctrl_pb.GetPrecedence(identityModel.DefaultHostingPrecedence), DefaultHostingCost: uint32(identityModel.DefaultHostingCost), ServiceHostingPrecedences: hostingPrecedences, ServiceHostingCosts: hostingCosts, AppDataJson: appDataJson, } } func newServicePolicy(storeModel *db.ServicePolicy) *edge_ctrl_pb.DataState_ServicePolicy { servicePolicy := &edge_ctrl_pb.DataState_ServicePolicy{ Id: storeModel.Id, Name: storeModel.Name, PolicyType: edge_ctrl_pb.PolicyType(storeModel.PolicyType.Id()), } return servicePolicy } func newServiceById(tx *bbolt.Tx, ae *env.AppEnv, id string) (*edge_ctrl_pb.DataState_Service, error) { storeModel, err := ae.GetStores().EdgeService.LoadById(tx, id) if err != nil { return nil, err } return newService(storeModel), nil } func newService(storeModel *db.EdgeService) *edge_ctrl_pb.DataState_Service { return &edge_ctrl_pb.DataState_Service{ Id: storeModel.Id, Name: storeModel.Name, } } func newPublicKey(data []byte, format edge_ctrl_pb.DataState_PublicKey_Format, usages []edge_ctrl_pb.DataState_PublicKey_Usage) *edge_ctrl_pb.DataState_PublicKey { return &edge_ctrl_pb.DataState_PublicKey{ Data: data, Kid: fmt.Sprintf("%x", sha1.Sum(data)), Usages: usages, Format: format, } } func newPostureCheckById(tx *bbolt.Tx, ae *env.AppEnv, id string) (*edge_ctrl_pb.DataState_PostureCheck, error) { postureModel, err := ae.GetStores().PostureCheck.LoadById(tx, id) if err != nil { return nil, err } return newPostureCheck(postureModel), nil } func newPostureCheck(postureModel *db.PostureCheck) *edge_ctrl_pb.DataState_PostureCheck { newVal := &edge_ctrl_pb.DataState_PostureCheck{ Id: postureModel.Id, Name: postureModel.Name, TypeId: postureModel.TypeId, } switch subType := postureModel.SubType.(type) { case *db.PostureCheckProcess: newVal.Subtype = &edge_ctrl_pb.DataState_PostureCheck_Process_{ Process: &edge_ctrl_pb.DataState_PostureCheck_Process{ OsType: subType.OperatingSystem, Path: subType.Path, Hashes: subType.Hashes, Fingerprints: []string{subType.Fingerprint}, }, } case *db.PostureCheckProcessMulti: processList := &edge_ctrl_pb.DataState_PostureCheck_ProcessMulti_{ ProcessMulti: &edge_ctrl_pb.DataState_PostureCheck_ProcessMulti{ Semantic: subType.Semantic, }, } for _, process := range subType.Processes { newProc := &edge_ctrl_pb.DataState_PostureCheck_Process{ OsType: process.OsType, Path: process.Path, Hashes: process.Hashes, Fingerprints: process.SignerFingerprints, } processList.ProcessMulti.Processes = append(processList.ProcessMulti.Processes, newProc) } newVal.Subtype = processList case *db.PostureCheckMfa: newVal.Subtype = &edge_ctrl_pb.DataState_PostureCheck_Mfa_{ Mfa: &edge_ctrl_pb.DataState_PostureCheck_Mfa{ TimeoutSeconds: subType.TimeoutSeconds, PromptOnWake: subType.PromptOnWake, PromptOnUnlock: subType.PromptOnUnlock, IgnoreLegacyEndpoints: subType.IgnoreLegacyEndpoints, }, } case *db.PostureCheckWindowsDomains: newVal.Subtype = &edge_ctrl_pb.DataState_PostureCheck_Domains_{ Domains: &edge_ctrl_pb.DataState_PostureCheck_Domains{ Domains: subType.Domains, }, } case *db.PostureCheckMacAddresses: newVal.Subtype = &edge_ctrl_pb.DataState_PostureCheck_Mac_{ Mac: &edge_ctrl_pb.DataState_PostureCheck_Mac{ MacAddresses: subType.MacAddresses, }, } case *db.PostureCheckOperatingSystem: osList := &edge_ctrl_pb.DataState_PostureCheck_OsList{} for _, os := range subType.OperatingSystems { newOs := &edge_ctrl_pb.DataState_PostureCheck_Os{ OsType: os.OsType, OsVersions: os.OsVersions, } osList.OsList = append(osList.OsList, newOs) } newVal.Subtype = &edge_ctrl_pb.DataState_PostureCheck_OsList_{ OsList: osList, } } return newVal } func (strategy *InstantStrategy) ServicePolicyCreate(index uint64, servicePolicy *db.ServicePolicy) { strategy.handleServicePolicy(index, edge_ctrl_pb.DataState_Create, servicePolicy) } func (strategy *InstantStrategy) ServicePolicyUpdate(index uint64, servicePolicy *db.ServicePolicy) { strategy.handleServicePolicy(index, edge_ctrl_pb.DataState_Update, servicePolicy) } func (strategy *InstantStrategy) ServicePolicyDelete(index uint64, servicePolicy *db.ServicePolicy) { strategy.handleServicePolicy(index, edge_ctrl_pb.DataState_Delete, servicePolicy) } func (strategy *InstantStrategy) handleServicePolicy(index uint64, action edge_ctrl_pb.DataState_Action, servicePolicy *db.ServicePolicy) { sp := newServicePolicy(servicePolicy) strategy.addToChangeSet(index, &edge_ctrl_pb.DataState_Event{ Action: action, Model: &edge_ctrl_pb.DataState_Event_ServicePolicy{ ServicePolicy: sp, }, }) } func (strategy *InstantStrategy) IdentityCreate(index uint64, identity *db.Identity) { strategy.handleIdentity(index, edge_ctrl_pb.DataState_Create, identity) } func (strategy *InstantStrategy) IdentityUpdate(index uint64, identity *db.Identity) { strategy.handleIdentity(index, edge_ctrl_pb.DataState_Update, identity) } func (strategy *InstantStrategy) IdentityDelete(index uint64, identity *db.Identity) { strategy.handleIdentity(index, edge_ctrl_pb.DataState_Delete, identity) } func (strategy *InstantStrategy) handleIdentity(index uint64, action edge_ctrl_pb.DataState_Action, identity *db.Identity) { id := newIdentity(identity) strategy.addToChangeSet(index, &edge_ctrl_pb.DataState_Event{ Action: action, Model: &edge_ctrl_pb.DataState_Event_Identity{ Identity: id, }, }) } func (strategy *InstantStrategy) ServiceCreate(index uint64, service *db.EdgeService) { strategy.handleService(index, edge_ctrl_pb.DataState_Create, service) } func (strategy *InstantStrategy) ServiceUpdate(index uint64, service *db.EdgeService) { strategy.handleService(index, edge_ctrl_pb.DataState_Update, service) } func (strategy *InstantStrategy) ServiceDelete(index uint64, service *db.EdgeService) { strategy.handleService(index, edge_ctrl_pb.DataState_Delete, service) } func (strategy *InstantStrategy) handleService(index uint64, action edge_ctrl_pb.DataState_Action, service *db.EdgeService) { svc := newService(service) strategy.addToChangeSet(index, &edge_ctrl_pb.DataState_Event{ Action: action, Model: &edge_ctrl_pb.DataState_Event_Service{ Service: svc, }, }) } func (strategy *InstantStrategy) handlePostureCheck(index uint64, action edge_ctrl_pb.DataState_Action, postureCheck *db.PostureCheck) { pc := newPostureCheck(postureCheck) strategy.addToChangeSet(index, &edge_ctrl_pb.DataState_Event{ Action: action, Model: &edge_ctrl_pb.DataState_Event_PostureCheck{ PostureCheck: pc, }, }) } func (strategy *InstantStrategy) PostureCheckCreate(index uint64, postureCheck *db.PostureCheck) { strategy.handlePostureCheck(index, edge_ctrl_pb.DataState_Create, postureCheck) } func (strategy *InstantStrategy) PostureCheckUpdate(index uint64, postureCheck *db.PostureCheck) { strategy.handlePostureCheck(index, edge_ctrl_pb.DataState_Update, postureCheck) } func (strategy *InstantStrategy) PostureCheckDelete(index uint64, postureCheck *db.PostureCheck) { strategy.handlePostureCheck(index, edge_ctrl_pb.DataState_Delete, postureCheck) } func (strategy *InstantStrategy) ControllerCreate(index uint64, controller *db.Controller) { certs := nfPem.PemStringToCertificates(controller.CertPem) cert := certs[0] strategy.handlePublicKey(index, edge_ctrl_pb.DataState_Create, newPublicKey(cert.Raw, edge_ctrl_pb.DataState_PublicKey_X509CertDer, []edge_ctrl_pb.DataState_PublicKey_Usage{edge_ctrl_pb.DataState_PublicKey_ClientX509CertValidation, edge_ctrl_pb.DataState_PublicKey_JWTValidation})) } func (strategy *InstantStrategy) ControllerUpdate(index uint64, controller *db.Controller) { certs := nfPem.PemStringToCertificates(controller.CertPem) cert := certs[0] strategy.handlePublicKey(index, edge_ctrl_pb.DataState_Create, newPublicKey(cert.Raw, edge_ctrl_pb.DataState_PublicKey_X509CertDer, []edge_ctrl_pb.DataState_PublicKey_Usage{edge_ctrl_pb.DataState_PublicKey_ClientX509CertValidation, edge_ctrl_pb.DataState_PublicKey_JWTValidation})) } func (strategy *InstantStrategy) CaCreate(index uint64, ca *db.Ca) { certs := nfPem.PemBytesToCertificates([]byte(ca.CertPem)) if len(certs) > 0 { strategy.handlePublicKey(index, edge_ctrl_pb.DataState_Create, newPublicKey(certs[0].Raw, edge_ctrl_pb.DataState_PublicKey_X509CertDer, []edge_ctrl_pb.DataState_PublicKey_Usage{edge_ctrl_pb.DataState_PublicKey_ClientX509CertValidation})) } } func (strategy *InstantStrategy) CaUpdate(index uint64, ca *db.Ca) { certs := nfPem.PemBytesToCertificates([]byte(ca.CertPem)) if len(certs) > 0 { strategy.handlePublicKey(index, edge_ctrl_pb.DataState_Update, newPublicKey(certs[0].Raw, edge_ctrl_pb.DataState_PublicKey_X509CertDer, []edge_ctrl_pb.DataState_PublicKey_Usage{edge_ctrl_pb.DataState_PublicKey_ClientX509CertValidation})) } } func (strategy *InstantStrategy) CaDelete(index uint64, ca *db.Ca) { certs := nfPem.PemBytesToCertificates([]byte(ca.CertPem)) if len(certs) > 0 { strategy.handlePublicKey(index, edge_ctrl_pb.DataState_Delete, newPublicKey(certs[0].Raw, edge_ctrl_pb.DataState_PublicKey_X509CertDer, []edge_ctrl_pb.DataState_PublicKey_Usage{edge_ctrl_pb.DataState_PublicKey_ClientX509CertValidation})) } } func (strategy *InstantStrategy) RevocationCreate(index uint64, revocation *db.Revocation) { strategy.handleRevocation(index, edge_ctrl_pb.DataState_Create, revocation) } func (strategy *InstantStrategy) RevocationUpdate(index uint64, revocation *db.Revocation) { strategy.handleRevocation(index, edge_ctrl_pb.DataState_Create, revocation) } func (strategy *InstantStrategy) RevocationDelete(index uint64, revocation *db.Revocation) { strategy.handleRevocation(index, edge_ctrl_pb.DataState_Create, revocation) } func (strategy *InstantStrategy) handlePublicKey(index uint64, action edge_ctrl_pb.DataState_Action, publicKey *edge_ctrl_pb.DataState_PublicKey) { strategy.addToChangeSet(index, &edge_ctrl_pb.DataState_Event{ Action: action, Model: &edge_ctrl_pb.DataState_Event_PublicKey{ PublicKey: publicKey, }, }) } func (strategy *InstantStrategy) sendDataStateChangeSet(rtx *RouterSender, stateEvent *edge_ctrl_pb.DataState_ChangeSet) error { content, err := proto.Marshal(stateEvent) if err != nil { return err } msg := channel.NewMessage(env.DataStateChangeSetType, content) return rtx.Send(msg) } func (strategy *InstantStrategy) sendDataState(rtx *RouterSender, state *edge_ctrl_pb.DataState) error { content, err := proto.Marshal(state) if err != nil { return err } msg := channel.NewMessage(env.DataStateType, content) return rtx.Send(msg) } func (strategy *InstantStrategy) handleRevocation(index uint64, action edge_ctrl_pb.DataState_Action, revocation *db.Revocation) { strategy.addToChangeSet(index, &edge_ctrl_pb.DataState_Event{ Action: action, Model: &edge_ctrl_pb.DataState_Event_Revocation{ Revocation: &edge_ctrl_pb.DataState_Revocation{ Id: revocation.Id, ExpiresAt: timestamppb.New(revocation.ExpiresAt), }, }, }) } func (strategy *InstantStrategy) addToChangeSet(index uint64, event *edge_ctrl_pb.DataState_Event) { strategy.changeSetLock.Lock() defer strategy.changeSetLock.Unlock() changeSet, found := strategy.changeSets[index] if !found { changeSet = &edge_ctrl_pb.DataState_ChangeSet{ Index: index, } strategy.changeSets[index] = changeSet } changeSet.Changes = append(changeSet.Changes, event) } func (strategy *InstantStrategy) completeChangeSet(ctx boltz.MutateContext) { strategy.changeSetLock.Lock() defer strategy.changeSetLock.Unlock() indexPtr := strategy.indexProvider.ContextIndex(ctx) if indexPtr == nil { return } index := *indexPtr changeSet := strategy.changeSets[index] for k := range strategy.changeSets { if k <= index { delete(strategy.changeSets, k) } } v := ctx.Context().Value(db.ServicePolicyEventsKey) if v != nil { policyEvents := v.([]*edge_ctrl_pb.DataState_ServicePolicyChange) if len(policyEvents) > 0 { if changeSet == nil { changeSet = &edge_ctrl_pb.DataState_ChangeSet{ Index: index, } } for _, policyEvent := range policyEvents { changeSet.Changes = append(changeSet.Changes, &edge_ctrl_pb.DataState_Event{ Action: 0, Model: &edge_ctrl_pb.DataState_Event_ServicePolicyChange{ ServicePolicyChange: policyEvent, }, }) } } } if changeSet != nil { strategy.ApplyChangeSet(changeSet) } } func (strategy *InstantStrategy) inspect(val string) (bool, *string, error) { if val == "router-data-model" { rdm := strategy.RouterDataModel js, err := json.Marshal(rdm) if err != nil { return true, nil, err } result := string(js) return true, &result, nil } return false, nil, nil } type IndexProvider interface { // NextIndex provides an index for the supplied MutateContext. NextIndex(ctx boltz.MutateContext) (uint64, error) // CurrentIndex provides the current index CurrentIndex() uint64 ContextIndex(ctx boltz.MutateContext) *uint64 } type nonHahIndexKeyType string const nonHaIndexKey = nonHahIndexKeyType("non-ha.index") type NonHaIndexProvider struct { ae *env.AppEnv initialLoad sync.Once index uint64 lock sync.Mutex } func (p *NonHaIndexProvider) load() { p.lock.Lock() defer p.lock.Unlock() ctx := boltz.NewMutateContext(context.Background()) err := p.ae.GetDbProvider().GetDb().Update(ctx, func(ctx boltz.MutateContext) error { zdb, err := ctx.Tx().CreateBucketIfNotExists([]byte(ZdbKey)) if err != nil { return err } indexBytes := zdb.Get([]byte(ZdbIndexKey)) if len(indexBytes) == 8 { p.index = binary.BigEndian.Uint64(indexBytes) } else { p.index = 0 indexBytes = make([]byte, 8) binary.BigEndian.PutUint64(indexBytes, p.index) _ = zdb.Put([]byte(ZdbIndexKey), indexBytes) } return nil }) if err != nil { pfxlog.Logger().WithError(err).Fatal("could not load initial index") } } func (p *NonHaIndexProvider) NextIndex(ctx boltz.MutateContext) (uint64, error) { p.initialLoad.Do(p.load) p.lock.Lock() defer p.lock.Unlock() if val := ctx.Context().Value(nonHaIndexKey); val != nil { return val.(uint64), nil } updateCtx := boltz.NewMutateContext(context.Background()) err := p.ae.GetDbProvider().GetDb().Update(updateCtx, func(updateCtx boltz.MutateContext) error { zdb := updateCtx.Tx().Bucket([]byte(ZdbKey)) newIndex := p.index + 1 indexBytes := make([]byte, 8) // Create a byte slice with 8 bytes binary.BigEndian.PutUint64(indexBytes, newIndex) err := zdb.Put([]byte(ZdbIndexKey), indexBytes) if err != nil { return err } p.index = newIndex return nil }) if err != nil { return 0, err } ctx.UpdateContext(func(ctx context.Context) context.Context { return context.WithValue(ctx, nonHaIndexKey, p.index) }) return p.index, nil } func (p *NonHaIndexProvider) CurrentIndex() uint64 { p.initialLoad.Do(p.load) p.lock.Lock() defer p.lock.Unlock() return p.index } func (p *NonHaIndexProvider) ContextIndex(ctx boltz.MutateContext) *uint64 { if val := ctx.Context().Value(nonHaIndexKey); val != nil { result, ok := val.(uint64) if ok { return &result } } return nil } type RaftIndexProvider struct { index uint64 lock sync.Mutex } func (p *RaftIndexProvider) NextIndex(ctx boltz.MutateContext) (uint64, error) { p.lock.Lock() defer p.lock.Unlock() changeCtx := change.FromContext(ctx.Context()) if changeCtx != nil { p.index = changeCtx.RaftIndex return changeCtx.RaftIndex, nil } return 0, errors.New("could not locate raft index from MutateContext") } func (p *RaftIndexProvider) CurrentIndex() uint64 { p.lock.Lock() defer p.lock.Unlock() return p.index } func (p *RaftIndexProvider) ContextIndex(ctx boltz.MutateContext) *uint64 { changeCtx := change.FromContext(ctx.Context()) if changeCtx != nil { return &changeCtx.RaftIndex } return nil } // constraintToIndexedEvents allows constraint events to be converted to events that provide the end state of an // entity and the index it was modified on. In non-HA scenarios, the index should be a locally tracked integer. // In HA scenarios, it should be the index from the event store. Constraint events are used as they provide // access to the HA raft index. type constraintToIndexedEvents[E boltz.Entity] struct { indexProvider IndexProvider createHandler func(uint64, E) updateHandler func(uint64, E) deleteHandler func(uint64, E) } // ProcessPreCommit is a pass through to satisfy interface requirements. func (h *constraintToIndexedEvents[E]) ProcessPreCommit(_ *boltz.EntityChangeState[E]) error { return nil } func (h *constraintToIndexedEvents[E]) ProcessPostCommit(state *boltz.EntityChangeState[E]) { switch state.ChangeType { case boltz.EntityCreated: if h.createHandler != nil { index, err := h.indexProvider.NextIndex(state.Ctx) if err != nil { pfxlog.Logger().WithError(err).Errorf("could not process post commit create for %T, could not acquire index", state.FinalState) return } h.createHandler(index, state.FinalState) } case boltz.EntityUpdated: if h.updateHandler != nil { index, err := h.indexProvider.NextIndex(state.Ctx) if err != nil { pfxlog.Logger().WithError(err).Errorf("could not process post commit update for %T, could not acquire index", state.FinalState) return } h.updateHandler(index, state.FinalState) } case boltz.EntityDeleted: if h.deleteHandler != nil { index, err := h.indexProvider.NextIndex(state.Ctx) if err != nil { pfxlog.Logger().WithError(err).Errorf("could not process post commit delete for %T, could not acquire index", state.FinalState) return } //initial state for delete has the actual value, final state is nil h.deleteHandler(index, state.InitialState) } } }