Merge pull request #1867 from openziti/release-next

Release 0.34.0

.github/dependabot.yml

@@ -1,12 +1,39 @@
 version: 2
 updates:
-  - package-ecosystem: gomod
-    directory: "/"
-    schedule:
-      interval: daily
-    open-pull-requests-limit: 10
-  - package-ecosystem: github-actions
-    directory: "/"
-    schedule:
-      interval: weekly
-    open-pull-requests-limit: 10
+- package-ecosystem: gomod
+  directory: "/"
+  schedule:
+    interval: daily
+  open-pull-requests-limit: 10
+  groups:
+    openziti:
+      applies-to: version-updates
+      update-types:
+        - "minor"
+        - "patch"
+      patterns:
+        - "openziti"
+
+    third-party:
+      applies-to: version-updates
+      update-types:
+        - "minor"
+        - "patch"
+      exclude-patterns:
+        - "openziti"
+        - "rabbitmq"
+        - "blackfriday"
+        - "zitadel"
+
+- package-ecosystem: github-actions
+  directory: "/"
+  schedule:
+    interval: weekly
+  open-pull-requests-limit: 10
+  groups:
+    all:
+      applies-to: version-updates
+      update-types:
+        - "major"
+        - "minor"
+        - "patch"

ADOPTERS.md

@@ -22,6 +22,7 @@ open source that have embedded OpenZiti into the project, or use OpenZiti for se
 | Project Name | Project Link    | Source URL | Logo                                                                                           | How OpenZiti is used or why OpenZiti was chosen                                                                          |
 |-------------|-----------------|------------|------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------|
 | zrok        | https://zrok.io | https://github.com/openziti/zrok           | <img src="https://zrok.io/wp-content/uploads/2023/01/space3-1327x1536.png.webp" width="100px"> | An open source sharing solution built on OpenZiti, the zero trust networking platform. Available as SaaS or self-hosted. |
+| BlueBubbles  | https://bluebubbles.app  | https://github.com/BlueBubblesApp | <img src="https://raw.githubusercontent.com/BlueBubblesApp/bluebubbles-server/master/icons/regular/icon-256.png" width="100px"> | BlueBubbles is an open-source app ecosystem dedicated to bringing iMessage to Android, Windows, and Linux. Shock your iPhone friends by iMessaging them from nearly any platform!  |
 
 
 ## Other Adopters

CHANGELOG.md

@@ -1,3 +1,34 @@
+# Release 0.34.0
+
+## What's New
+
+* Bug fixes and performance enhancements
+* Version number is bumped as a large chunk of HA was merged up. The next version bump is likely to bring HA to alpha status.
+
+## Component Updates and Bug Fixes
+
+* github.com/openziti/channel/v2: [v2.0.119 -> v2.0.122](https://github.com/openziti/channel/compare/v2.0.119...v2.0.122)
+* github.com/openziti/edge-api: [v0.26.12 -> v0.26.14](https://github.com/openziti/edge-api/compare/v0.26.12...v0.26.14)
+* github.com/openziti/foundation/v2: [v2.0.37 -> v2.0.40](https://github.com/openziti/foundation/compare/v2.0.37...v2.0.40)
+* github.com/openziti/identity: [v1.0.70 -> v1.0.73](https://github.com/openziti/identity/compare/v1.0.70...v1.0.73)
+* github.com/openziti/metrics: [v1.2.45 -> v1.2.48](https://github.com/openziti/metrics/compare/v1.2.45...v1.2.48)
+* github.com/openziti/runzmd: [v1.0.38 -> v1.0.41](https://github.com/openziti/runzmd/compare/v1.0.38...v1.0.41)
+* github.com/openziti/sdk-golang: [v0.22.28 -> v0.23.14](https://github.com/openziti/sdk-golang/compare/v0.22.28...v0.23.14)
+    * [Issue #524](https://github.com/openziti/sdk-golang/issues/524) - Add circuit id to edge.Conn, so sdk connections can be correlated with network traffic
+    * [Issue #515](https://github.com/openziti/sdk-golang/issues/515) - Service hosting improvements
+    * [Issue #501](https://github.com/openziti/sdk-golang/issues/501) - Improve hosting session management
+
+* github.com/openziti/secretstream: [v0.1.16 -> v0.1.17](https://github.com/openziti/secretstream/compare/v0.1.16...v0.1.17)
+* github.com/openziti/storage: [v0.2.30 -> v0.2.33](https://github.com/openziti/storage/compare/v0.2.30...v0.2.33)
+* github.com/openziti/transport/v2: [v2.0.122 -> v2.0.125](https://github.com/openziti/transport/compare/v2.0.122...v2.0.125)
+* github.com/openziti/ziti: [v0.33.1 -> v0.34.0](https://github.com/openziti/ziti/compare/v0.33.1...v0.34.0)
+    * [Issue #1858](https://github.com/openziti/ziti/issues/1858) - add option to create a generic env file instead of a BASH script
+    * [Issue #1428](https://github.com/openziti/ziti/issues/1428) - Investigate policy integrity performance
+    * [Issue #1854](https://github.com/openziti/ziti/issues/1854) - Controller can try to send unroute to router which has since disconnected, causing panic
+    * [Issue #1576](https://github.com/openziti/ziti/issues/1576) - Don't scan for posture checks if there are no posture checks 
+    * [Issue #1849](https://github.com/openziti/ziti/issues/1849) - Session Sync shouldn't be able to block the control channel
+    * [Issue #1846](https://github.com/openziti/ziti/issues/1846) - Looking up api session certs for api sessions is inefficient
+
 # Release 0.33.1
 
 ## What's New
@@ -8,7 +39,6 @@
 * github.com/openziti/ziti: [v0.33.0 -> v0.33.1](https://github.com/openziti/ziti/compare/v0.33.0...v0.33.1)
     * [Issue #1826](https://github.com/openziti/ziti/issues/1826) - 0.33.+ routers can cause panic in pre-0.33 controller with new metric
 
-
 # Release 0.33.0
 
 ## What's New

common/ctrl_msg/messages.go

@@ -87,12 +87,17 @@ func NewRouteResultFailedMessage(sessionId string, attempt int, rerr string) *ch
 }
 
 type CreateCircuitRequest struct {
+	ApiSessionToken      string
 	SessionToken         string
 	Fingerprints         []string
 	TerminatorInstanceId string
 	PeerData             map[uint32][]byte
 }
 
+func (self *CreateCircuitRequest) GetApiSessionToken() string {
+	return self.ApiSessionToken
+}
+
 func (self *CreateCircuitRequest) GetSessionToken() string {
 	return self.SessionToken
 }

common/event_cache.go

@@ -0,0 +1,240 @@
+package common
+
+import (
+	"fmt"
+	"github.com/openziti/ziti/common/pb/edge_ctrl_pb"
+	"sync"
+)
+
+type OnStoreSuccess func(index uint64, event *edge_ctrl_pb.DataState_Event)
+
+type EventCache interface {
+	// Store allows storage of an event and execution of an onSuccess callback while the event cache remains locked.
+	// onSuccess may be nil. This function is blocking.
+	Store(event *edge_ctrl_pb.DataState_Event, onSuccess OnStoreSuccess) error
+
+	// CurrentIndex returns the latest event index applied. This function is blocking.
+	CurrentIndex() (uint64, bool)
+
+	// ReplayFrom returns an array of events from startIndex and true if the replay may be facilitated.
+	// An empty slice and true is returned in cases where the requested startIndex is the current index.
+	// An empty slice and false is returned in cases where the replay cannot be facilitated.
+	// This function is blocking.
+	ReplayFrom(startIndex uint64) ([]*edge_ctrl_pb.DataState_Event, bool)
+
+	// WhileLocked allows the execution of arbitrary functionality while the event cache is locked. This function
+	// is blocking.
+	WhileLocked(func(uint64, bool))
+
+	// SetCurrentIndex sets the current index to the supplied value. All event log history may be lost.
+	SetCurrentIndex(uint64)
+}
+
+// ForgetfulEventCache does not store events or support replaying. It tracks
+// the event index and that is it. It is a stand in for LoggingEventCache
+// when replaying events is not expected (i.e. in routers)
+type ForgetfulEventCache struct {
+	lock  sync.Mutex
+	index *uint64
+}
+
+func NewForgetfulEventCache() *ForgetfulEventCache {
+	return &ForgetfulEventCache{}
+}
+
+func (cache *ForgetfulEventCache) SetCurrentIndex(index uint64) {
+	cache.lock.Lock()
+	defer cache.lock.Unlock()
+	cache.index = &index
+}
+
+func (cache *ForgetfulEventCache) WhileLocked(callback func(uint64, bool)) {
+	cache.lock.Lock()
+	defer cache.lock.Unlock()
+
+	callback(cache.currentIndex())
+}
+
+func (cache *ForgetfulEventCache) Store(event *edge_ctrl_pb.DataState_Event, onSuccess OnStoreSuccess) error {
+	cache.lock.Lock()
+	defer cache.lock.Unlock()
+
+	// Synthetic events are not backed by any kind of data store that provides and index. They are not stored and
+	// trigger the on success callback immediately.
+	if event.IsSynthetic {
+		onSuccess(event.Index, event)
+		return nil
+	}
+
+	if cache.index != nil {
+		if *cache.index >= event.Index {
+			return fmt.Errorf("out of order event detected, currentIndex: %d, recievedIndex: %d, type :%T", *cache.index, event.Index, cache)
+		}
+	}
+
+	cache.index = &event.Index
+
+	if onSuccess != nil {
+		onSuccess(*cache.index, event)
+	}
+
+	return nil
+}
+
+func (cache *ForgetfulEventCache) ReplayFrom(_ uint64) ([]*edge_ctrl_pb.DataState_Event, bool) {
+	return nil, false
+}
+
+func (cache *ForgetfulEventCache) CurrentIndex() (uint64, bool) {
+	cache.lock.Lock()
+	defer cache.lock.Unlock()
+
+	return cache.currentIndex()
+}
+
+func (cache *ForgetfulEventCache) currentIndex() (uint64, bool) {
+	if cache.index == nil {
+		return 0, false
+	}
+
+	return *cache.index, true
+}
+
+// LoggingEventCache stores events in order to support replaying (i.e. in controllers).
+type LoggingEventCache struct {
+	lock         sync.Mutex
+	HeadLogIndex uint64
+	LogSize      uint64
+	Log          []uint64
+	Events       map[uint64]*edge_ctrl_pb.DataState_Event
+}
+
+func NewLoggingEventCache(logSize uint64) *LoggingEventCache {
+	return &LoggingEventCache{
+		HeadLogIndex: 0,
+		LogSize:      logSize,
+		Log:          make([]uint64, logSize),
+		Events:       map[uint64]*edge_ctrl_pb.DataState_Event{},
+	}
+}
+
+func (cache *LoggingEventCache) SetCurrentIndex(index uint64) {
+	cache.lock.Lock()
+	defer cache.lock.Unlock()
+
+	cache.HeadLogIndex = 0
+	cache.Log = make([]uint64, cache.LogSize)
+	cache.Log[0] = index
+	cache.Events = map[uint64]*edge_ctrl_pb.DataState_Event{}
+}
+
+func (cache *LoggingEventCache) WhileLocked(callback func(uint64, bool)) {
+	cache.lock.Lock()
+	defer cache.lock.Unlock()
+
+	callback(cache.currentIndex())
+}
+
+func (cache *LoggingEventCache) Store(event *edge_ctrl_pb.DataState_Event, onSuccess OnStoreSuccess) error {
+	cache.lock.Lock()
+	defer cache.lock.Unlock()
+
+	// Synthetic events are not backed by any kind of data store that provides and index. They are not stored and
+	// trigger the on success callback immediately.
+	if event.IsSynthetic {
+		onSuccess(event.Index, event)
+		return nil
+	}
+
+	currentIndex, ok := cache.currentIndex()
+
+	if ok && currentIndex >= event.Index {
+		return fmt.Errorf("out of order event detected, currentIndex: %d, recievedIndex: %d, type :%T", currentIndex, event.Index, cache)
+	}
+
+	targetLogIndex := uint64(0)
+	targetLogIndex = (cache.HeadLogIndex + 1) % cache.LogSize
+
+	// delete old value if we have looped
+	prevKey := cache.Log[targetLogIndex]
+
+	if prevKey != 0 {
+		delete(cache.Events, prevKey)
+	}
+
+	// add new values
+	cache.Log[targetLogIndex] = event.Index
+	cache.Events[event.Index] = event
+
+	//update head
+	cache.HeadLogIndex = targetLogIndex
+
+	onSuccess(event.Index, event)
+	return nil
+}
+
+func (cache *LoggingEventCache) CurrentIndex() (uint64, bool) {
+	cache.lock.Lock()
+	defer cache.lock.Unlock()
+
+	return cache.currentIndex()
+}
+
+func (cache *LoggingEventCache) currentIndex() (uint64, bool) {
+	if len(cache.Log) == 0 {
+		return 0, false
+	}
+
+	return cache.Log[cache.HeadLogIndex], true
+}
+
+func (cache *LoggingEventCache) ReplayFrom(startIndex uint64) ([]*edge_ctrl_pb.DataState_Event, bool) {
+	cache.lock.Lock()
+	defer cache.lock.Unlock()
+
+	_, eventFound := cache.Events[startIndex]
+
+	if !eventFound {
+		return nil, false
+	}
+
+	var startLogIndex *uint64
+
+	for logIndex, eventIndex := range cache.Log {
+		if eventIndex == startIndex {
+			tmp := uint64(logIndex)
+			startLogIndex = &tmp
+			break
+		}
+	}
+
+	if startLogIndex == nil {
+		return nil, false
+	}
+
+	// no replay
+	if *startLogIndex == cache.HeadLogIndex {
+		return nil, true
+	}
+
+	// ez replay
+	if *startLogIndex < cache.HeadLogIndex {
+		var result []*edge_ctrl_pb.DataState_Event
+		for _, key := range cache.Log[*startLogIndex:cache.HeadLogIndex] {
+			result = append(result, cache.Events[key])
+		}
+		return result, true
+	}
+
+	//looping replay
+	var result []*edge_ctrl_pb.DataState_Event
+	for _, key := range cache.Log[*startLogIndex:] {
+		result = append(result, cache.Events[key])
+	}
+
+	for _, key := range cache.Log[0:cache.HeadLogIndex] {
+		result = append(result, cache.Events[key])
+	}
+
+	return result, true
+}

common/oidc_tokens.go

@@ -1,9 +1,10 @@
-package oidc_auth
+package common
 
 import (
 	"encoding/json"
 	"fmt"
 	"github.com/golang-jwt/jwt/v5"
+	"github.com/openziti/edge-api/rest_model"
 	"github.com/zitadel/oidc/v2/pkg/oidc"
 	"time"
 )
@@ -17,23 +18,51 @@ const (
 	CustomClaimsConfigTypes      = "z_ct"
 	CustomClaimsCertFingerprints = "z_cfs"
 
+	// CustomClaimsTokenType and other constants below may not appear as referenced, but are used in `json: ""` tags. Provided here for external use.
+	CustomClaimsTokenType    = "z_t"
+	CustomClaimServiceId     = "z_sid"
+	CustomClaimIdentityId    = "z_iid"
+	CustomClaimServiceType   = "z_st"
+	CustomClaimRemoteAddress = "z_ra"
+
 	DefaultAccessTokenDuration  = 30 * time.Minute
 	DefaultIdTokenDuration      = 30 * time.Minute
 	DefaultRefreshTokenDuration = 24 * time.Hour
 
-	TokenTypeAccess  = "a"
-	TokenTypeRefresh = "r"
+	TokenTypeAccess        = "a"
+	TokenTypeRefresh       = "r"
+	TokenTypeServiceAccess = "s"
 )
 
 type CustomClaims struct {
-	ApiSessionId     string   `json:"z_asid,omitempty"`
-	ExternalId       string   `json:"z_eid,omitempty"`
-	IsAdmin          bool     `json:"z_ia,omitempty"`
-	ConfigTypes      []string `json:"z_ct,omitempty"`
-	ApplicationId    string   `json:"z_aid,omitempty"`
-	Type             string   `json:"z_t"`
-	CertFingerprints []string `json:"z_cfs"`
-	Scopes           []string `json:"scopes,omitempty"`
+	ApiSessionId     string              `json:"z_asid,omitempty"`
+	ExternalId       string              `json:"z_eid,omitempty"`
+	IsAdmin          bool                `json:"z_ia,omitempty"`
+	ConfigTypes      []string            `json:"z_ct,omitempty"`
+	ApplicationId    string              `json:"z_aid,omitempty"`
+	Type             string              `json:"z_t"`
+	CertFingerprints []string            `json:"z_cfs"`
+	Scopes           []string            `json:"scopes,omitempty"`
+	SdkInfo          *rest_model.SdkInfo `json:"z_sdk"`
+	EnvInfo          *rest_model.EnvInfo `json:"z_env"`
+	RemoteAddress    string              `json:"z_ra"`
+}
+
+func (c *CustomClaims) ToMap() (map[string]any, error) {
+	out := map[string]any{}
+	str, err := json.Marshal(c)
+
+	if err != nil {
+		return nil, err
+	}
+
+	err = json.Unmarshal(str, &out)
+
+	if err != nil {
+		return nil, err
+	}
+
+	return out, nil
 }
 
 type RefreshClaims struct {
@@ -99,11 +128,49 @@ func (c *RefreshClaims) UnmarshalJSON(data []byte) error {
 	return nil
 }
 
+type ServiceAccessClaims struct {
+	jwt.RegisteredClaims
+	ApiSessionId string `json:"z_asid"`
+	IdentityId   string `json:"z_iid"`
+	TokenType    string `json:"z_t"`
+	Type         string `json:"z_st"`
+}
+
+func (c *ServiceAccessClaims) HasAudience(targetAud string) bool {
+	for _, aud := range c.Audience {
+		if aud == targetAud {
+			return true
+		}
+	}
+	return false
+}
+
 type AccessClaims struct {
 	oidc.AccessTokenClaims
 	CustomClaims
 }
 
+func (r *AccessClaims) ConfigTypesAsMap() map[string]struct{} {
+	result := map[string]struct{}{}
+
+	for _, configType := range r.ConfigTypes {
+		result[configType] = struct{}{}
+	}
+
+	return result
+}
+
+func (r *AccessClaims) UnmarshalJSON(raw []byte) error {
+	err := json.Unmarshal(raw, &r.AccessTokenClaims)
+	if err != nil {
+		return err
+	}
+
+	err = json.Unmarshal(raw, &r.CustomClaims)
+
+	return err
+}
+
 func (r *AccessClaims) GetExpirationTime() (*jwt.NumericDate, error) {
 	return &jwt.NumericDate{Time: r.TokenClaims.GetExpiration()}, nil
 }
@@ -139,6 +206,15 @@ func (c *AccessClaims) TotpComplete() bool {
 	return false
 }
 
+func (c *AccessClaims) HasAudience(targetAud string) bool {
+	for _, aud := range c.Audience {
+		if aud == targetAud {
+			return true
+		}
+	}
+	return false
+}
+
 type IdTokenClaims struct {
 	oidc.IDTokenClaims
 	CustomClaims

common/pb/edge_cmd_pb/edge_cmd.proto

@@ -165,6 +165,18 @@ message ConfigType {
   map<string, TagValue> tags = 4;
 }
 
+// Controllers
+message Controller {
+  string id = 1;
+  string name = 2;
+  string address = 3;
+  string certPem = 4;
+  string fingerprint = 5;
+  bool isOnline = 6;
+  google.protobuf.Timestamp lastJoinedAt = 7;
+  map<string, TagValue> tags = 8;
+}
+
 // Edge Routers
 message EdgeRouter {
   string id = 1;

common/pb/edge_ctrl_pb/README.md

@@ -1,6 +1,7 @@
 # Prerequisites
 
 1. Install the protoc binary from: https://github.com/protocolbuffers/protobuf/releases
+    1. The release should include an `include` folder that needs to be next to the `protoc` binary
 2. Install the protoc plugin for Go ```go install google.golang.org/protobuf/cmd/protoc-gen-go@latest```
 3. Ensure ```protoc``` is on your path.
 4. Ensure your Go bin directory is on your path
@@ -13,11 +14,11 @@ Two options, run the command manually or use `go generate`
 ## Go Generate
 
 1. Navigate to the root project directory `edge`
-2. run `go generate ./pb/edge_ctrl_pb/...` or  `go generate .\pb\edge_ctrl_pb\...`
+2. run `go generate ./common/pb/edge_ctrl_pb/...` or  `go generate \common\pb\edge_ctrl_pb\...`
 
 Note: Running a naked `go generate` will trigger all `go:generate` tags in the project, which you most likely do not want
 
 ## Manually
 
 1. Navigate to the project root
-2. Run: ```protoc -I ./pb/edge_ctrl_pb/ ./pb/edge_ctrl_pb/edge_ctrl.proto --go_out=./pb/edge_ctrl_pb```
+2. Run: ```protoc -I ./common/pb/edge_ctrl_pb/ ./common/pb/edge_ctrl_pb/edge_ctrl.proto --go_out=./pb/edge_ctrl_pb```

common/pb/edge_ctrl_pb/edge_ctrl.proto

@@ -2,7 +2,9 @@ syntax = "proto3";
 
 package ziti.edge_ctrl.pb;
 
-option go_package = "github.com/openziti/edge/pb/edge_ctrl_pb";
+import "google/protobuf/timestamp.proto";
+
+option go_package = "github.com/openziti/ziti/common/pb/edge_ctrl_pb";
 
 enum ContentType {
   Zero = 0;
@@ -27,7 +29,6 @@ enum ContentType {
   RemoveTerminatorResponseType = 20212;
   ValidateSessionsRequestType = 20213;
   HealthEventType = 20214;
-  SigningCertAddedType = 20215;
   CreateTerminatorV2RequestType = 20216;
   CreateTerminatorV2ResponseType = 20217;
   CreateCircuitV2RequestType = 20218;
@@ -54,6 +55,10 @@ enum ContentType {
   RemoveTunnelTerminatorResponseType = 20411;
 
   TunnelHealthEventType = 20412;
+
+  DataStateType = 20500;
+  DataStateEventType = 20501;
+  UpdateTokenType = 20502;
 }
 
 enum SessionType {
@@ -65,6 +70,8 @@ enum Header {
   HeaderZero = 0;
   TerminatorId = 1022;
   CheckPassed = 1023;
+  RouterDataModel = 1024;
+  RouterDataModelIndex = 1025;
 }
 
 message ServerHello {
@@ -115,44 +122,124 @@ message Service {
   bool encryptionRequired = 3;
 }
 
-enum ConfigType {
-  Add = 0;
-  Remove = 1;
-  Replace = 2;
-}
+message DataState {
+  repeated Event events = 1;
+  uint64 endIndex  = 2;
 
+  enum Action {
+    Create = 0;
+    Update = 1;
+    Delete = 2;
+  }
 
-message Config {
-  string syncId = 1;
-  string id = 2;
-  ConfigType type = 3;
-  repeated bytes signerKeys = 4;
-  repeated Identity identities = 5;
-  repeated Service services = 6;
-  repeated PostureSet postureSet = 7;
-  repeated PostureCheck postureChecks = 8;
-}
+  message Identity {
+    string id = 1;
+    string name = 2;
+    repeated string servicePolicyIds = 3;
+  }
 
-message SignerCerts {
-  repeated bytes keys = 1;
-}
+  message Service {
+    string id = 1;
+    string name = 2;
+    bool encryptionRequired = 3;
+  }
 
-message Identity {
-  string id = 1;
-  string name = 2;
-  repeated string serviceIds = 3;
-}
+  message ServicePolicy {
+    string id = 1;
+    string name = 2;
+    repeated string serviceIds = 3;
+    repeated string postureCheckIds =4;
+    repeated string identityIds = 5;
+  }
 
-message PostureSet {
-  string id = 1;
-  string name = 2;
-  repeated string postureCheckIds = 3;
-}
+  message Revocation {
+    string id = 1;
+    google.protobuf.Timestamp ExpiresAt = 2;
+  }
 
-message PostureCheck {
-  string id = 1;
-  string type = 2;
-  map<string, bytes> data = 3;
+  message Event {
+    Action action = 1;
+    uint64 index = 2;
+    bool isSynthetic = 3;
+
+    oneof Model{
+      Identity identity = 10;
+      Service service = 11;
+      ServicePolicy servicePolicy = 12;
+      PostureCheck postureCheck = 13;
+      PublicKey publicKey = 14;
+      Revocation revocation = 15;
+    }
+  }
+
+  message PublicKey {
+    bytes data = 1; //public key
+    string kid = 2; //key id/fingerprint
+    repeated Usage usages = 3; // what the public key in data is used for
+    Format format = 4; //the format of the public key in data and chain
+
+    enum Usage {
+      JWTValidation = 0;
+      ClientX509CertValidation = 1;
+    }
+
+    enum Format {
+      X509CertDer = 0; //x509 cert
+      PKIXPublicKey = 1; //asn.1 public key
+      //JWK = 3; //JSON Web Key
+    }
+  }
+
+  message PostureCheck {
+    message Mac {
+      repeated string macAddresses = 1;
+    }
+
+    message Mfa {
+      int64 TimeoutSeconds = 1;
+      bool PromptOnWake = 2;
+      bool PromptOnUnlock = 3;
+      bool IgnoreLegacyEndpoints = 4;
+    }
+
+    message Os {
+      string OsType = 1;
+      repeated string OsVersions = 2;
+    }
+
+    message OsList {
+      repeated Os osList = 1;
+    }
+
+    message Process {
+      string OsType = 1;
+      string Path = 2;
+      repeated string Hashes = 3;
+      repeated string Fingerprints = 4;
+    }
+
+    message ProcessMulti {
+      string semantic = 1;
+      repeated Process processes = 2;
+    }
+
+    message Domains {
+      repeated string domains = 1;
+    }
+
+    string id = 1;
+    string name = 2;
+    string typeId = 4;
+
+    oneof subtype {
+      Mac mac = 7;
+      Mfa mfa = 8;
+      OsList osList = 9;
+      Process process = 10;
+      ProcessMulti processMulti = 11;
+      Domains domains = 12;
+    };
+  }
 }
 
 message Session {
@@ -203,6 +290,7 @@ message CreateCircuitRequest {
   repeated string fingerprints = 2;
   string terminatorInstanceId = 3;
   map<uint32, bytes> peerData = 4;
+  string apiSessionToken = 5;
 }
 
 message CreateCircuitResponse {
@@ -226,6 +314,7 @@ message CreateTerminatorRequest {
   TerminatorPrecedence precedence = 5;
   string instanceId = 6;
   bytes instanceSecret = 7;
+  string apiSessionToken = 8;
 }
 
 message CreateTerminatorV2Request {
@@ -237,6 +326,7 @@ message CreateTerminatorV2Request {
   TerminatorPrecedence precedence = 6;
   string instanceId = 7;
   bytes instanceSecret = 8;
+  string apiSessionToken = 9;
 }
 
 enum CreateTerminatorResult {
@@ -257,6 +347,7 @@ message RemoveTerminatorRequest {
   string sessionToken = 1;
   repeated string fingerprints = 2;
   string terminatorId = 3;
+  string apiSessionToken = 4;
 }
 
 message UpdateTerminatorRequest {
@@ -267,6 +358,7 @@ message UpdateTerminatorRequest {
   TerminatorPrecedence precedence = 5;
   bool updatePrecedence = 6;
   bool updateCost = 7;
+  string apiSessionToken = 8;
 }
 
 message HealthEventRequest {
@@ -274,6 +366,7 @@ message HealthEventRequest {
   repeated string fingerprints = 2;
   string terminatorId = 3;
   bool checkPassed = 4;
+  string apiSessionToken = 5;
 }
 
 message ValidateSessionsRequest {

common/pb/edge_ctrl_pb/headers.go

@@ -1,3 +0,0 @@
-package edge_ctrl_pb
-
-const SignerPublicCertsHeader = "signerPublicKeys"

common/router_data_model.go

@@ -0,0 +1,586 @@
+package common
+
+import (
+	"compress/gzip"
+	"encoding/json"
+	"fmt"
+	"github.com/michaelquigley/pfxlog"
+	"github.com/openziti/foundation/v2/stringz"
+	"github.com/openziti/ziti/common/pb/edge_ctrl_pb"
+	cmap "github.com/orcaman/concurrent-map/v2"
+	"io"
+	"os"
+)
+
+// AccessPolicies represents the Identity's access to a Service through many Policies. The PostureChecks provided
+// are referenced by the granting Policies. The PostureChecks for each of the Policies may be evaluated to determine
+// a valid policy and posture access path.
+type AccessPolicies struct {
+	Identity      *edge_ctrl_pb.DataState_Identity
+	Service       *edge_ctrl_pb.DataState_Service
+	Policies      []*edge_ctrl_pb.DataState_ServicePolicy
+	PostureChecks map[string]*edge_ctrl_pb.DataState_PostureCheck
+}
+
+// RouterDataModel represents a sub-set of a controller's data model. Enough to validate an identities access to dial/bind
+// a service through policies and posture checks. RouterDataModel can operate in two modes: sender (controller) and
+// receiver (router). Sender mode allows a controller support an event cache that supports replays for routers connecting
+// for the first time/after disconnects. Receive mode does not maintain an event cache and does not support replays.
+// It instead is used as a reference data structure for authorization computations.
+type RouterDataModel struct {
+	EventCache
+	listeners map[chan *edge_ctrl_pb.DataState_Event]struct{}
+
+	Identities      cmap.ConcurrentMap[string, *edge_ctrl_pb.DataState_Identity]      `json:"identities"`
+	Services        cmap.ConcurrentMap[string, *edge_ctrl_pb.DataState_Service]       `json:"services"`
+	ServicePolicies cmap.ConcurrentMap[string, *edge_ctrl_pb.DataState_ServicePolicy] `json:"servicePolicies"`
+	PostureChecks   cmap.ConcurrentMap[string, *edge_ctrl_pb.DataState_PostureCheck]  `json:"postureChecks"`
+	PublicKeys      cmap.ConcurrentMap[string, *edge_ctrl_pb.DataState_PublicKey]     `json:"publicKeys"`
+	Revocations     cmap.ConcurrentMap[string, *edge_ctrl_pb.DataState_Revocation]    `json:"revocations"`
+
+	listenerBufferSize uint
+	lastSaveIndex      *uint64
+}
+
+// NewSenderRouterDataModel creates a new RouterDataModel that will store events in a circular buffer of
+// logSize. listenerBufferSize affects the buffer size of channels returned to listeners of the data model.
+func NewSenderRouterDataModel(logSize uint64, listenerBufferSize uint) *RouterDataModel {
+	return &RouterDataModel{
+		EventCache:         NewLoggingEventCache(logSize),
+		Identities:         cmap.New[*edge_ctrl_pb.DataState_Identity](),
+		Services:           cmap.New[*edge_ctrl_pb.DataState_Service](),
+		ServicePolicies:    cmap.New[*edge_ctrl_pb.DataState_ServicePolicy](),
+		PostureChecks:      cmap.New[*edge_ctrl_pb.DataState_PostureCheck](),
+		PublicKeys:         cmap.New[*edge_ctrl_pb.DataState_PublicKey](),
+		Revocations:        cmap.New[*edge_ctrl_pb.DataState_Revocation](),
+		listenerBufferSize: listenerBufferSize,
+	}
+}
+
+// NewReceiverRouterDataModel creates a new RouterDataModel that does not store events. listenerBufferSize affects the
+// buffer size of channels returned to listeners of the data model.
+func NewReceiverRouterDataModel(listenerBufferSize uint) *RouterDataModel {
+	return &RouterDataModel{
+		EventCache:         NewForgetfulEventCache(),
+		Identities:         cmap.New[*edge_ctrl_pb.DataState_Identity](),
+		Services:           cmap.New[*edge_ctrl_pb.DataState_Service](),
+		ServicePolicies:    cmap.New[*edge_ctrl_pb.DataState_ServicePolicy](),
+		PostureChecks:      cmap.New[*edge_ctrl_pb.DataState_PostureCheck](),
+		PublicKeys:         cmap.New[*edge_ctrl_pb.DataState_PublicKey](),
+		Revocations:        cmap.New[*edge_ctrl_pb.DataState_Revocation](),
+		listenerBufferSize: listenerBufferSize,
+	}
+}
+
+// NewReceiverRouterDataModelFromFile creates a new RouterDataModel that does not store events and is initialized from
+// a file backup. listenerBufferSize affects the buffer size of channels returned to listeners of the data model.
+func NewReceiverRouterDataModelFromFile(path string, listenerBufferSize uint) (*RouterDataModel, error) {
+	file, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer func() { _ = file.Close() }()
+
+	gz, err := gzip.NewReader(file)
+	if err != nil {
+		return nil, err
+	}
+	defer func() { _ = gz.Close() }()
+
+	data, err := io.ReadAll(gz)
+	if err != nil {
+		return nil, err
+	}
+
+	rdmContents := &rdmDb{
+		RouterDataModel: NewReceiverRouterDataModel(listenerBufferSize),
+	}
+
+	err = json.Unmarshal(data, rdmContents)
+	if err != nil {
+		return nil, err
+	}
+
+	rdmContents.RouterDataModel.lastSaveIndex = &rdmContents.Index
+
+	return rdmContents.RouterDataModel, nil
+}
+
+// NewListener returns a channel that will receive the events applied to this data model.
+func (rdm *RouterDataModel) NewListener() <-chan *edge_ctrl_pb.DataState_Event {
+	if rdm.listeners == nil {
+		rdm.listeners = map[chan *edge_ctrl_pb.DataState_Event]struct{}{}
+	}
+
+	newCh := make(chan *edge_ctrl_pb.DataState_Event, rdm.listenerBufferSize)
+	rdm.listeners[newCh] = struct{}{}
+
+	return newCh
+}
+
+func (rdm *RouterDataModel) sendEvent(event *edge_ctrl_pb.DataState_Event) {
+	for listener := range rdm.listeners {
+		listener <- event
+	}
+}
+
+// Apply applies the given even to the router data model.
+func (rdm *RouterDataModel) Apply(event *edge_ctrl_pb.DataState_Event) {
+	switch typedModel := event.Model.(type) {
+	case *edge_ctrl_pb.DataState_Event_Identity:
+		rdm.ApplyIdentityEvent(event, typedModel)
+	case *edge_ctrl_pb.DataState_Event_Service:
+		rdm.ApplyServiceEvent(event, typedModel)
+	case *edge_ctrl_pb.DataState_Event_ServicePolicy:
+		rdm.ApplyServicePolicyEvent(event, typedModel)
+	case *edge_ctrl_pb.DataState_Event_PostureCheck:
+		rdm.ApplyPostureCheckEvent(event, typedModel)
+	case *edge_ctrl_pb.DataState_Event_PublicKey:
+		rdm.ApplyPublicKeyEvent(event, typedModel)
+	case *edge_ctrl_pb.DataState_Event_Revocation:
+		rdm.ApplyRevocationEvent(event, typedModel)
+	}
+}
+
+func (rdm *RouterDataModel) Handle(event *edge_ctrl_pb.DataState_Event) {
+	switch typedModel := event.Model.(type) {
+	case *edge_ctrl_pb.DataState_Event_Identity:
+		rdm.HandleIdentityEvent(event, typedModel)
+	case *edge_ctrl_pb.DataState_Event_Service:
+		rdm.HandleServiceEvent(event, typedModel)
+	case *edge_ctrl_pb.DataState_Event_ServicePolicy:
+		rdm.HandleServicePolicyEvent(event, typedModel)
+	case *edge_ctrl_pb.DataState_Event_PostureCheck:
+		rdm.HandlePostureCheckEvent(event, typedModel)
+	case *edge_ctrl_pb.DataState_Event_PublicKey:
+		rdm.HandlePublicKeyEvent(event, typedModel)
+	case *edge_ctrl_pb.DataState_Event_Revocation:
+		rdm.HandleRevocationEvent(event, typedModel)
+	}
+}
+
+// HandleIdentityEvent will apply the delta event to the router data model. It is not restricted by index calculations.
+// Use ApplyIdentityEvent for event logged event handling. This method is generally meant for bulk loading of data
+// during startup.
+func (rdm *RouterDataModel) HandleIdentityEvent(event *edge_ctrl_pb.DataState_Event, model *edge_ctrl_pb.DataState_Event_Identity) {
+	if event.Action == edge_ctrl_pb.DataState_Delete {
+		rdm.Identities.Remove(model.Identity.Id)
+
+		rdm.ServicePolicies.IterCb(func(servicePolicyId string, servicePolicy *edge_ctrl_pb.DataState_ServicePolicy) {
+			servicePolicy.IdentityIds = stringz.Remove(servicePolicy.IdentityIds, model.Identity.Id)
+		})
+	} else {
+		rdm.Identities.Upsert(model.Identity.Id, nil, func(exist bool, valueInMap *edge_ctrl_pb.DataState_Identity, newValue *edge_ctrl_pb.DataState_Identity) *edge_ctrl_pb.DataState_Identity {
+			if !exist {
+				return model.Identity
+			}
+
+			valueInMap.Name = model.Identity.Name
+
+			return valueInMap
+		})
+	}
+}
+
+func (rdm *RouterDataModel) ApplyIdentityEvent(event *edge_ctrl_pb.DataState_Event, model *edge_ctrl_pb.DataState_Event_Identity) {
+	err := rdm.EventCache.Store(event, func(index uint64, event *edge_ctrl_pb.DataState_Event) {
+		rdm.HandleIdentityEvent(event, model)
+	})
+
+	if err != nil {
+		pfxlog.Logger().WithError(err).
+			WithFields(map[string]interface{}{
+				"event": event,
+			}).
+			Error("could not store identity event")
+		return
+	}
+
+	rdm.sendEvent(event)
+}
+
+// HandleServiceEvent will apply the delta event to the router data model. It is not restricted by index calculations.
+// Use ApplyServiceEvent for event logged event handling. This method is generally meant for bulk loading of data
+// during startup.
+func (rdm *RouterDataModel) HandleServiceEvent(event *edge_ctrl_pb.DataState_Event, model *edge_ctrl_pb.DataState_Event_Service) {
+	if event.Action == edge_ctrl_pb.DataState_Delete {
+		rdm.Services.Remove(model.Service.Id)
+	} else {
+		rdm.Services.Set(model.Service.Id, model.Service)
+	}
+}
+
+func (rdm *RouterDataModel) ApplyServiceEvent(event *edge_ctrl_pb.DataState_Event, model *edge_ctrl_pb.DataState_Event_Service) {
+	err := rdm.EventCache.Store(event, func(index uint64, event *edge_ctrl_pb.DataState_Event) {
+		rdm.HandleServiceEvent(event, model)
+	})
+
+	if err != nil {
+		pfxlog.Logger().WithError(err).
+			WithFields(map[string]interface{}{
+				"event": event,
+			}).
+			Error("could not store service event")
+		return
+	}
+
+	rdm.sendEvent(event)
+}
+
+func (rdm *RouterDataModel) applyCreateServicePolicyEvent(_ *edge_ctrl_pb.DataState_Event, model *edge_ctrl_pb.DataState_Event_ServicePolicy) {
+	servicePolicy := model.ServicePolicy
+	rdm.ServicePolicies.Set(servicePolicy.Id, servicePolicy)
+
+	for _, identityId := range servicePolicy.IdentityIds {
+		rdm.Identities.Upsert(identityId, nil, func(exist bool, valueInMap *edge_ctrl_pb.DataState_Identity, newValue *edge_ctrl_pb.DataState_Identity) *edge_ctrl_pb.DataState_Identity {
+			if !exist {
+				return &edge_ctrl_pb.DataState_Identity{
+					Id:               identityId,
+					Name:             "UNKNOWN",
+					ServicePolicyIds: []string{servicePolicy.Id},
+				}
+			}
+
+			if !stringz.Contains(valueInMap.ServicePolicyIds, servicePolicy.Id) {
+				valueInMap.ServicePolicyIds = append(valueInMap.ServicePolicyIds, servicePolicy.Id)
+			}
+
+			return valueInMap
+		})
+	}
+}
+
+func (rdm *RouterDataModel) applyUpdateServicePolicyEvent(event *edge_ctrl_pb.DataState_Event, model *edge_ctrl_pb.DataState_Event_ServicePolicy) {
+	oldPolicy, ok := rdm.ServicePolicies.Get(model.ServicePolicy.Id)
+
+	if !ok {
+		rdm.applyCreateServicePolicyEvent(event, model)
+		return
+	}
+
+	removeFromIdentities := stringz.Difference(oldPolicy.IdentityIds, model.ServicePolicy.IdentityIds)
+	addToIdentities := stringz.Difference(model.ServicePolicy.IdentityIds, oldPolicy.IdentityIds)
+
+	for _, identityId := range removeFromIdentities {
+		rdm.Identities.Upsert(identityId, nil, func(exist bool, valueInMap *edge_ctrl_pb.DataState_Identity, newValue *edge_ctrl_pb.DataState_Identity) *edge_ctrl_pb.DataState_Identity {
+			if !exist {
+				return &edge_ctrl_pb.DataState_Identity{
+					Id: identityId,
+				}
+			}
+			valueInMap.ServicePolicyIds = stringz.Remove(valueInMap.ServicePolicyIds, model.ServicePolicy.Id)
+
+			return valueInMap
+		})
+	}
+
+	for _, identityId := range addToIdentities {
+		rdm.Identities.Upsert(identityId, nil, func(exist bool, valueInMap *edge_ctrl_pb.DataState_Identity, newValue *edge_ctrl_pb.DataState_Identity) *edge_ctrl_pb.DataState_Identity {
+			if !exist {
+				return &edge_ctrl_pb.DataState_Identity{
+					Id:               identityId,
+					ServicePolicyIds: []string{model.ServicePolicy.Id},
+				}
+			}
+
+			if !stringz.Contains(valueInMap.ServicePolicyIds, model.ServicePolicy.Id) {
+				valueInMap.ServicePolicyIds = append(valueInMap.ServicePolicyIds, model.ServicePolicy.Id)
+			}
+
+			return valueInMap
+		})
+	}
+}
+
+func (rdm *RouterDataModel) applyDeleteServicePolicyEvent(_ *edge_ctrl_pb.DataState_Event, model *edge_ctrl_pb.DataState_Event_ServicePolicy) {
+	for _, identityId := range model.ServicePolicy.IdentityIds {
+		if identity, ok := rdm.Identities.Get(identityId); ok {
+			identity.ServicePolicyIds = stringz.Remove(identity.ServicePolicyIds, model.ServicePolicy.Id)
+		}
+	}
+
+	rdm.ServicePolicies.Remove(model.ServicePolicy.Id)
+}
+
+// HandleServicePolicyEvent will apply the delta event to the router data model. It is not restricted by index calculations.
+// Use ApplyServicePolicyEvent for event logged event handling. This method is generally meant for bulk loading of data
+// during startup.
+func (rdm *RouterDataModel) HandleServicePolicyEvent(event *edge_ctrl_pb.DataState_Event, model *edge_ctrl_pb.DataState_Event_ServicePolicy) {
+	switch event.Action {
+	case edge_ctrl_pb.DataState_Create:
+		rdm.applyCreateServicePolicyEvent(event, model)
+	case edge_ctrl_pb.DataState_Update:
+		rdm.applyUpdateServicePolicyEvent(event, model)
+	case edge_ctrl_pb.DataState_Delete:
+		rdm.applyDeleteServicePolicyEvent(event, model)
+	}
+}
+
+func (rdm *RouterDataModel) ApplyServicePolicyEvent(event *edge_ctrl_pb.DataState_Event, model *edge_ctrl_pb.DataState_Event_ServicePolicy) {
+	err := rdm.EventCache.Store(event, func(index uint64, event *edge_ctrl_pb.DataState_Event) {
+		rdm.HandleServicePolicyEvent(event, model)
+	})
+
+	if err != nil {
+		pfxlog.Logger().WithError(err).
+			WithFields(map[string]interface{}{
+				"event": event,
+			}).
+			Error("could not store service policy event")
+		return
+	}
+
+	rdm.sendEvent(event)
+}
+
+// HandlePostureCheckEvent will apply the delta event to the router data model. It is not restricted by index calculations.
+// Use ApplyPostureCheckEvent for event logged event handling. This method is generally meant for bulk loading of data
+// during startup.
+func (rdm *RouterDataModel) HandlePostureCheckEvent(event *edge_ctrl_pb.DataState_Event, model *edge_ctrl_pb.DataState_Event_PostureCheck) {
+	if event.Action == edge_ctrl_pb.DataState_Delete {
+		rdm.PostureChecks.Remove(model.PostureCheck.Id)
+	} else {
+		rdm.PostureChecks.Set(model.PostureCheck.Id, model.PostureCheck)
+	}
+}
+
+func (rdm *RouterDataModel) ApplyPostureCheckEvent(event *edge_ctrl_pb.DataState_Event, model *edge_ctrl_pb.DataState_Event_PostureCheck) {
+	err := rdm.EventCache.Store(event, func(index uint64, event *edge_ctrl_pb.DataState_Event) {
+		rdm.HandlePostureCheckEvent(event, model)
+	})
+
+	if err != nil {
+		pfxlog.Logger().WithError(err).
+			WithFields(map[string]interface{}{
+				"event": event,
+			}).
+			Error("could not store posture check event")
+		return
+	}
+
+	rdm.sendEvent(event)
+}
+
+// HandlePublicKeyEvent will apply the delta event to the router data model. It is not restricted by index calculations.
+// Use ApplyPublicKeyEvent for event logged event handling. This method is generally meant for bulk loading of data
+// during startup.
+func (rdm *RouterDataModel) HandlePublicKeyEvent(event *edge_ctrl_pb.DataState_Event, model *edge_ctrl_pb.DataState_Event_PublicKey) {
+	if event.Action == edge_ctrl_pb.DataState_Delete {
+		rdm.PublicKeys.Remove(model.PublicKey.Kid)
+	} else {
+		rdm.PublicKeys.Set(model.PublicKey.Kid, model.PublicKey)
+	}
+}
+
+func (rdm *RouterDataModel) ApplyPublicKeyEvent(event *edge_ctrl_pb.DataState_Event, model *edge_ctrl_pb.DataState_Event_PublicKey) {
+	err := rdm.EventCache.Store(event, func(index uint64, event *edge_ctrl_pb.DataState_Event) {
+		rdm.HandlePublicKeyEvent(event, model)
+	})
+
+	if err != nil {
+		pfxlog.Logger().WithError(err).
+			WithFields(map[string]interface{}{
+				"event": event,
+			}).
+			Error("could not store public key event")
+		return
+	}
+
+	rdm.sendEvent(event)
+}
+
+// HandleRevocationEvent will apply the delta event to the router data model. It is not restricted by index calculations.
+// Use ApplyRevocationEvent for event logged event handling. This method is generally meant for bulk loading of data
+// during startup.
+func (rdm *RouterDataModel) HandleRevocationEvent(event *edge_ctrl_pb.DataState_Event, model *edge_ctrl_pb.DataState_Event_Revocation) {
+	if event.Action == edge_ctrl_pb.DataState_Delete {
+		rdm.Revocations.Remove(model.Revocation.Id)
+	} else {
+		rdm.Revocations.Set(model.Revocation.Id, model.Revocation)
+	}
+}
+
+func (rdm *RouterDataModel) ApplyRevocationEvent(event *edge_ctrl_pb.DataState_Event, model *edge_ctrl_pb.DataState_Event_Revocation) {
+	err := rdm.EventCache.Store(event, func(index uint64, event *edge_ctrl_pb.DataState_Event) {
+		rdm.HandleRevocationEvent(event, model)
+	})
+
+	if err != nil {
+		pfxlog.Logger().WithError(err).
+			WithFields(map[string]interface{}{
+				"event": event,
+			}).
+			Error("could not store revocation event")
+		return
+	}
+
+	rdm.sendEvent(event)
+}
+
+func (rdm *RouterDataModel) GetDataState() *edge_ctrl_pb.DataState {
+	var events []*edge_ctrl_pb.DataState_Event
+
+	rdm.EventCache.WhileLocked(func(_ uint64, _ bool) {
+		identityBuffer := rdm.Identities.IterBuffered()
+		serviceBuffer := rdm.Services.IterBuffered()
+		ServicePoliciesBuffer := rdm.ServicePolicies.IterBuffered()
+		postureCheckBuffer := rdm.PostureChecks.IterBuffered()
+		publicKeysBuffer := rdm.PublicKeys.IterBuffered()
+
+		for entry := range identityBuffer {
+			newEvent := &edge_ctrl_pb.DataState_Event{
+				Action: edge_ctrl_pb.DataState_Create,
+				Model: &edge_ctrl_pb.DataState_Event_Identity{
+					Identity: entry.Val,
+				},
+			}
+			events = append(events, newEvent)
+		}
+
+		for entry := range serviceBuffer {
+			newEvent := &edge_ctrl_pb.DataState_Event{
+				Action: edge_ctrl_pb.DataState_Create,
+				Model: &edge_ctrl_pb.DataState_Event_Service{
+					Service: entry.Val,
+				},
+			}
+			events = append(events, newEvent)
+		}
+
+		for entry := range postureCheckBuffer {
+			newEvent := &edge_ctrl_pb.DataState_Event{
+				Action: edge_ctrl_pb.DataState_Create,
+				Model: &edge_ctrl_pb.DataState_Event_PostureCheck{
+					PostureCheck: entry.Val,
+				},
+			}
+			events = append(events, newEvent)
+		}
+
+		for entry := range ServicePoliciesBuffer {
+			newEvent := &edge_ctrl_pb.DataState_Event{
+				Action: edge_ctrl_pb.DataState_Create,
+				Model: &edge_ctrl_pb.DataState_Event_ServicePolicy{
+					ServicePolicy: entry.Val,
+				},
+			}
+			events = append(events, newEvent)
+		}
+
+		for entry := range publicKeysBuffer {
+			newEvent := &edge_ctrl_pb.DataState_Event{
+				Action: edge_ctrl_pb.DataState_Create,
+				Model: &edge_ctrl_pb.DataState_Event_PublicKey{
+					PublicKey: entry.Val,
+				},
+			}
+			events = append(events, newEvent)
+		}
+	})
+
+	return &edge_ctrl_pb.DataState{
+		Events: events,
+	}
+}
+
+// rdmDb is a helper structure of serializing router data models to JSON gzipped files.
+type rdmDb struct {
+	RouterDataModel *RouterDataModel `json:"model"`
+	Index           uint64           `json:"index"`
+}
+
+func (rdm *RouterDataModel) Save(path string) {
+	rdm.EventCache.WhileLocked(func(index uint64, indexInitialized bool) {
+		if !indexInitialized {
+			pfxlog.Logger().Debug("could not save router data model, no index")
+			return
+		}
+
+		//nothing to save
+		if rdm.lastSaveIndex != nil && *rdm.lastSaveIndex == index {
+			pfxlog.Logger().Debug("no changes to router model, nothing to save")
+			return
+		}
+
+		rdm.lastSaveIndex = &index
+
+		rdmFile := rdmDb{
+			RouterDataModel: rdm,
+			Index:           index,
+		}
+
+		jsonBytes, err := json.Marshal(rdmFile)
+
+		if err != nil {
+			pfxlog.Logger().WithError(err).Error("could not marshal router data model")
+			return
+		}
+
+		// Create a new gzip file
+		file, err := os.Create(path)
+		if err != nil {
+			pfxlog.Logger().WithError(err).Error("could not marshal router data model, could not create file")
+			return
+		}
+		defer func() { _ = file.Close() }()
+
+		// Create a gzip writer
+		gz := gzip.NewWriter(file)
+		defer func() { _ = gz.Close() }()
+
+		// Write the gzipped JSON data to the file
+		_, err = gz.Write(jsonBytes)
+
+		if err != nil {
+			pfxlog.Logger().WithError(err).Error("could not marshal router data model, could not compress and write")
+			return
+		}
+	})
+}
+
+// GetServiceAccessPolicies returns an AccessPolicies instance for an identity attempting to access a service.
+func (rdm *RouterDataModel) GetServiceAccessPolicies(identityId string, serviceId string) (*AccessPolicies, error) {
+	identity, ok := rdm.Identities.Get(identityId)
+
+	if !ok {
+		return nil, fmt.Errorf("identity not foud by id")
+	}
+
+	service, ok := rdm.Services.Get(serviceId)
+
+	if !ok {
+		return nil, fmt.Errorf("service not found by id")
+	}
+
+	var policies []*edge_ctrl_pb.DataState_ServicePolicy
+
+	postureChecks := map[string]*edge_ctrl_pb.DataState_PostureCheck{}
+
+	for _, servicePolicyId := range identity.ServicePolicyIds {
+		servicePolicy, ok := rdm.ServicePolicies.Get(servicePolicyId)
+
+		if !ok {
+			continue
+		}
+
+		if stringz.Contains(servicePolicy.IdentityIds, identityId) {
+			policies = append(policies, servicePolicy)
+
+			for _, postureCheckId := range servicePolicy.PostureCheckIds {
+				if _, ok := postureChecks[postureCheckId]; !ok {
+					//ignore ok, if !ok postureCheck == nil which will trigger
+					//failure during evaluation
+					postureCheck, _ := rdm.PostureChecks.Get(postureCheckId)
+					postureChecks[postureCheckId] = postureCheck
+				}
+			}
+		}
+	}
+
+	return &AccessPolicies{
+		Identity:      identity,
+		Service:       service,
+		Policies:      policies,
+		PostureChecks: postureChecks,
+	}, nil
+}

common/version/info_generated.go

@@ -2,7 +2,7 @@
 
 package version
 
-const (
+var (
 	Version   = "v0.0.0"
 	Revision  = "local"
 	Branch    = "local"

controller/apierror/helpers.go

@@ -304,6 +304,14 @@ func NewMfaExistsError() *errorz.ApiError {
 	}
 }
 
+func NewMfaEnrollmentNotStarted() *errorz.ApiError {
+	return &errorz.ApiError{
+		Code:    MfaEnrollmentNotStartedCode,
+		Message: MfaEnrollmentNotStartedMessage,
+		Status:  MfaEnrollmentNotStartedStatus,
+	}
+}
+
 func NewMfaNotEnrolledError() *errorz.ApiError {
 	return &errorz.ApiError{
 		Code:    MfaNotEnrolledCode,

controller/apierror/messages.go

@@ -154,6 +154,10 @@ const (
 	MfaExistsMessage string = "An MFA record already exists, try removing it"
 	MfaExistsStatus  int    = http.StatusConflict
 
+	MfaEnrollmentNotStartedCode    = "MFA_ENROLLMENT_NOT_STARTED"
+	MfaEnrollmentNotStartedMessage = "MFA enrollment has not be started"
+	MfaEnrollmentNotStartedStatus  = http.StatusNotFound
+
 	/* #nosec */
 	MfaInvalidTokenCode    string = "MFA_INVALID_TOKEN"
 	MfaInvalidTokenMessage string = "An invalid token/code was provided"

controller/config/config.go

@@ -495,7 +495,7 @@ func CalculateCaPems(caPems *bytes.Buffer) *bytes.Buffer {
 			if block.Type != "CERTIFICATE" {
 				pfxlog.Logger().
 					WithField("type", block.Type).
-					WithField("block", pem.EncodeToMemory(block)).
+					WithField("block", string(pem.EncodeToMemory(block))).
 					Warn("encountered an invalid PEM block type loading configured CAs, block will be ignored")
 				continue
 			}
@@ -505,7 +505,7 @@ func CalculateCaPems(caPems *bytes.Buffer) *bytes.Buffer {
 			if err != nil {
 				pfxlog.Logger().
 					WithField("type", block.Type).
-					WithField("block", pem.EncodeToMemory(block)).
+					WithField("block", string(pem.EncodeToMemory(block))).
 					WithError(err).
 					Warn("block could not be parsed as a certificate, block will be ignored")
 				continue
@@ -514,7 +514,7 @@ func CalculateCaPems(caPems *bytes.Buffer) *bytes.Buffer {
 			if !cert.IsCA {
 				pfxlog.Logger().
 					WithField("type", block.Type).
-					WithField("block", pem.EncodeToMemory(block)).
+					WithField("block", string(pem.EncodeToMemory(block))).
 					Warn("block is not a CA, block will be ignored")
 				continue
 			}

controller/controller.go

@@ -101,6 +101,21 @@ func (c *Controller) GetPeerSigners() []*x509.Certificate {
 	return certs
 }
 
+func (c *Controller) GetPeerAddresses() []string {
+	if c.raftController == nil || c.raftController.Mesh == nil {
+		return nil
+	}
+
+	var addresses []string
+
+	for _, peer := range c.raftController.Mesh.GetPeers() {
+		addresses = append(addresses, peer.Address)
+	}
+
+	return addresses
+
+}
+
 func (c *Controller) GetId() *identity.TokenId {
 	return c.config.Id
 }
@@ -133,6 +148,12 @@ func (c *Controller) IsRaftEnabled() bool {
 	return c.raftController != nil
 }
 
+func (c *Controller) IsRaftLeader() bool { return c.raftController.IsLeader() }
+
+func (c *Controller) GetRaftIndex() uint64 {
+	return c.raftController.Raft.LastIndex()
+}
+
 func (c *Controller) GetDb() boltz.Db {
 	return c.config.Db
 }

controller/db/api_session_certificate_store.go

@@ -69,11 +69,10 @@ func (store *ApiSessionCertificateStoreImpl) initializeLocal() {
 	store.AddSymbol(FieldApiSessionCertificateSubject, ast.NodeTypeString)
 	store.AddSymbol(FieldApiSessionCertificateFingerprint, ast.NodeTypeString)
 	store.symbolApiSession = store.AddFkSymbol(FieldApiSessionCertificateApiSession, store.stores.apiSession)
-
-	store.AddFkConstraint(store.symbolApiSession, false, boltz.CascadeDelete)
 }
 
 func (store *ApiSessionCertificateStoreImpl) initializeLinked() {
+	store.AddFkIndex(store.symbolApiSession, store.stores.apiSession.apiSessionCertsSymbol)
 }
 
 func (store *ApiSessionCertificateStoreImpl) NewEntity() *ApiSessionCertificate {

controller/db/api_session_store.go

@@ -92,9 +92,10 @@ func newApiSessionStore(stores *stores) *apiSessionStoreImpl {
 type apiSessionStoreImpl struct {
 	*baseStore[*ApiSession]
 
-	indexToken     boltz.ReadIndex
-	symbolIdentity boltz.EntitySymbol
-	eventsEmitter  events.EventEmmiter
+	indexToken            boltz.ReadIndex
+	symbolIdentity        boltz.EntitySymbol
+	eventsEmitter         events.EventEmmiter
+	apiSessionCertsSymbol boltz.EntitySetSymbol
 }
 
 func (store *apiSessionStoreImpl) NewEntity() *ApiSession {
@@ -220,6 +221,12 @@ func (store *apiSessionStoreImpl) Update(ctx boltz.MutateContext, entity *ApiSes
 }
 
 func (store *apiSessionStoreImpl) DeleteById(ctx boltz.MutateContext, id string) error {
+	for _, apiSessionCertId := range store.GetRelatedEntitiesIdList(ctx.Tx(), id, EntityTypeApiSessionCertificates) {
+		if err := store.stores.apiSessionCertificate.DeleteById(ctx, apiSessionCertId); err != nil {
+			return err
+		}
+	}
+
 	err := store.baseStore.DeleteById(ctx, id)
 
 	if err == nil {
@@ -249,6 +256,7 @@ func (store *apiSessionStoreImpl) initializeLocal() {
 	store.AddSymbol(FieldApiSessionLastActivityAt, ast.NodeTypeDatetime)
 
 	store.AddFkConstraint(store.symbolIdentity, false, boltz.CascadeDelete)
+	store.apiSessionCertsSymbol = store.AddFkSetSymbol(EntityTypeApiSessionCertificates, store.stores.apiSessionCertificate)
 }
 
 func (store *apiSessionStoreImpl) initializeLinked() {
@@ -257,7 +265,7 @@ func (store *apiSessionStoreImpl) initializeLinked() {
 func (store *apiSessionStoreImpl) LoadOneByToken(tx *bbolt.Tx, token string) (*ApiSession, error) {
 	id := store.indexToken.Read(tx, []byte(token))
 	if id != nil {
-		return store.LoadOneById(tx, string(id))
+		return store.LoadById(tx, string(id))
 	}
 	return nil, boltz.NewNotFoundError(store.GetSingularEntityType(), "token", token)
 }

controller/db/api_session_store_test.go

@@ -178,11 +178,11 @@ func (ctx *TestContext) testUpdateApiSessions(t *testing.T) {
 	mutateCtx := change.New().NewMutateContext()
 	err := ctx.GetDb().Update(mutateCtx, func(mutateCtx boltz.MutateContext) error {
 		tx := mutateCtx.Tx()
-		original, err := ctx.stores.ApiSession.LoadOneById(tx, entities.apiSession1.Id)
+		original, err := ctx.stores.ApiSession.LoadById(tx, entities.apiSession1.Id)
 		ctx.NoError(err)
 		ctx.NotNil(original)
 
-		apiSession, err := ctx.stores.ApiSession.LoadOneById(tx, entities.apiSession1.Id)
+		apiSession, err := ctx.stores.ApiSession.LoadById(tx, entities.apiSession1.Id)
 		ctx.NoError(err)
 		ctx.NotNil(apiSession)
 
@@ -196,7 +196,7 @@ func (ctx *TestContext) testUpdateApiSessions(t *testing.T) {
 
 		err = ctx.stores.ApiSession.Update(mutateCtx, apiSession, nil)
 		ctx.NoError(err)
-		loaded, err := ctx.stores.ApiSession.LoadOneById(tx, entities.apiSession1.Id)
+		loaded, err := ctx.stores.ApiSession.LoadById(tx, entities.apiSession1.Id)
 		ctx.NoError(err)
 		ctx.NotNil(loaded)
 		ctx.EqualValues(original.CreatedAt, loaded.CreatedAt)

controller/db/base_entity.go

@@ -29,6 +29,7 @@ const (
 	EntityTypeCas                       = "cas"
 	EntityTypeConfigs                   = "configs"
 	EntityTypeConfigTypes               = "configTypes"
+	EntityTypeControllers               = "controllers"
 	EntityTypeEdgeRouterPolicies        = "edgeRouterPolicies"
 	EntityTypeExternalJwtSigners        = "externalJwtSigners"
 	EntityTypeIdentities                = "identities"

controller/db/base_store.go

@@ -34,7 +34,6 @@ type initializableStore interface {
 type Store[E boltz.ExtEntity] interface {
 	boltz.EntityStore[E]
 	initializableStore
-	LoadOneById(tx *bbolt.Tx, id string) (E, error)
 }
 
 type baseStore[E boltz.ExtEntity] struct {
@@ -51,25 +50,6 @@ func (store *baseStore[E]) initializeIndexes(tx *bbolt.Tx, errorHolder errorz.Er
 	store.InitializeIndexes(tx, errorHolder)
 }
 
-func (store *baseStore[E]) LoadOneById(tx *bbolt.Tx, id string) (E, error) {
-	entity := store.NewStoreEntity()
-	if err := store.baseLoadOneById(tx, id, entity); err != nil {
-		return *new(E), err
-	}
-	return entity, nil
-}
-
-func (store *baseStore[E]) baseLoadOneById(tx *bbolt.Tx, id string, entity E) error {
-	found, err := store.LoadEntity(tx, id, entity)
-	if err != nil {
-		return err
-	}
-	if !found {
-		return boltz.NewNotFoundError(store.GetSingularEntityType(), "id", id)
-	}
-	return nil
-}
-
 func (store *baseStore[E]) deleteEntityReferences(tx *bbolt.Tx, entity boltz.NamedExtEntity, rolesSymbol boltz.EntitySetSymbol) error {
 	idRef := entityRef(entity.GetId())
 

controller/db/config_store_test.go

@@ -64,7 +64,7 @@ func (ctx *TestContext) testConfigCrud(*testing.T) {
 	boltztest.ValidateBaseline(ctx, config)
 
 	err = ctx.GetDb().View(func(tx *bbolt.Tx) error {
-		testConfig, err := ctx.stores.Config.LoadOneById(tx, config.Id)
+		testConfig, err := ctx.stores.Config.LoadById(tx, config.Id)
 		ctx.NoError(err)
 		ctx.NotNil(testConfig)
 		ctx.Equal(config.Name, testConfig.Name)

controller/db/config_type_store.go

@@ -120,7 +120,7 @@ func (store *configTypeStoreImpl) PersistEntity(entity *ConfigType, ctx *boltz.P
 func (store *configTypeStoreImpl) LoadOneByName(tx *bbolt.Tx, name string) (*ConfigType, error) {
 	id := store.indexName.Read(tx, []byte(name))
 	if id != nil {
-		return store.LoadOneById(tx, string(id))
+		return store.LoadById(tx, string(id))
 	}
 	return nil, nil
 }

controller/db/controller_store.go

@@ -0,0 +1,109 @@
+/*
+	Copyright NetFoundry Inc.
+
+	Licensed under the Apache License, Version 2.0 (the "License");
+	you may not use this file except in compliance with the License.
+	You may obtain a copy of the License at
+
+	https://www.apache.org/licenses/LICENSE-2.0
+
+	Unless required by applicable law or agreed to in writing, software
+	distributed under the License is distributed on an "AS IS" BASIS,
+	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+	See the License for the specific language governing permissions and
+	limitations under the License.
+*/
+
+package db
+
+import (
+	"github.com/openziti/storage/ast"
+	"github.com/openziti/storage/boltz"
+	"time"
+)
+
+const (
+	FieldControllerAddress      = "address"
+	FieldControllerCertPem      = "certPem"
+	FieldControllerFingerprint  = "fingerprint"
+	FieldControllerIsOnline     = "isOnline"
+	FieldControllerLastJoinedAt = "lastJoinedAt"
+)
+
+type Controller struct {
+	boltz.BaseExtEntity
+	Name         string     `json:"name"`
+	Address      string     `json:"address"`
+	CertPem      string     `json:"certPem"`
+	Fingerprint  string     `json:"fingerprint"`
+	IsOnline     bool       `json:"isOnline"`
+	LastJoinedAt *time.Time `json:"lastJoinedAt"`
+}
+
+func (entity *Controller) GetName() string {
+	return entity.Name
+}
+
+func (entity *Controller) GetEntityType() string {
+	return EntityTypeControllers
+}
+
+var _ ControllerStore = (*controllerStoreImpl)(nil)
+
+type ControllerStore interface {
+	Store[*Controller]
+	GetNameIndex() boltz.ReadIndex
+}
+
+func newControllerStore(stores *stores) *controllerStoreImpl {
+	store := &controllerStoreImpl{}
+	store.baseStore = newBaseStore[*Controller](stores, store)
+	store.InitImpl(store)
+	return store
+}
+
+type controllerStoreImpl struct {
+	*baseStore[*Controller]
+	indexName boltz.ReadIndex
+}
+
+func (store *controllerStoreImpl) GetNameIndex() boltz.ReadIndex {
+	return store.indexName
+}
+
+func (store *controllerStoreImpl) initializeLocal() {
+	store.AddExtEntitySymbols()
+	store.indexName = store.addUniqueNameField()
+
+	store.AddSymbol(FieldControllerAddress, ast.NodeTypeString)
+	store.AddSymbol(FieldControllerCertPem, ast.NodeTypeString)
+	store.AddSymbol(FieldControllerFingerprint, ast.NodeTypeString)
+	store.AddSymbol(FieldControllerIsOnline, ast.NodeTypeBool)
+	store.AddSymbol(FieldControllerLastJoinedAt, ast.NodeTypeDatetime)
+}
+
+func (store *controllerStoreImpl) initializeLinked() {}
+
+func (store *controllerStoreImpl) NewEntity() *Controller {
+	return &Controller{}
+}
+
+func (store *controllerStoreImpl) FillEntity(entity *Controller, bucket *boltz.TypedBucket) {
+	entity.LoadBaseValues(bucket)
+	entity.Name = bucket.GetStringOrError(FieldName)
+	entity.Address = bucket.GetStringWithDefault(FieldControllerAddress, "")
+	entity.CertPem = bucket.GetStringOrError(FieldControllerCertPem)
+	entity.Fingerprint = bucket.GetStringOrError(FieldControllerFingerprint)
+	entity.IsOnline = bucket.GetBoolWithDefault(FieldControllerIsOnline, false)
+	entity.LastJoinedAt = bucket.GetTime(FieldControllerLastJoinedAt)
+}
+
+func (store *controllerStoreImpl) PersistEntity(entity *Controller, ctx *boltz.PersistContext) {
+	entity.SetBaseValues(ctx)
+	ctx.SetString(FieldName, entity.Name)
+	ctx.SetString(FieldControllerAddress, entity.Address)
+	ctx.SetString(FieldControllerCertPem, entity.CertPem)
+	ctx.SetString(FieldControllerFingerprint, entity.Fingerprint)
+	ctx.SetBool(FieldControllerIsOnline, entity.IsOnline)
+	ctx.SetTimeP(FieldControllerLastJoinedAt, entity.LastJoinedAt)
+}

controller/db/edge_router_policy_store.go

@@ -164,7 +164,7 @@ func (store *edgeRouterPolicyStoreImpl) identityRolesUpdated(persistCtx *boltz.P
 }
 
 func (store *edgeRouterPolicyStoreImpl) DeleteById(ctx boltz.MutateContext, id string) error {
-	policy, err := store.LoadOneById(ctx.Tx(), id)
+	policy, err := store.LoadById(ctx.Tx(), id)
 	if err != nil {
 		return err
 	}

controller/db/edge_router_policy_store_test.go

@@ -38,7 +38,7 @@ func (ctx *TestContext) testCreateEdgeRouterPolicy(t *testing.T) {
 		ctx.Equal(0, len(ctx.stores.EdgeRouterPolicy.GetRelatedEntitiesIdList(tx, policy.Id, EntityTypeRouters)))
 		ctx.Equal(0, len(ctx.stores.EdgeRouterPolicy.GetRelatedEntitiesIdList(tx, policy.Id, EntityTypeIdentities)))
 
-		testPolicy, err := ctx.stores.EdgeRouterPolicy.LoadOneById(tx, policy.Id)
+		testPolicy, err := ctx.stores.EdgeRouterPolicy.LoadById(tx, policy.Id)
 		ctx.NoError(err)
 		ctx.NotNil(testPolicy)
 		ctx.Equal(policy.Name, testPolicy.Name)

controller/db/edge_router_store.go

@@ -231,7 +231,7 @@ func (store *edgeRouterStoreImpl) GetNameIndex() boltz.ReadIndex {
 }
 
 func (store *edgeRouterStoreImpl) cleanupEdgeRouter(ctx boltz.MutateContext, id string) error {
-	if entity, _ := store.LoadOneById(ctx.Tx(), id); entity != nil {
+	if entity, _ := store.LoadById(ctx.Tx(), id); entity != nil {
 		// Remove entity from EdgeRouterRoles in edge router policies
 		if err := store.deleteEntityReferences(ctx.Tx(), entity, store.stores.edgeRouterPolicy.symbolEdgeRouterRoles); err != nil {
 			return err
@@ -311,7 +311,7 @@ func (self *routerIdentityConstraint) ProcessAfterUpdate(ctx *boltz.IndexingCont
 	}
 
 	if !createEntities && currentTunnelerEnabled && oldName != name {
-		identity, err := self.stores.identity.LoadOneById(ctx.Tx(), routerId)
+		identity, err := self.stores.identity.LoadById(ctx.Tx(), routerId)
 		if err != nil {
 			if boltz.IsErrNotFoundErr(err) {
 				logrus.Errorf("identity for router with id %v not found", routerId)

controller/db/edge_service_store.go

@@ -251,7 +251,7 @@ func (store *edgeServiceStoreImpl) Update(ctx boltz.MutateContext, entity *EdgeS
 }
 
 func (store *edgeServiceStoreImpl) cleanupEdgeService(ctx boltz.MutateContext, id string) error {
-	if entity, _ := store.LoadOneById(ctx.Tx(), id); entity != nil {
+	if entity, _ := store.LoadById(ctx.Tx(), id); entity != nil {
 		// Remove entity from ServiceRoles in service policies
 		if err := store.deleteEntityReferences(ctx.Tx(), entity, store.stores.servicePolicy.symbolServiceRoles); err != nil {
 			return err

controller/db/edge_service_store_test.go

@@ -181,7 +181,7 @@ func (ctx *TestContext) testLoadQueryEdgeServices(_ *testing.T) {
 	entities := ctx.createEdgeServiceTestEntities()
 
 	err := ctx.GetDb().View(func(tx *bbolt.Tx) error {
-		service, err := ctx.stores.EdgeService.LoadOneById(tx, entities.service1.Id)
+		service, err := ctx.stores.EdgeService.LoadById(tx, entities.service1.Id)
 		ctx.NoError(err)
 		ctx.NotNil(service)
 		ctx.EqualValues(entities.service1.Id, service.Id)
@@ -217,11 +217,11 @@ func (ctx *TestContext) testUpdateEdgeServices(_ *testing.T) {
 	mutateCtx := change.New().NewMutateContext()
 	err := ctx.GetDb().Update(mutateCtx, func(mutateCtx boltz.MutateContext) error {
 		tx := mutateCtx.Tx()
-		original, err := ctx.stores.EdgeService.LoadOneById(tx, entities.service1.Id)
+		original, err := ctx.stores.EdgeService.LoadById(tx, entities.service1.Id)
 		ctx.NoError(err)
 		ctx.NotNil(original)
 
-		service, err := ctx.stores.EdgeService.LoadOneById(tx, entities.service1.Id)
+		service, err := ctx.stores.EdgeService.LoadById(tx, entities.service1.Id)
 		ctx.NoError(err)
 		ctx.NotNil(service)
 
@@ -234,7 +234,7 @@ func (ctx *TestContext) testUpdateEdgeServices(_ *testing.T) {
 
 		err = ctx.stores.EdgeService.Update(mutateCtx, service, nil)
 		ctx.NoError(err)
-		loaded, err := ctx.stores.EdgeService.LoadOneById(tx, entities.service1.Id)
+		loaded, err := ctx.stores.EdgeService.LoadById(tx, entities.service1.Id)
 		ctx.NoError(err)
 		ctx.NotNil(loaded)
 		ctx.EqualValues(original.CreatedAt, loaded.CreatedAt)

controller/db/enrollment_store.go

@@ -142,7 +142,7 @@ func (store *enrollmentStoreImpl) PersistEntity(entity *Enrollment, ctx *boltz.P
 func (store *enrollmentStoreImpl) LoadOneByToken(tx *bbolt.Tx, token string) (*Enrollment, error) {
 	id := store.tokenIndex.Read(tx, []byte(token))
 	if id != nil {
-		return store.LoadOneById(tx, string(id))
+		return store.LoadById(tx, string(id))
 	}
 	return nil, nil
 }

controller/db/eventual_eventer.go

@@ -489,7 +489,7 @@ func (a *EventualEventerBbolt) getEventualEvents() ([]string, []*EventualEvent,
 		}
 
 		for _, id := range ids {
-			event, err := a.store.LoadOneById(tx, id)
+			event, err := a.store.LoadById(tx, id)
 
 			if err != nil {
 				pfxlog.Logger().WithField("id", id).WithError(err).Errorf("error could not load event id %s", id)

controller/db/identity_store.go

@@ -400,7 +400,7 @@ func (store *identityStoreImpl) DeleteById(ctx boltz.MutateContext, id string) e
 		}
 	}
 
-	if entity, _ := store.LoadOneById(ctx.Tx(), id); entity != nil {
+	if entity, _ := store.LoadById(ctx.Tx(), id); entity != nil {
 		if entity.IsDefaultAdmin {
 			return errorz.NewEntityCanNotBeDeleted()
 		}
@@ -444,7 +444,7 @@ func (store *identityStoreImpl) AssignServiceConfigs(tx *bbolt.Tx, identityId st
 	}
 	configTypes := map[string]struct{}{}
 	for _, serviceConfig := range serviceConfigs {
-		config, err := store.stores.config.LoadOneById(tx, serviceConfig.ConfigId)
+		config, err := store.stores.config.LoadById(tx, serviceConfig.ConfigId)
 		if err != nil {
 			return err
 		}
@@ -625,7 +625,7 @@ func (store *identityStoreImpl) LoadServiceConfigsByServiceAndType(tx *bbolt.Tx,
 				_, wantsType = configTypes[configTypeId]
 			}
 			if wantsType {
-				if config, _ := store.stores.config.LoadOneById(tx, configId); config != nil {
+				if config, _ := store.stores.config.LoadById(tx, configId); config != nil {
 					serviceMap, ok := result[serviceId]
 					if !ok {
 						serviceMap = map[string]map[string]interface{}{}

controller/db/migration_v16.go

@@ -15,7 +15,7 @@ func (m *Migrations) removeOrphanedOttCaEnrollments(step *boltz.MigrationStep) {
 		current := cursor.Current()
 		currentEnrollmentId := string(current)
 
-		enrollment, err := m.stores.Enrollment.LoadOneById(step.Ctx.Tx(), currentEnrollmentId)
+		enrollment, err := m.stores.Enrollment.LoadById(step.Ctx.Tx(), currentEnrollmentId)
 
 		if err != nil {
 			step.SetError(fmt.Errorf("error iterating ids of enrollments, enrollment [%s]: %v", currentEnrollmentId, err))
@@ -23,7 +23,7 @@ func (m *Migrations) removeOrphanedOttCaEnrollments(step *boltz.MigrationStep) {
 		}
 
 		if enrollment.CaId != nil && *enrollment.CaId != "" {
-			_, err := m.stores.Ca.LoadOneById(step.Ctx.Tx(), *enrollment.CaId)
+			_, err := m.stores.Ca.LoadById(step.Ctx.Tx(), *enrollment.CaId)
 
 			if err != nil && boltz.IsErrNotFoundErr(err) {
 				enrollmentsToDelete = append(enrollmentsToDelete, currentEnrollmentId)

controller/db/migration_v18.go

@@ -8,7 +8,7 @@ import (
 // Primes API Session's lastActivityAt proper to their previous updatedAt value
 func (m *Migrations) setLastActivityAt(step *boltz.MigrationStep) {
 	for cursor := m.stores.ApiSession.IterateIds(step.Ctx.Tx(), ast.BoolNodeTrue); cursor.IsValid(); cursor.Next() {
-		if apiSession, err := m.stores.ApiSession.LoadOneById(step.Ctx.Tx(), string(cursor.Current())); err == nil {
+		if apiSession, err := m.stores.ApiSession.LoadById(step.Ctx.Tx(), string(cursor.Current())); err == nil {
 			apiSession.LastActivityAt = apiSession.UpdatedAt
 			step.SetError(m.stores.ApiSession.Update(step.Ctx, apiSession, UpdateLastActivityAtChecker{}))
 		} else {

controller/db/migration_v19.go

@@ -35,7 +35,7 @@ func (m *Migrations) updateIdentityTypes(step *boltz.MigrationStep) {
 
 	for cursor := identityStore.IterateIds(step.Ctx.Tx(), ast.BoolNodeTrue); cursor.IsValid(); cursor.Next() {
 		current := cursor.Current()
-		identity, err := identityStore.LoadOneById(step.Ctx.Tx(), string(current))
+		identity, err := identityStore.LoadById(step.Ctx.Tx(), string(current))
 		if step.SetError(err) {
 			return
 		}
@@ -64,7 +64,7 @@ func (m *Migrations) updateIdentityTypes(step *boltz.MigrationStep) {
 }
 
 func (m *Migrations) migrateIdentityType(step *boltz.MigrationStep, id string) {
-	idType, err := m.stores.IdentityType.LoadOneById(step.Ctx.Tx(), id)
+	idType, err := m.stores.IdentityType.LoadById(step.Ctx.Tx(), id)
 	if step.SetError(err) {
 		return
 	}

controller/db/migration_v24.go

@@ -15,7 +15,7 @@ func (m *Migrations) addIdentityIdToSessions(step *boltz.MigrationStep) {
 
 	for cursor.IsValid() {
 		sessionId := string(cursor.Current())
-		session, err := m.stores.Session.LoadOneById(step.Ctx.Tx(), sessionId)
+		session, err := m.stores.Session.LoadById(step.Ctx.Tx(), sessionId)
 
 		if err != nil {
 			step.SetError(fmt.Errorf("could no load session by id [%s]: %v", sessionId, err))
@@ -28,7 +28,7 @@ func (m *Migrations) addIdentityIdToSessions(step *boltz.MigrationStep) {
 		}
 
 		if session.IdentityId == "" {
-			if apiSession, err := m.stores.ApiSession.LoadOneById(step.Ctx.Tx(), session.ApiSessionId); err == nil {
+			if apiSession, err := m.stores.ApiSession.LoadById(step.Ctx.Tx(), session.ApiSessionId); err == nil {
 				if apiSession != nil {
 					session.IdentityId = apiSession.IdentityId
 					if err = m.stores.Session.Update(step.Ctx, session, fieldChecker); err != nil {

controller/db/migrations.go

@@ -23,7 +23,7 @@ import (
 )
 
 const (
-	CurrentDbVersion = 35
+	CurrentDbVersion = 36
 	FieldVersion     = "version"
 )
 
@@ -165,6 +165,12 @@ func (m *Migrations) migrate(step *boltz.MigrationStep) int {
 		step.SetError(m.stores.ConfigType.Update(step.Ctx, hostV2ConfigType, nil))
 	}
 
+	if step.CurrentVersion < 36 {
+		m.dropEntity(step, EntityTypeSessions)
+		m.dropEntity(step, EntityTypeApiSessions)
+		m.dropEntity(step, EntityTypeApiSessionCertificates)
+	}
+
 	// current version
 	if step.CurrentVersion <= CurrentDbVersion {
 		return CurrentDbVersion

controller/db/policy_common.go

@@ -1,7 +1,6 @@
 package db
 
 import (
-	"bytes"
 	"fmt"
 	"github.com/michaelquigley/pfxlog"
 	"github.com/openziti/foundation/v2/errorz"
@@ -311,24 +310,36 @@ type denormCheckCtx struct {
 
 func validatePolicyDenormalization(ctx *denormCheckCtx) error {
 	tx := ctx.mutateCtx.Tx()
+
+	links := map[string]map[string]int{}
+
+	for policyCursor := ctx.policyStore.IterateIds(tx, ast.BoolNodeTrue); policyCursor.IsValid(); policyCursor.Next() {
+		policyId := policyCursor.Current()
+		if ctx.policyFilter == nil || ctx.policyFilter(policyId) {
+			for sourceCursor := ctx.sourceCollection.IterateLinks(tx, policyId); sourceCursor.IsValid(); sourceCursor.Next() {
+				sourceId := string(sourceCursor.Current())
+				for destCursor := ctx.targetCollection.IterateLinks(tx, policyId); destCursor.IsValid(); destCursor.Next() {
+					destId := string(destCursor.Current())
+					destMap := links[sourceId]
+					if destMap == nil {
+						destMap = map[string]int{}
+						links[sourceId] = destMap
+					}
+					destMap[destId] = destMap[destId] + 1
+				}
+			}
+		}
+	}
+
 	for sourceCursor := ctx.sourceStore.IterateIds(tx, ast.BoolNodeTrue); sourceCursor.IsValid(); sourceCursor.Next() {
 		sourceEntityId := sourceCursor.Current()
 		for targetCursor := ctx.targetStore.IterateIds(tx, ast.BoolNodeTrue); targetCursor.IsValid(); targetCursor.Next() {
 			targetEntityId := targetCursor.Current()
 
-			var relatedPolicies []string
-
-			for policyCursor := ctx.policyStore.IterateIds(tx, ast.BoolNodeTrue); policyCursor.IsValid(); policyCursor.Next() {
-				policyId := policyCursor.Current()
-				if ctx.policyFilter == nil || ctx.policyFilter(policyId) {
-					sourceRelated := isRelatedByLinkCollection(tx, ctx.sourceCollection, policyId, sourceEntityId)
-					targetRelated := isRelatedByLinkCollection(tx, ctx.targetCollection, policyId, targetEntityId)
-					if sourceRelated && targetRelated {
-						relatedPolicies = append(relatedPolicies, string(policyId))
-					}
-				}
+			linkCount := 0
+			if destMap, ok := links[string(sourceEntityId)]; ok {
+				linkCount = destMap[string(targetEntityId)]
 			}
-			linkCount := len(relatedPolicies)
 			var sourceLinkCount, targetLinkCount *int32
 			var err error
 			if ctx.repair {
@@ -367,9 +378,3 @@ func logDiscrepencies(ctx *denormCheckCtx, count int, sourceId, targetId []byte,
 		ctx.errorSink(err, ctx.repair)
 	}
 }
-
-func isRelatedByLinkCollection(tx *bbolt.Tx, linkCollection boltz.LinkCollection, entityId, relatedId []byte) bool {
-	cursor := linkCollection.IterateLinks(tx, entityId)
-	cursor.Seek(relatedId)
-	return bytes.Equal(cursor.Current(), relatedId)
-}

controller/db/posture_check_store.go

@@ -83,7 +83,6 @@ func (entity *PostureCheck) GetEntityType() string {
 
 type PostureCheckStore interface {
 	Store[*PostureCheck]
-	LoadOneById(tx *bbolt.Tx, id string) (*PostureCheck, error)
 	GetRoleAttributesIndex() boltz.SetReadIndex
 	GetRoleAttributesCursorProvider(filters []string, semantic string) (ast.SetCursorProvider, error)
 }
@@ -180,7 +179,7 @@ func (store *postureCheckStoreImpl) GetNameIndex() boltz.ReadIndex {
 }
 
 func (store *postureCheckStoreImpl) DeleteById(ctx boltz.MutateContext, id string) error {
-	if entity, _ := store.LoadOneById(ctx.Tx(), id); entity != nil {
+	if entity, _ := store.LoadById(ctx.Tx(), id); entity != nil {
 		// Remove entity from PostureCheckRoles in service policies
 		if err := store.deleteEntityReferences(ctx.Tx(), entity, store.stores.servicePolicy.symbolPostureCheckRoles); err != nil {
 			return err

controller/db/service_edge_router_policy_store.go

@@ -163,7 +163,7 @@ func (store *serviceEdgeRouterPolicyStoreImpl) serviceRolesUpdated(persistCtx *b
 }
 
 func (store *serviceEdgeRouterPolicyStoreImpl) DeleteById(ctx boltz.MutateContext, id string) error {
-	policy, err := store.LoadOneById(ctx.Tx(), id)
+	policy, err := store.LoadById(ctx.Tx(), id)
 	if err != nil {
 		return err
 	}

controller/db/service_edge_router_policy_store_test.go

@@ -33,7 +33,7 @@ func (ctx *TestContext) testCreateServiceEdgeRouterPolicy(_ *testing.T) {
 		ctx.Equal(0, len(ctx.stores.ServiceEdgeRouterPolicy.GetRelatedEntitiesIdList(tx, policy.Id, EntityTypeRouters)))
 		ctx.Equal(0, len(ctx.stores.ServiceEdgeRouterPolicy.GetRelatedEntitiesIdList(tx, policy.Id, EntityTypeServices)))
 
-		testPolicy, err := ctx.stores.ServiceEdgeRouterPolicy.LoadOneById(tx, policy.Id)
+		testPolicy, err := ctx.stores.ServiceEdgeRouterPolicy.LoadById(tx, policy.Id)
 		ctx.NoError(err)
 		ctx.NotNil(testPolicy)
 		ctx.Equal(policy.Name, testPolicy.Name)

controller/db/service_policy_store.go

@@ -274,7 +274,7 @@ func (store *servicePolicyStoreImpl) postureCheckRolesUpdated(persistCtx *boltz.
 }
 
 func (store *servicePolicyStoreImpl) DeleteById(ctx boltz.MutateContext, id string) error {
-	policy, err := store.LoadOneById(ctx.Tx(), id)
+	policy, err := store.LoadById(ctx.Tx(), id)
 	if err != nil {
 		return err
 	}

controller/db/service_policy_store_test.go

@@ -49,7 +49,7 @@ func (ctx *TestContext) testCreateServicePolicy(_ *testing.T) {
 		ctx.Equal(0, len(ctx.stores.ServicePolicy.GetRelatedEntitiesIdList(tx, policy.Id, EntityTypeServices)))
 		ctx.Equal(0, len(ctx.stores.ServicePolicy.GetRelatedEntitiesIdList(tx, policy.Id, EntityTypeIdentities)))
 
-		testPolicy, err := ctx.stores.ServicePolicy.LoadOneById(tx, policy.Id)
+		testPolicy, err := ctx.stores.ServicePolicy.LoadById(tx, policy.Id)
 		ctx.NoError(err)
 		ctx.NotNil(testPolicy)
 		ctx.Equal(policy.Name, testPolicy.Name)

controller/db/session_store.go

@@ -145,14 +145,14 @@ func (*sessionStoreImpl) PersistEntity(entity *Session, ctx *boltz.PersistContex
 	ctx.SetStringList(FieldSessionServicePolicies, entity.ServicePolicies)
 
 	if entity.ApiSession == nil {
-		entity.ApiSession, _ = ctx.Store.(*sessionStoreImpl).stores.apiSession.LoadOneById(ctx.Bucket.Tx(), entity.ApiSessionId)
+		entity.ApiSession, _ = ctx.Store.(*sessionStoreImpl).stores.apiSession.LoadById(ctx.Bucket.Tx(), entity.ApiSessionId)
 	}
 }
 
 func (store *sessionStoreImpl) LoadOneByToken(tx *bbolt.Tx, token string) (*Session, error) {
 	id := store.indexToken.Read(tx, []byte(token))
 	if id != nil {
-		return store.LoadOneById(tx, string(id))
+		return store.LoadById(tx, string(id))
 	}
 	return nil, boltz.NewNotFoundError(store.GetSingularEntityType(), "token", token)
 }

controller/db/session_store_test.go

@@ -100,7 +100,7 @@ func (ctx *TestContext) testUpdateInvalidSessions(_ *testing.T) {
 	boltztest.RequireUpdate(ctx, session)
 
 	err := ctx.GetDb().View(func(tx *bbolt.Tx) error {
-		loaded, err := ctx.stores.Session.LoadOneById(tx, session.Id)
+		loaded, err := ctx.stores.Session.LoadById(tx, session.Id)
 		ctx.NoError(err)
 		ctx.NotNil(loaded)
 		ctx.Equal(apiSession.Id, loaded.ApiSessionId)
@@ -236,11 +236,11 @@ func (ctx *TestContext) testUpdateSessions(_ *testing.T) {
 	mutateCtx := change.New().NewMutateContext()
 	err := ctx.GetDb().Update(mutateCtx, func(mutateCtx boltz.MutateContext) error {
 		tx := mutateCtx.Tx()
-		original, err := ctx.stores.Session.LoadOneById(tx, entities.session1.Id)
+		original, err := ctx.stores.Session.LoadById(tx, entities.session1.Id)
 		ctx.NoError(err)
 		ctx.NotNil(original)
 
-		session, err := ctx.stores.Session.LoadOneById(tx, entities.session1.Id)
+		session, err := ctx.stores.Session.LoadById(tx, entities.session1.Id)
 		ctx.NoError(err)
 		ctx.NotNil(session)
 
@@ -252,7 +252,7 @@ func (ctx *TestContext) testUpdateSessions(_ *testing.T) {
 
 		err = ctx.stores.Session.Update(mutateCtx, session, nil)
 		ctx.NoError(err)
-		loaded, err := ctx.stores.Session.LoadOneById(tx, entities.session1.Id)
+		loaded, err := ctx.stores.Session.LoadById(tx, entities.session1.Id)
 		ctx.NoError(err)
 		ctx.NotNil(loaded)
 		ctx.EqualValues(original.CreatedAt, loaded.CreatedAt)

controller/db/stores.go

@@ -103,6 +103,7 @@ type Stores struct {
 	Ca                      CaStore
 	Config                  ConfigStore
 	ConfigType              ConfigTypeStore
+	Controller              ControllerStore
 	EdgeRouter              EdgeRouterStore
 	EdgeRouterPolicy        EdgeRouterPolicyStore
 	EdgeService             EdgeServiceStore
@@ -203,6 +204,7 @@ type stores struct {
 	ca                      *caStoreImpl
 	config                  *configStoreImpl
 	configType              *configTypeStoreImpl
+	controller              *controllerStoreImpl
 	edgeRouter              *edgeRouterStoreImpl
 	edgeRouterPolicy        *edgeRouterPolicyStoreImpl
 	edgeService             *edgeServiceStoreImpl
@@ -258,6 +260,7 @@ func InitStores(db boltz.Db, rateLimiter command.RateLimiter) (*Stores, error) {
 	internalStores.ca = newCaStore(internalStores)
 	internalStores.config = newConfigsStore(internalStores)
 	internalStores.configType = newConfigTypesStore(internalStores)
+	internalStores.controller = newControllerStore(internalStores)
 	internalStores.edgeRouter = newEdgeRouterStore(internalStores)
 	internalStores.edgeRouterPolicy = newEdgeRouterPolicyStore(internalStores)
 	internalStores.edgeService = newEdgeServiceStore(internalStores)
@@ -288,6 +291,7 @@ func InitStores(db boltz.Db, rateLimiter command.RateLimiter) (*Stores, error) {
 		Ca:                      internalStores.ca,
 		Config:                  internalStores.config,
 		ConfigType:              internalStores.configType,
+		Controller:              internalStores.controller,
 		EdgeRouter:              internalStores.edgeRouter,
 		EdgeRouterPolicy:        internalStores.edgeRouterPolicy,
 		EdgeService:             internalStores.edgeService,

controller/db/transit_router_store.go

@@ -133,7 +133,7 @@ func (store *transitRouterStoreImpl) PersistEntity(entity *TransitRouter, ctx *b
 }
 
 func (store *transitRouterStoreImpl) cleanupEnrollments(ctx boltz.MutateContext, id string) error {
-	if entity, _ := store.LoadOneById(ctx.Tx(), id); entity != nil {
+	if entity, _ := store.LoadById(ctx.Tx(), id); entity != nil {
 		// Remove outstanding enrollments
 		if err := store.stores.enrollment.DeleteWhere(ctx, fmt.Sprintf(`transitRouter="%s"`, entity.Id)); err != nil {
 			return err

controller/env/appenv.go

@@ -45,11 +45,12 @@ import (
 	"github.com/openziti/sdk-golang/ziti"
 	"github.com/openziti/storage/boltz"
 	"github.com/openziti/xweb/v2"
+	"github.com/openziti/ziti/common"
 	"github.com/openziti/ziti/common/cert"
 	"github.com/openziti/ziti/common/eid"
 	"github.com/openziti/ziti/controller/api"
 	"github.com/openziti/ziti/controller/command"
-	edgeConfig "github.com/openziti/ziti/controller/config"
+	"github.com/openziti/ziti/controller/config"
 	"github.com/openziti/ziti/controller/db"
 	"github.com/openziti/ziti/controller/event"
 	"github.com/openziti/ziti/controller/events"
@@ -58,7 +59,6 @@ import (
 	"github.com/openziti/ziti/controller/model"
 	"github.com/openziti/ziti/controller/models"
 	"github.com/openziti/ziti/controller/network"
-	"github.com/openziti/ziti/controller/oidc_auth"
 	"github.com/openziti/ziti/controller/response"
 	"github.com/openziti/ziti/controller/xctrl"
 	"github.com/openziti/ziti/controller/xmgmt"
@@ -83,7 +83,7 @@ const (
 
 type AppEnv struct {
 	Managers *model.Managers
-	Config   *edgeConfig.Config
+	Config   *config.Config
 
 	Versions *ziti.Versions
 
@@ -91,22 +91,27 @@ type AppEnv struct {
 	ApiClientCsrSigner     cert.Signer
 	ControlClientCsrSigner cert.Signer
 
-	FingerprintGenerator    cert.FingerprintGenerator
-	AuthRegistry            model.AuthRegistry
-	EnrollRegistry          model.EnrollmentRegistry
-	Broker                  *Broker
-	HostController          HostController
-	ManagementApi           *managementOperations.ZitiEdgeManagementAPI
-	ClientApi               *clientOperations.ZitiEdgeClientAPI
-	IdentityRefreshMap      cmap.ConcurrentMap[string, time.Time]
-	identityRefreshMeter    metrics.Meter
-	StartupTime             time.Time
-	InstanceId              string
-	enrollmentSigner        jwtsigner.Signer
-	TraceManager            *TraceManager
-	ServerCert              *tls.Certificate
-	ServerCertSigningMethod jwt.SigningMethod
-	AuthRateLimiter         command.AdaptiveRateLimiter
+	FingerprintGenerator cert.FingerprintGenerator
+	AuthRegistry         model.AuthRegistry
+	EnrollRegistry       model.EnrollmentRegistry
+	Broker               *Broker
+	HostController       HostController
+	ManagementApi        *managementOperations.ZitiEdgeManagementAPI
+	ClientApi            *clientOperations.ZitiEdgeClientAPI
+	IdentityRefreshMap   cmap.ConcurrentMap[string, time.Time]
+	identityRefreshMeter metrics.Meter
+	StartupTime          time.Time
+	InstanceId           string
+	AuthRateLimiter      command.AdaptiveRateLimiter
+
+	serverSigner jwtsigner.Signer
+	ServerCert   *tls.Certificate
+
+	TraceManager *TraceManager
+}
+
+func (ae *AppEnv) GetPeerControllerAddresses() []string {
+	return ae.HostController.GetPeerAddresses()
 }
 
 // JwtSignerKeyFunc is used in combination with jwt.Parse or jwt.ParseWithClaims to
@@ -136,9 +141,106 @@ func (ae *AppEnv) JwtSignerKeyFunc(token *jwt.Token) (interface{}, error) {
 	return nil, errors.New("invalid kid: " + targetKid)
 }
 
+func (ae *AppEnv) ValidateAccessToken(token string) (*common.AccessClaims, error) {
+	accessClaims := &common.AccessClaims{}
+
+	parsedToken, err := jwt.ParseWithClaims(token, accessClaims, ae.JwtSignerKeyFunc)
+
+	if err != nil {
+		return nil, err
+	}
+
+	if !parsedToken.Valid {
+		return nil, errors.New("access token is invalid")
+	}
+
+	if !accessClaims.HasAudience(common.ClaimAudienceOpenZiti) {
+		return nil, fmt.Errorf("invalind audience, expected an instance of %s, got %v", common.ClaimAudienceOpenZiti, accessClaims.Audience)
+	}
+
+	if accessClaims.Type != common.TokenTypeAccess {
+		return nil, fmt.Errorf("invalid token type, expected %s, got %s", common.TokenTypeAccess, accessClaims.Type)
+	}
+
+	tokenRevocation, err := ae.GetManagers().Revocation.Read(accessClaims.JWTID)
+
+	if err != nil && !boltz.IsErrNotFoundErr(err) {
+		return nil, err
+	}
+
+	if tokenRevocation != nil {
+		return nil, errors.New("access token has been revoked by id")
+	}
+
+	revocation, err := ae.GetManagers().Revocation.Read(accessClaims.Subject)
+
+	if err != nil && !boltz.IsErrNotFoundErr(err) {
+		return nil, err
+	}
+
+	if revocation != nil && tokenRevocation.CreatedAt.After(accessClaims.IssuedAt.AsTime()) {
+		return nil, errors.New("access token has been revoked by identity")
+	}
+
+	return accessClaims, nil
+}
+
+func (ae *AppEnv) ValidateServiceAccessToken(token string, apiSessionId *string) (*common.ServiceAccessClaims, error) {
+	serviceAccessClaims := &common.ServiceAccessClaims{}
+
+	parsedToken, err := jwt.ParseWithClaims(token, serviceAccessClaims, ae.JwtSignerKeyFunc)
+
+	if err != nil {
+		return nil, err
+	}
+
+	if !parsedToken.Valid {
+		return nil, errors.New("service access token is invalid")
+	}
+
+	if !serviceAccessClaims.HasAudience(common.ClaimAudienceOpenZiti) {
+		return nil, fmt.Errorf("invalind audience, expected an instance of %s, got %v", common.ClaimAudienceOpenZiti, serviceAccessClaims.Audience)
+	}
+
+	if serviceAccessClaims.TokenType != common.TokenTypeServiceAccess {
+		return nil, fmt.Errorf("invalid token type, expected %s, got %s", common.TokenTypeServiceAccess, serviceAccessClaims.Type)
+	}
+
+	if apiSessionId != nil {
+		if *apiSessionId == "" {
+			return nil, errors.New("invalid target api session id, must not be empty string")
+		}
+
+		if serviceAccessClaims.ApiSessionId != *apiSessionId {
+			return nil, fmt.Errorf("invalid api sessoin id, expected %s, got %s", *apiSessionId, serviceAccessClaims.ApiSessionId)
+		}
+	}
+
+	tokenRevocation, err := ae.GetManagers().Revocation.Read(serviceAccessClaims.ID)
+
+	if err != nil && !boltz.IsErrNotFoundErr(err) {
+		return nil, err
+	}
+
+	if tokenRevocation != nil {
+		return nil, errors.New("service access token has been revoked by id")
+	}
+
+	revocation, err := ae.GetManagers().Revocation.Read(serviceAccessClaims.IdentityId)
+
+	if err != nil && !boltz.IsErrNotFoundErr(err) {
+		return nil, err
+	}
+
+	if revocation != nil && tokenRevocation.CreatedAt.After(serviceAccessClaims.IssuedAt.Time) {
+		return nil, errors.New("service access token has been revoked by identity")
+	}
+
+	return serviceAccessClaims, nil
+}
+
 func (ae *AppEnv) GetServerCert() (serverCert *tls.Certificate, kid string, signingMethod jwt.SigningMethod) {
-	kid = fmt.Sprintf("%x", sha1.Sum(ae.ServerCert.Certificate[0]))
-	return ae.ServerCert, kid, ae.ServerCertSigningMethod
+	return ae.ServerCert, ae.serverSigner.KeyId(), ae.serverSigner.SigningMethod()
 }
 
 func (ae *AppEnv) GetApiServerCsrSigner() cert.Signer {
@@ -161,12 +263,12 @@ func (ae *AppEnv) GetManagers() *model.Managers {
 	return ae.Managers
 }
 
-func (ae *AppEnv) GetConfig() *edgeConfig.Config {
+func (ae *AppEnv) GetConfig() *config.Config {
 	return ae.Config
 }
 
-func (ae *AppEnv) GetJwtSigner() jwtsigner.Signer {
-	return ae.enrollmentSigner
+func (ae *AppEnv) GetServerJwtSigner() jwtsigner.Signer {
+	return ae.serverSigner
 }
 
 func (ae *AppEnv) GetDbProvider() network.DbProvider {
@@ -207,8 +309,11 @@ type HostController interface {
 	Shutdown()
 	Identity() identity.Identity
 	IsRaftEnabled() bool
+	IsRaftLeader() bool
 	GetPeerSigners() []*x509.Certificate
 	GetEventDispatcher() event.Dispatcher
+	GetRaftIndex() uint64
+	GetPeerAddresses() []string
 }
 
 type Schemes struct {
@@ -373,13 +478,13 @@ func (ae *AppEnv) ProcessZtSession(rc *response.RequestContext, ztSession string
 func (ae *AppEnv) ProcessJwt(rc *response.RequestContext, token *jwt.Token) error {
 	rc.SessionToken = token.Raw
 	rc.Jwt = token
-	rc.Claims = token.Claims.(*oidc_auth.AccessClaims)
+	rc.Claims = token.Claims.(*common.AccessClaims)
 
 	if rc.Claims == nil {
 		return fmt.Errorf("could not convert tonek.Claims from %T to %T", rc.Jwt.Claims, rc.Claims)
 	}
 
-	if rc.Claims.Type != oidc_auth.TokenTypeAccess {
+	if rc.Claims.Type != common.TokenTypeAccess {
 		return errors.New("invalid token")
 	}
 
@@ -405,7 +510,7 @@ func (ae *AppEnv) ProcessJwt(rc *response.RequestContext, token *jwt.Token) erro
 
 	rc.ApiSession = &model.ApiSession{
 		BaseEntity: models.BaseEntity{
-			Id:        rc.Claims.JWTID,
+			Id:        rc.Claims.ApiSessionId,
 			CreatedAt: rc.Claims.IssuedAt.AsTime(),
 			UpdatedAt: rc.Claims.IssuedAt.AsTime(),
 			IsSystem:  false,
@@ -514,7 +619,7 @@ func ProcessAuthQueries(ae *AppEnv, rc *response.RequestContext) {
 	}
 }
 
-func NewAppEnv(c *edgeConfig.Config, host HostController) *AppEnv {
+func NewAppEnv(c *config.Config, host HostController) *AppEnv {
 	clientSpec, err := loads.Embedded(clientServer.SwaggerJSON, clientServer.FlatSwaggerJSON)
 	if err != nil {
 		pfxlog.Logger().Fatalln(err)
@@ -695,7 +800,8 @@ func (ae *AppEnv) getJwtTokenFromRequest(r *http.Request) *jwt.Token {
 	for _, header := range headers {
 		if strings.HasPrefix(header, "Bearer ") {
 			token := header[7:]
-			parsedToken, err := jwt.ParseWithClaims(token, &oidc_auth.AccessClaims{}, ae.ControllersKeyFunc)
+			claims := &common.AccessClaims{}
+			parsedToken, err := jwt.ParseWithClaims(token, claims, ae.ControllersKeyFunc)
 
 			if err != nil {
 				pfxlog.Logger().WithError(err).Error("error during JWT parsing during API request")
@@ -855,14 +961,20 @@ func (ae *AppEnv) HandleServiceUpdatedEventForIdentityId(identityId string) {
 	ae.identityRefreshMeter.Mark(1)
 }
 
-func (ae *AppEnv) SetEnrollmentSigningCert(serverCert *tls.Certificate) {
+func (ae *AppEnv) SetServerCert(serverCert *tls.Certificate) {
+	ae.ServerCert = serverCert
+
 	signMethod := getJwtSigningMethod(serverCert)
 	kid := fmt.Sprintf("%x", sha1.Sum(serverCert.Certificate[0]))
-	ae.enrollmentSigner = jwtsigner.New(ae.Config.Api.Address, signMethod, serverCert.PrivateKey, kid)
+	ae.serverSigner = jwtsigner.New(signMethod, serverCert.PrivateKey, kid)
+
+	ae.Broker.routerSyncStrategy.AddPublicKey(ae.ServerCert)
 }
 
-func (ae *AppEnv) SetServerIdentity(certificate *tls.Certificate) {
-	ae.ServerCert = certificate
-	ae.ServerCertSigningMethod = getJwtSigningMethod(certificate)
-
+func (ae *AppEnv) OidcIssuer() string {
+	return ae.RootIssuer() + "/oidc"
+}
+
+func (ae *AppEnv) RootIssuer() string {
+	return "https://" + ae.Config.Api.Address
 }

controller/env/broker.go

@@ -35,7 +35,8 @@ const (
 	ApiSessionAddedType     = int32(edge_ctrl_pb.ContentType_ApiSessionAddedType)
 	ApiSessionUpdatedType   = int32(edge_ctrl_pb.ContentType_ApiSessionUpdatedType)
 	RequestClientReSyncType = int32(edge_ctrl_pb.ContentType_RequestClientReSyncType)
-	SigningCertAdded        = int32(edge_ctrl_pb.ContentType_SigningCertAddedType)
+	DataStateType           = int32(edge_ctrl_pb.ContentType_DataStateType)
+	DataStateEventType      = int32(edge_ctrl_pb.ContentType_DataStateEventType)
 
 	ServerHelloType = int32(edge_ctrl_pb.ContentType_ServerHelloType)
 	ClientHelloType = int32(edge_ctrl_pb.ContentType_ClientHelloType)
@@ -45,6 +46,9 @@ const (
 	EnrollmentExtendRouterVerifyRequestType = int32(edge_ctrl_pb.ContentType_EnrollmentExtendRouterVerifyRequestType)
 )
 
+type RouterSyncCache struct {
+}
+
 // The Broker delegates Ziti Edge events to a RouterSyncStrategy. Handling the details of which events to watch
 // and dealing with casting arguments to their proper concrete types.
 type Broker struct {
@@ -69,16 +73,26 @@ func NewBroker(ae *AppEnv, synchronizer RouterSyncStrategy) *Broker {
 	broker.ae.GetStores().ApiSession.GetEventsEmitter().AddListener(db.EventFullyAuthenticated, broker.apiSessionFullyAuthenticated)
 	broker.ae.GetStores().ApiSessionCertificate.AddEntityEventListenerF(broker.apiSessionCertificateCreated, boltz.EntityCreatedAsync)
 	broker.ae.GetStores().ApiSessionCertificate.AddEntityEventListenerF(broker.apiSessionCertificateDeleted, boltz.EntityDeletedAsync)
+	broker.ae.GetStores().Controller.AddEntityEventListenerF(broker.controllerAdded, boltz.EntityCreated)
+	broker.ae.GetStores().Controller.AddEntityEventListenerF(broker.controllerAdded, boltz.EntityUpdated)
 
 	ae.HostController.GetNetwork().AddRouterPresenceHandler(broker)
 	ae.HostController.GetEventDispatcher().AddClusterEventHandler(broker)
 
+	ae.GetHostController().GetPeerSigners()
+
 	return broker
 }
 
 func (broker *Broker) AcceptClusterEvent(clusterEvent *event.ClusterEvent) {
-	if clusterEvent.EventType == event.ClusterPeerConnected && len(clusterEvent.Peers) > 0 {
-		broker.routerSyncStrategy.PeerAdded(clusterEvent.Peers)
+	if broker.ae.HostController.IsRaftLeader() {
+		if clusterEvent.EventType == event.ClusterPeerConnected {
+			broker.ae.Managers.Controller.PeersConnected(clusterEvent.Peers)
+		}
+
+		if clusterEvent.EventType == event.ClusterPeerDisconnected {
+			broker.ae.Managers.Controller.PeersDisconnected(clusterEvent.Peers)
+		}
 	}
 }
 
@@ -145,7 +159,7 @@ func (broker *Broker) apiSessionCertificateHandler(delete bool, apiSessionCert *
 	var apiSession *db.ApiSession
 	var err error
 	err = broker.ae.GetDbProvider().GetDb().View(func(tx *bbolt.Tx) error {
-		apiSession, err = broker.ae.GetStores().ApiSession.LoadOneById(tx, apiSessionCert.ApiSessionId)
+		apiSession, err = broker.ae.GetStores().ApiSession.LoadById(tx, apiSessionCert.ApiSessionId)
 		return err
 	})
 
@@ -172,3 +186,7 @@ func (broker *Broker) GetEdgeRouterState(id string) RouterStateValues {
 func (broker *Broker) Stop() {
 	broker.routerSyncStrategy.Stop()
 }
+
+func (broker *Broker) controllerAdded(controller *db.Controller) {
+	broker.routerSyncStrategy.PeerAdded(controller)
+}

controller/env/sync.go

@@ -17,19 +17,19 @@
 package env
 
 import (
+	"crypto/tls"
 	"github.com/openziti/channel/v2"
 	"github.com/openziti/foundation/v2/versions"
 	"github.com/openziti/ziti/controller/db"
-	"github.com/openziti/ziti/controller/event"
 	"github.com/openziti/ziti/controller/model"
 	"github.com/openziti/ziti/controller/network"
 	"sync"
 )
 
-// Aliased type for router strategies
+// RouterSyncStrategyType aliased type for router strategies
 type RouterSyncStrategyType string
 
-// Aliased type for router sync status
+// RouterSyncStatus aliased type for router sync status
 type RouterSyncStatus string
 
 const (
@@ -50,6 +50,7 @@ const (
 	//msg headers
 	SyncStrategyTypeHeader  = 1013
 	SyncStrategyStateHeader = 1014
+	SyncStrategyLastIndex   = 1015
 )
 
 // RouterSyncStrategy handles the life cycle of an Edge Router connecting to the controller, synchronizing
@@ -60,7 +61,7 @@ type RouterSyncStrategy interface {
 	Stop()
 	RouterConnectionHandler
 	RouterSynchronizerEventHandler
-	PeerAdded(peers []*event.ClusterPeer)
+	AddPublicKey(cert *tls.Certificate)
 }
 
 // RouterConnectionHandler is responsible for handling router connect/disconnect for synchronizing state.
@@ -79,6 +80,8 @@ type RouterSynchronizerEventHandler interface {
 	ApiSessionDeleted(apiSession *db.ApiSession)
 
 	SessionDeleted(session *db.Session)
+
+	PeerAdded(*db.Controller)
 }
 
 // RouterState provides a thread save mechanism to access and set router status information that may be influx

controller/handler_edge_ctrl/common.go

@@ -2,10 +2,8 @@ package handler_edge_ctrl
 
 import (
 	"fmt"
-	"github.com/openziti/ziti/common/pb/edge_ctrl_pb"
-	"github.com/openziti/ziti/controller/change"
-	"github.com/openziti/ziti/controller/fields"
 	"math"
+	"strings"
 	"time"
 
 	"github.com/michaelquigley/pfxlog"
@@ -14,12 +12,19 @@ import (
 	"github.com/openziti/identity"
 	"github.com/openziti/sdk-golang/ziti/edge"
 	"github.com/openziti/storage/boltz"
+	"github.com/openziti/ziti/common"
 	"github.com/openziti/ziti/common/logcontext"
+	"github.com/openziti/ziti/common/pb/edge_ctrl_pb"
+	"github.com/openziti/ziti/controller/change"
 	"github.com/openziti/ziti/controller/db"
 	"github.com/openziti/ziti/controller/env"
+	"github.com/openziti/ziti/controller/fields"
 	"github.com/openziti/ziti/controller/model"
+	"github.com/openziti/ziti/controller/models"
 	"github.com/openziti/ziti/controller/network"
+	"github.com/openziti/ziti/controller/oidc_auth"
 	"github.com/openziti/ziti/controller/xt"
+	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 )
 
@@ -111,6 +116,8 @@ type baseSessionRequestContext struct {
 	service      *model.Service
 	newSession   bool
 	logContext   logcontext.Context
+	env          model.Env
+	accessClaims *common.AccessClaims
 }
 
 func (self *baseSessionRequestContext) newChangeContext() *change.Context {
@@ -170,53 +177,140 @@ func (self *baseSessionRequestContext) loadRouter() bool {
 	return true
 }
 
-func (self *baseSessionRequestContext) loadSession(token string) {
-	if self.err == nil {
-		var err error
-		self.session, err = self.handler.getAppEnv().Managers.Session.ReadByToken(token)
-		if err != nil {
-			if boltz.IsErrNotFoundErr(err) {
-				self.err = InvalidSessionError{}
-			} else {
-				self.err = internalError(err)
-			}
-			logrus.
-				WithField("token", token).
-				WithField("operation", self.handler.Label()).
-				WithError(self.err).Errorf("invalid session")
-			return
-		}
-		apiSession, err := self.handler.getAppEnv().Managers.ApiSession.Read(self.session.ApiSessionId)
-		if err != nil {
-			if boltz.IsErrNotFoundErr(err) {
-				self.err = InvalidApiSessionError{}
-			} else {
-				self.err = internalError(err)
-			}
-			logrus.
-				WithField("token", token).
-				WithField("operation", self.handler.Label()).
-				WithError(self.err).Errorf("invalid api-session")
-			return
-		}
-		self.apiSession = apiSession
-
-		self.logContext = logcontext.NewContext()
-		traceSpec := self.handler.getAppEnv().TraceManager.GetIdentityTrace(apiSession.IdentityId)
-		traceEnabled := traceSpec != nil && time.Now().Before(traceSpec.Until)
-		if traceEnabled {
-			self.logContext.SetChannelsMask(traceSpec.ChannelMask)
-			self.logContext.WithField("traceId", traceSpec.TraceId)
-		}
-		self.logContext.WithField("sessionId", self.session.Id)
-		self.logContext.WithField("apiSessionId", apiSession.Id)
-
-		if traceEnabled {
-			pfxlog.ChannelLogger(logcontext.EstablishPath).
-				Wire(self.logContext).
-				Debug("tracing enabled for this session")
-		}
+func (self *baseSessionRequestContext) loadSession(sessionToken string, apiSessionToken string) {
+	if strings.HasPrefix(sessionToken, oidc_auth.JwtTokenPrefix) {
+		self.loadFromTokens(sessionToken, apiSessionToken)
+	} else {
+		self.loadFromBolt(sessionToken)
 	}
+
+	if self.err != nil {
+		return
+	}
+
+	if self.session == nil {
+		self.err = internalError(errors.New("session was not found after load"))
+		return
+	}
+
+	if self.apiSession == nil {
+		self.err = internalError(errors.New("api session was not found after load"))
+		return
+	}
+
+	self.logContext = logcontext.NewContext()
+	traceSpec := self.handler.getAppEnv().TraceManager.GetIdentityTrace(self.apiSession.IdentityId)
+	traceEnabled := traceSpec != nil && time.Now().Before(traceSpec.Until)
+	if traceEnabled {
+		self.logContext.SetChannelsMask(traceSpec.ChannelMask)
+		self.logContext.WithField("traceId", traceSpec.TraceId)
+	}
+	self.logContext.WithField("sessionId", self.session.Id)
+	self.logContext.WithField("apiSessionId", self.apiSession.Id)
+
+	if traceEnabled {
+		pfxlog.ChannelLogger(logcontext.EstablishPath).
+			Wire(self.logContext).
+			Debug("tracing enabled for this session")
+	}
+}
+
+func (self *baseSessionRequestContext) loadFromTokens(sessionToken, apiSessionToken string) {
+	if self.err != nil {
+		return
+	}
+
+	var err error
+	self.accessClaims, err = self.env.ValidateAccessToken(apiSessionToken)
+
+	if err != nil {
+		self.err = internalError(err)
+		return
+	}
+
+	serviceAccessClaims, err := self.env.ValidateServiceAccessToken(sessionToken, &self.accessClaims.ApiSessionId)
+
+	if err != nil {
+		self.err = internalError(err)
+		return
+	}
+
+	if self.accessClaims.Subject != serviceAccessClaims.IdentityId {
+		self.err = internalError(fmt.Errorf("access and service tokens do not match, got access identity id %s and service identity id %s", self.accessClaims.Subject, serviceAccessClaims.IdentityId))
+		return
+	}
+
+	self.session = &model.Session{
+		BaseEntity: models.BaseEntity{
+			Id:        serviceAccessClaims.ID,
+			CreatedAt: serviceAccessClaims.IssuedAt.Time,
+			UpdatedAt: serviceAccessClaims.IssuedAt.Time,
+		},
+		Token:        sessionToken,
+		IdentityId:   serviceAccessClaims.IdentityId,
+		ApiSessionId: serviceAccessClaims.ApiSessionId,
+		ServiceId:    serviceAccessClaims.Subject,
+		Type:         serviceAccessClaims.Type,
+	}
+
+	tokenIdentity, err := self.env.GetManagers().Identity.Read(self.accessClaims.Subject)
+
+	if err != nil {
+		self.err = internalError(err)
+		return
+	}
+
+	self.apiSession = &model.ApiSession{
+		BaseEntity: models.BaseEntity{
+			Id: serviceAccessClaims.ApiSessionId,
+		},
+		Token:              apiSessionToken,
+		IdentityId:         serviceAccessClaims.IdentityId,
+		Identity:           tokenIdentity,
+		IPAddress:          self.accessClaims.RemoteAddress,
+		ConfigTypes:        self.accessClaims.ConfigTypesAsMap(),
+		MfaComplete:        false,
+		MfaRequired:        false,
+		ExpiresAt:          time.Time{},
+		ExpirationDuration: 0,
+		LastActivityAt:     time.Time{},
+		AuthenticatorId:    "",
+	}
+}
+
+func (self *baseSessionRequestContext) loadFromBolt(token string) {
+	if self.err != nil {
+		return
+	}
+
+	var err error
+	self.session, err = self.handler.getAppEnv().Managers.Session.ReadByToken(token)
+	if err != nil {
+		if boltz.IsErrNotFoundErr(err) {
+			self.err = InvalidSessionError{}
+		} else {
+			self.err = internalError(err)
+		}
+		logrus.
+			WithField("token", token).
+			WithField("operation", self.handler.Label()).
+			WithError(self.err).Errorf("invalid session")
+		return
+	}
+	apiSession, err := self.handler.getAppEnv().Managers.ApiSession.Read(self.session.ApiSessionId)
+	if err != nil {
+		if boltz.IsErrNotFoundErr(err) {
+			self.err = InvalidApiSessionError{}
+		} else {
+			self.err = internalError(err)
+		}
+		logrus.
+			WithField("token", token).
+			WithField("operation", self.handler.Label()).
+			WithError(self.err).Errorf("invalid api-session")
+		return
+	}
+	self.apiSession = apiSession
 }
 
 func (self *baseSessionRequestContext) checkSessionType(sessionType string) {
@@ -232,10 +326,25 @@ func (self *baseSessionRequestContext) checkSessionType(sessionType string) {
 }
 
 func (self *baseSessionRequestContext) checkSessionFingerprints(fingerprints []string) {
-	if self.err == nil {
-		var apiSessionCertFingerprints []string
+	if self.err != nil {
+		return
+	}
 
-		found := false
+	var apiSessionCertFingerprints []string
+	found := false
+
+	if self.accessClaims != nil {
+		apiSessionCertFingerprints = self.accessClaims.CertFingerprints
+
+		for _, fingerprint := range fingerprints {
+			found = stringz.Contains(apiSessionCertFingerprints, fingerprint)
+
+			if found {
+				break
+			}
+		}
+
+	} else {
 		err := self.GetHandler().getAppEnv().Managers.ApiSession.VisitFingerprintsForApiSessionId(self.session.ApiSessionId, func(fingerprint string) bool {
 			apiSessionCertFingerprints = append(apiSessionCertFingerprints, fingerprint)
 			if stringz.Contains(fingerprints, fingerprint) {
@@ -246,19 +355,20 @@ func (self *baseSessionRequestContext) checkSessionFingerprints(fingerprints []s
 		})
 
 		self.err = internalError(err)
-
-		if self.err != nil || !found {
-			if self.err == nil {
-				self.err = InvalidApiSessionError{}
-			}
-			logrus.
-				WithField("sessionId", self.session.Id).
-				WithField("operation", self.handler.Label()).
-				WithField("apiSessionFingerprints", apiSessionCertFingerprints).
-				WithField("clientFingerprints", fingerprints).
-				Error("matching fingerprint not found for connect")
-		}
 	}
+
+	if self.err != nil || !found {
+		if self.err == nil {
+			self.err = InvalidApiSessionError{}
+		}
+		logrus.
+			WithField("sessionId", self.session.Id).
+			WithField("operation", self.handler.Label()).
+			WithField("apiSessionFingerprints", apiSessionCertFingerprints).
+			WithField("clientFingerprints", fingerprints).
+			Error("matching fingerprint not found for connect")
+	}
+
 }
 
 func (self *baseSessionRequestContext) verifyEdgeRouterAccess() {

controller/handler_edge_ctrl/create_circuit.go

@@ -101,7 +101,7 @@ func (self *createCircuitHandler) HandleReceiveCreateCircuitV2(msg *channel.Mess
 	}
 
 	ctx := &CreateCircuitRequestContext{
-		baseSessionRequestContext: baseSessionRequestContext{handler: self, msg: msg},
+		baseSessionRequestContext: baseSessionRequestContext{handler: self, msg: msg, env: self.appEnv},
 		req:                       req,
 	}
 
@@ -123,7 +123,7 @@ func (self *createCircuitHandler) CreateCircuit(ctx *CreateCircuitRequestContext
 	if !ctx.loadRouter() {
 		return
 	}
-	ctx.loadSession(ctx.req.GetSessionToken())
+	ctx.loadSession(ctx.req.GetSessionToken(), ctx.req.GetApiSessionToken())
 	ctx.checkSessionType(db.SessionTypeDial)
 	ctx.checkSessionFingerprints(ctx.req.GetFingerprints())
 	ctx.verifyEdgeRouterAccess()
@@ -154,6 +154,7 @@ var _ CreateCircuitRequest = (*edge_ctrl_pb.CreateCircuitRequest)(nil)
 
 type CreateCircuitRequest interface {
 	GetSessionToken() string
+	GetApiSessionToken() string
 	GetFingerprints() []string
 	GetTerminatorInstanceId() string
 	GetPeerData() map[uint32][]byte

controller/handler_edge_ctrl/create_terminator.go

@@ -61,7 +61,7 @@ func (self *createTerminatorHandler) HandleReceive(msg *channel.Message, ch chan
 	}
 
 	ctx := &CreateTerminatorRequestContext{
-		baseSessionRequestContext: baseSessionRequestContext{handler: self, msg: msg},
+		baseSessionRequestContext: baseSessionRequestContext{handler: self, msg: msg, env: self.appEnv},
 		req:                       req,
 	}
 
@@ -74,7 +74,7 @@ func (self *createTerminatorHandler) CreateTerminator(ctx *CreateTerminatorReque
 	if !ctx.loadRouter() {
 		return
 	}
-	ctx.loadSession(ctx.req.SessionToken)
+	ctx.loadSession(ctx.req.SessionToken, ctx.req.ApiSessionToken)
 	ctx.checkSessionType(db.SessionTypeBind)
 	ctx.checkSessionFingerprints(ctx.req.Fingerprints)
 	ctx.verifyEdgeRouterAccess()

controller/handler_edge_ctrl/create_terminator_v2.go

@@ -66,7 +66,7 @@ func (self *createTerminatorV2Handler) HandleReceive(msg *channel.Message, ch ch
 	}
 
 	ctx := &CreateTerminatorV2RequestContext{
-		baseSessionRequestContext: baseSessionRequestContext{handler: self, msg: msg},
+		baseSessionRequestContext: baseSessionRequestContext{handler: self, msg: msg, env: self.appEnv},
 		req:                       req,
 	}
 
@@ -84,7 +84,7 @@ func (self *createTerminatorV2Handler) CreateTerminatorV2(ctx *CreateTerminatorV
 		return
 	}
 	ctx.verifyTerminatorId(ctx.req.Address)
-	ctx.loadSession(ctx.req.SessionToken)
+	ctx.loadSession(ctx.req.SessionToken, ctx.req.ApiSessionToken)
 	ctx.checkSessionType(db.SessionTypeBind)
 	ctx.checkSessionFingerprints(ctx.req.Fingerprints)
 	ctx.verifyEdgeRouterAccess()

controller/handler_edge_ctrl/create_tunnel_api_session.go

@@ -43,7 +43,7 @@ func (self *createApiSessionHandler) HandleReceive(msg *channel.Message, _ chann
 
 	ctx := &createApiSessionRequestContext{
 		baseTunnelRequestContext: baseTunnelRequestContext{
-			baseSessionRequestContext: baseSessionRequestContext{handler: self, msg: msg},
+			baseSessionRequestContext: baseSessionRequestContext{handler: self, msg: msg, env: self.appEnv},
 		},
 		req: req,
 	}

controller/handler_edge_ctrl/create_tunnel_circuit.go

@@ -74,7 +74,7 @@ func (self *createCircuitForServiceHandler) HandleReceive(msg *channel.Message,
 
 	ctx := &CreateCircuitForServiceRequestContext{
 		baseTunnelRequestContext: baseTunnelRequestContext{
-			baseSessionRequestContext: baseSessionRequestContext{handler: self, msg: msg},
+			baseSessionRequestContext: baseSessionRequestContext{handler: self, msg: msg, env: self.appEnv},
 		},
 		req: req,
 	}

controller/handler_edge_ctrl/create_tunnel_terminator.go

@@ -59,7 +59,6 @@ func (self *createTunnelTerminatorHandler) Label() string {
 
 func (self *createTunnelTerminatorHandler) HandleReceive(msg *channel.Message, ch channel.Channel) {
 	startTime := time.Now()
-
 	req := &edge_ctrl_pb.CreateTunnelTerminatorRequest{}
 	if err := proto.Unmarshal(msg.Body, req); err != nil {
 		pfxlog.ContextLogger(ch.Label()).WithError(err).Error("could not unmarshal CreateTerminatorRequest")
@@ -68,7 +67,7 @@ func (self *createTunnelTerminatorHandler) HandleReceive(msg *channel.Message, c
 
 	ctx := &CreateTunnelTerminatorRequestContext{
 		baseTunnelRequestContext: baseTunnelRequestContext{
-			baseSessionRequestContext: baseSessionRequestContext{handler: self, msg: msg},
+			baseSessionRequestContext: baseSessionRequestContext{handler: self, msg: msg, env: self.appEnv},
 			apiSession:                nil,
 			identity:                  nil,
 		},

controller/handler_edge_ctrl/health_event.go

@@ -61,7 +61,7 @@ func (self *healthEventHandler) HandleReceive(msg *channel.Message, ch channel.C
 	}
 
 	ctx := &HealthEventRequestContext{
-		baseSessionRequestContext: baseSessionRequestContext{handler: self, msg: msg},
+		baseSessionRequestContext: baseSessionRequestContext{handler: self, msg: msg, env: self.appEnv},
 		req:                       req,
 	}
 
@@ -73,7 +73,7 @@ func (self *healthEventHandler) handleHealthEvent(ctx *HealthEventRequestContext
 		return
 	}
 
-	ctx.loadSession(ctx.req.SessionToken)
+	ctx.loadSession(ctx.req.SessionToken, ctx.req.ApiSessionToken)
 	ctx.checkSessionType(db.SessionTypeBind)
 	ctx.checkSessionFingerprints(ctx.req.Fingerprints)
 

controller/handler_edge_ctrl/hello.go

@@ -26,10 +26,10 @@ import (
 
 type helloHandler struct {
 	appEnv   *env.AppEnv
-	callback func(routerId string, respHello *edge_ctrl_pb.ClientHello)
+	callback func(routerId string, msg *channel.Message, respHello *edge_ctrl_pb.ClientHello)
 }
 
-func NewHelloHandler(appEnv *env.AppEnv, callback func(routerId string, respHello *edge_ctrl_pb.ClientHello)) *helloHandler {
+func NewHelloHandler(appEnv *env.AppEnv, callback func(routerId string, msg *channel.Message, respHello *edge_ctrl_pb.ClientHello)) *helloHandler {
 	return &helloHandler{
 		appEnv:   appEnv,
 		callback: callback,
@@ -47,5 +47,5 @@ func (h *helloHandler) HandleReceive(msg *channel.Message, ch channel.Channel) {
 		return
 	}
 
-	h.callback(ch.Id(), respHello)
+	h.callback(ch.Id(), msg, respHello)
 }

controller/handler_edge_ctrl/list_tunnel_services.go

@@ -40,7 +40,7 @@ func (self *listTunnelServicesHandler) HandleReceive(msg *channel.Message, _ cha
 
 	ctx := &listTunnelServicesRequestContext{
 		baseTunnelRequestContext: baseTunnelRequestContext{
-			baseSessionRequestContext: baseSessionRequestContext{handler: self, msg: msg},
+			baseSessionRequestContext: baseSessionRequestContext{handler: self, msg: msg, env: self.appEnv},
 		},
 	}
 

controller/handler_edge_ctrl/remove_terminator.go

@@ -65,7 +65,7 @@ func (self *removeTerminatorHandler) HandleReceive(msg *channel.Message, ch chan
 	}
 
 	ctx := &RemoveTerminatorRequestContext{
-		baseSessionRequestContext: baseSessionRequestContext{handler: self, msg: msg},
+		baseSessionRequestContext: baseSessionRequestContext{handler: self, msg: msg, env: self.appEnv},
 		req:                       req,
 	}
 

controller/handler_edge_ctrl/remove_tunnel_terminator.go

@@ -47,7 +47,7 @@ func (self *removeTunnelTerminatorHandler) Label() string {
 
 func (self *removeTunnelTerminatorHandler) HandleReceive(msg *channel.Message, _ channel.Channel) {
 	ctx := &RemoveTunnelTerminatorRequestContext{
-		baseSessionRequestContext: baseSessionRequestContext{handler: self, msg: msg},
+		baseSessionRequestContext: baseSessionRequestContext{handler: self, msg: msg, env: self.appEnv},
 		terminatorId:              string(msg.Body),
 	}
 

controller/handler_edge_ctrl/tunnel_health_event.go

@@ -18,10 +18,10 @@ package handler_edge_ctrl
 
 import (
 	"github.com/openziti/channel/v2"
+	"github.com/openziti/metrics"
 	"github.com/openziti/ziti/common"
 	"github.com/openziti/ziti/common/pb/edge_ctrl_pb"
 	"github.com/openziti/ziti/controller/env"
-	"github.com/openziti/metrics"
 	"time"
 )
 
@@ -56,7 +56,7 @@ func (self *tunnelHealthEventHandler) HandleReceive(msg *channel.Message, _ chan
 	checkPassed, _ := msg.GetBoolHeader(int32(edge_ctrl_pb.Header_CheckPassed))
 
 	ctx := &TunnelHealthEventRequestContext{
-		baseSessionRequestContext: baseSessionRequestContext{handler: self, msg: msg},
+		baseSessionRequestContext: baseSessionRequestContext{handler: self, msg: msg, env: self.appEnv},
 		terminatorId:              terminatorId,
 		checkPassed:               checkPassed,
 	}

controller/handler_edge_ctrl/update_terminator.go

@@ -56,7 +56,7 @@ func (self *updateTerminatorHandler) HandleReceive(msg *channel.Message, ch chan
 	}
 
 	ctx := &UpdateTerminatorRequestContext{
-		baseSessionRequestContext: baseSessionRequestContext{handler: self, msg: msg},
+		baseSessionRequestContext: baseSessionRequestContext{handler: self, msg: msg, env: self.appEnv},
 		req:                       req,
 	}
 
@@ -79,7 +79,7 @@ func (self *updateTerminatorHandler) UpdateTerminator(ctx *UpdateTerminatorReque
 
 	logger.Debug("update request received")
 
-	ctx.loadSession(ctx.req.SessionToken)
+	ctx.loadSession(ctx.req.SessionToken, ctx.req.ApiSessionToken)
 	ctx.checkSessionType(db.SessionTypeBind)
 	ctx.checkSessionFingerprints(ctx.req.Fingerprints)
 	terminator := ctx.verifyTerminator(ctx.req.TerminatorId, common.EdgeBinding)

controller/handler_edge_ctrl/update_tunnel_terminator.go

@@ -55,7 +55,7 @@ func (self *updateTunnelTerminatorHandler) HandleReceive(msg *channel.Message, c
 	}
 
 	ctx := &UpdateTunnelTerminatorRequestContext{
-		baseSessionRequestContext: baseSessionRequestContext{handler: self, msg: msg},
+		baseSessionRequestContext: baseSessionRequestContext{handler: self, msg: msg, env: self.appEnv},
 		req:                       req,
 	}
 

controller/internal/policy/service_policy_enforcer.go

@@ -78,7 +78,7 @@ func (enforcer *ServicePolicyEnforcer) handleServiceEvent(event *db.ServiceEvent
 	var sessionsToDelete []string
 
 	err := enforcer.appEnv.GetDbProvider().GetDb().View(func(tx *bbolt.Tx) error {
-		identity, err := enforcer.appEnv.GetStores().Identity.LoadOneById(tx, event.IdentityId)
+		identity, err := enforcer.appEnv.GetStores().Identity.LoadById(tx, event.IdentityId)
 		if err != nil {
 			return err
 		}
@@ -134,12 +134,12 @@ func (enforcer *ServicePolicyEnforcer) Run() error {
 	var sessionsToRemove []string
 	err = enforcer.appEnv.GetDbProvider().GetDb().View(func(tx *bbolt.Tx) error {
 		for _, session := range result.Sessions {
-			apiSession, err := enforcer.appEnv.GetStores().ApiSession.LoadOneById(tx, session.ApiSessionId)
+			apiSession, err := enforcer.appEnv.GetStores().ApiSession.LoadById(tx, session.ApiSessionId)
 			if err != nil {
 				return err
 			}
 
-			identity, err := enforcer.appEnv.GetStores().Identity.LoadOneById(tx, apiSession.IdentityId)
+			identity, err := enforcer.appEnv.GetStores().Identity.LoadById(tx, apiSession.IdentityId)
 			if err != nil {
 				return err
 			}

controller/internal/routes/ca_router.go

@@ -266,11 +266,13 @@ func (r *CaRouter) generateJwt(ae *env.AppEnv, rc *response.RequestContext) {
 	claims := &ziti.EnrollmentClaims{
 		EnrollmentMethod: method,
 		RegisteredClaims: jwt.RegisteredClaims{
-			Issuer: fmt.Sprintf(`https://%s/`, ae.Config.Api.Address),
+			Issuer:  fmt.Sprintf(`https://%s/`, ae.Config.Api.Address),
+			Subject: ca.Id,
+			ID:      ca.Id,
 		},
 	}
 
-	jwtStr, genErr := ae.GetJwtSigner().Generate(ca.Id, ca.Id, claims)
+	jwtStr, genErr := ae.GetServerJwtSigner().Generate(claims)
 
 	if genErr != nil {
 		rc.RespondWithError(errors.New("could not generate claims"))

controller/internal/routes/controller_api_model.go

@@ -0,0 +1,49 @@
+/*
+	Copyright NetFoundry Inc.
+
+	Licensed under the Apache License, Version 2.0 (the "License");
+	you may not use this file except in compliance with the License.
+	You may obtain a copy of the License at
+
+	https://www.apache.org/licenses/LICENSE-2.0
+
+	Unless required by applicable law or agreed to in writing, software
+	distributed under the License is distributed on an "AS IS" BASIS,
+	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+	See the License for the specific language governing permissions and
+	limitations under the License.
+*/
+
+package routes
+
+import (
+	"github.com/openziti/edge-api/rest_model"
+	"github.com/openziti/ziti/controller/env"
+	"github.com/openziti/ziti/controller/model"
+	"github.com/openziti/ziti/controller/response"
+)
+
+const EntityNameController = "controllers"
+
+var ControllerLinkFactory = NewBasicLinkFactory(EntityNameController)
+
+func MapControllerToRestEntity(_ *env.AppEnv, _ *response.RequestContext, Controller *model.Controller) (interface{}, error) {
+	return MapControllerToRestModel(Controller)
+}
+
+func MapControllerToRestModel(controller *model.Controller) (*rest_model.ControllerDetail, error) {
+	ret := &rest_model.ControllerDetail{
+		BaseEntity:  BaseEntityToRestModel(controller, ControllerLinkFactory),
+		Name:        &controller.Name,
+		Address:     &controller.Address,
+		CertPem:     &controller.CertPem,
+		Fingerprint: &controller.Fingerprint,
+		IsOnline:    &controller.IsOnline,
+	}
+
+	if controller.LastJoinedAt != nil {
+		ret.LastJoinedAt = toStrFmtDateTimeP(*controller.LastJoinedAt)
+	}
+
+	return ret, nil
+}

controller/internal/routes/controller_router.go

@@ -0,0 +1,56 @@
+/*
+	Copyright NetFoundry Inc.
+
+	Licensed under the Apache License, Version 2.0 (the "License");
+	you may not use this file except in compliance with the License.
+	You may obtain a copy of the License at
+
+	https://www.apache.org/licenses/LICENSE-2.0
+
+	Unless required by applicable law or agreed to in writing, software
+	distributed under the License is distributed on an "AS IS" BASIS,
+	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+	See the License for the specific language governing permissions and
+	limitations under the License.
+*/
+
+package routes
+
+import (
+	"github.com/go-openapi/runtime/middleware"
+	controllersClient "github.com/openziti/edge-api/rest_client_api_server/operations/controllers"
+	controllersMan "github.com/openziti/edge-api/rest_management_api_server/operations/controllers"
+	"github.com/openziti/ziti/controller/env"
+	"github.com/openziti/ziti/controller/internal/permissions"
+	"github.com/openziti/ziti/controller/model"
+	"github.com/openziti/ziti/controller/response"
+)
+
+func init() {
+	r := NewControllerRouter()
+	env.AddRouter(r)
+}
+
+type ControllerRouter struct {
+	BasePath string
+}
+
+func NewControllerRouter() *ControllerRouter {
+	return &ControllerRouter{
+		BasePath: "/" + EntityNameController,
+	}
+}
+
+func (r *ControllerRouter) Register(ae *env.AppEnv) {
+	ae.ManagementApi.ControllersListControllersHandler = controllersMan.ListControllersHandlerFunc(func(params controllersMan.ListControllersParams, _ interface{}) middleware.Responder {
+		return ae.IsAllowed(r.List, params.HTTPRequest, "", "", permissions.IsAdmin())
+	})
+
+	ae.ClientApi.ControllersListControllersHandler = controllersClient.ListControllersHandlerFunc(func(params controllersClient.ListControllersParams, _ interface{}) middleware.Responder {
+		return ae.IsAllowed(r.List, params.HTTPRequest, "", "", permissions.IsAdmin())
+	})
+}
+
+func (r *ControllerRouter) List(ae *env.AppEnv, rc *response.RequestContext) {
+	ListWithHandler[*model.Controller](ae, rc, ae.Managers.Controller, MapControllerToRestEntity)
+}

controller/internal/routes/current_identity_router.go

@@ -18,20 +18,15 @@ package routes
 
 import (
 	"github.com/go-openapi/runtime/middleware"
-	"github.com/michaelquigley/pfxlog"
 	clientCurrentIdentity "github.com/openziti/edge-api/rest_client_api_server/operations/current_identity"
 	managementCurrentIdentity "github.com/openziti/edge-api/rest_management_api_server/operations/current_identity"
 	"github.com/openziti/edge-api/rest_model"
-	"github.com/openziti/foundation/v2/errorz"
 	"github.com/openziti/foundation/v2/stringz"
-	"github.com/openziti/storage/boltz"
 	"github.com/openziti/ziti/controller/apierror"
-	"github.com/openziti/ziti/controller/db"
 	"github.com/openziti/ziti/controller/env"
 	"github.com/openziti/ziti/controller/internal/permissions"
 	"github.com/openziti/ziti/controller/model"
 	"github.com/openziti/ziti/controller/response"
-	"github.com/pkg/errors"
 	"net/http"
 )
 
@@ -141,54 +136,23 @@ func (r *CurrentIdentityRouter) Register(ae *env.AppEnv) {
 }
 
 func (r *CurrentIdentityRouter) verifyMfa(ae *env.AppEnv, rc *response.RequestContext, body *rest_model.MfaCode) {
-	mfa, err := ae.Managers.Mfa.ReadOneByIdentityId(rc.Identity.Id)
+	changeCtx := rc.NewChangeContext()
+	err := ae.Managers.Mfa.CompleteTotpEnrollment(rc.Identity.Id, *body.Code, changeCtx)
 
 	if err != nil {
 		rc.RespondWithError(err)
 		return
 	}
 
-	if mfa == nil {
-		rc.RespondWithNotFound()
-		return
-	}
-
-	if mfa.IsVerified {
-		rc.RespondWithNotFound()
-		return
-	}
-
-	ok, err := ae.Managers.Mfa.VerifyTOTP(mfa, *body.Code)
+	err = ae.Managers.ApiSession.SetMfaPassed(rc.ApiSession, changeCtx)
 
 	if err != nil {
-		rc.RespondWithError(apierror.NewInvalidMfaTokenError())
+		rc.RespondWithError(err)
 		return
 	}
 
-	if ok {
-		changeCtx := rc.NewChangeContext()
-		mfa.IsVerified = true
-		if err := ae.Managers.Mfa.Update(mfa, nil, changeCtx); err != nil {
-			pfxlog.Logger().Errorf("could not update MFA with new MFA status: %v", err)
-			rc.RespondWithApiError(errorz.NewUnhandled(errors.New("could not update MFA status")))
-			return
-		}
+	rc.RespondWithEmptyOk()
 
-		rc.ApiSession.MfaComplete = true
-		rc.ApiSession.MfaRequired = true
-
-		if err := ae.Managers.ApiSession.UpdateWithFieldChecker(rc.ApiSession, boltz.MapFieldChecker{db.FieldApiSessionMfaComplete: struct{}{}, db.FieldApiSessionMfaRequired: struct{}{}}, changeCtx); err != nil {
-			pfxlog.Logger().Errorf("could not update API Session with new MFA status: %v", err)
-		}
-
-		ae.Managers.PostureResponse.SetMfaPosture(rc.Identity.Id, rc.ApiSession.Id, true)
-
-		rc.RespondWithEmptyOk()
-		return
-
-	}
-
-	rc.RespondWithError(apierror.NewInvalidMfaTokenError())
 }
 
 func (r *CurrentIdentityRouter) createMfa(ae *env.AppEnv, rc *response.RequestContext) {

controller/internal/routes/service_api_model.go

@@ -17,6 +17,7 @@
 package routes
 
 import (
+	"fmt"
 	"github.com/michaelquigley/pfxlog"
 	"github.com/openziti/edge-api/rest_model"
 	"github.com/openziti/foundation/v2/stringz"
@@ -148,8 +149,20 @@ func MapServiceToRestModel(ae *env.AppEnv, rc *response.RequestContext, service
 
 	policyPostureCheckMap := ae.GetManagers().EdgeService.GetPolicyPostureChecks(rc.Identity.Id, *ret.ID)
 
-	for policyId, policyPostureChecks := range policyPostureCheckMap {
+	if len(policyPostureCheckMap) == 0 {
+		for _, permission := range ret.Permissions {
+			passing := true
+			id := fmt.Sprintf("dummy %s policy: no posture checks defined", strings.ToLower(string(permission)))
+			ret.PostureQueries = append(ret.PostureQueries, &rest_model.PostureQueries{
+				PolicyID:       &id,
+				PostureQueries: []*rest_model.PostureQuery{},
+				PolicyType:     permission,
+				IsPassing:      &passing,
+			})
+		}
+	}
 
+	for policyId, policyPostureChecks := range policyPostureCheckMap {
 		isPolicyPassing := true
 		policyIdCopy := policyId
 		querySet := &rest_model.PostureQueries{

controller/internal/routes/service_router.go

@@ -22,6 +22,8 @@ import (
 	"github.com/michaelquigley/pfxlog"
 	clientService "github.com/openziti/edge-api/rest_client_api_server/operations/service"
 	managementService "github.com/openziti/edge-api/rest_management_api_server/operations/service"
+	"github.com/openziti/edge-api/rest_model"
+	"github.com/openziti/foundation/v2/errorz"
 	"github.com/openziti/metrics"
 	"github.com/openziti/storage/boltz"
 	"github.com/openziti/ziti/controller/db"
@@ -69,6 +71,12 @@ func (r *ServiceRouter) Register(ae *env.AppEnv) {
 		return ae.IsAllowed(r.listClientTerminators, params.HTTPRequest, params.ID, "", permissions.IsAuthenticated())
 	})
 
+	ae.ClientApi.ServiceListServiceEdgeRoutersHandler = clientService.ListServiceEdgeRoutersHandlerFunc(func(params clientService.ListServiceEdgeRoutersParams, i interface{}) middleware.Responder {
+		return ae.IsAllowed(func(ae *env.AppEnv, rc *response.RequestContext) {
+			r.listClientEdgeRouters(ae, rc, params)
+		}, params.HTTPRequest, params.ID, "", permissions.IsAuthenticated())
+	})
+
 	//Management
 	ae.ManagementApi.ServiceDeleteServiceHandler = managementService.DeleteServiceHandlerFunc(func(params managementService.DeleteServiceParams, _ interface{}) middleware.Responder {
 		return ae.IsAllowed(r.Delete, params.HTTPRequest, params.ID, "", permissions.IsAdmin())
@@ -316,3 +324,71 @@ func (r *ServiceRouter) listEdgeRouters(ae *env.AppEnv, rc *response.RequestCont
 	filterTemplate := `not isEmpty(from serviceEdgeRouterPolicies where anyOf(services) = "%v")`
 	ListAssociationsWithFilter[*model.EdgeRouter](ae, rc, filterTemplate, ae.Managers.EdgeRouter, MapEdgeRouterToRestEntity)
 }
+
+func (r *ServiceRouter) listClientEdgeRouters(ae *env.AppEnv, rc *response.RequestContext, params clientService.ListServiceEdgeRoutersParams) {
+	serviceId, err := rc.GetEntityId()
+
+	if err != nil {
+		rc.RespondWithError(err)
+		return
+	}
+
+	if params.SessionToken != nil {
+		_, err := ae.ValidateServiceAccessToken(*params.SessionToken, &rc.ApiSession.Id)
+		if err != nil {
+			apiErr := errorz.NewUnauthorized()
+			apiErr.Cause = err
+			rc.RespondWithError(apiErr)
+			return
+		}
+	}
+
+	svc, err := ae.Managers.EdgeService.ReadForIdentity(serviceId, rc.Identity.Id, nil)
+
+	if err != nil {
+		if boltz.IsErrNotFoundErr(err) {
+			rc.RespondWithNotFound()
+		} else {
+			rc.RespondWithError(err)
+		}
+		return
+	}
+
+	if svc == nil {
+		rc.RespondWithNotFound()
+		return
+	}
+
+	Detail(rc, func(rc *response.RequestContext, id string) (interface{}, error) {
+		return getServiceEdgeRouters(ae, rc, serviceId)
+	})
+}
+
+func getServiceEdgeRouters(ae *env.AppEnv, rc *response.RequestContext, serviceId string) (*rest_model.ServiceEdgeRouters, error) {
+	edgeRouters := &rest_model.ServiceEdgeRouters{}
+
+	edgeRoutersForSvc, err := ae.Managers.EdgeRouter.ListForIdentityAndService(rc.Identity.Id, serviceId, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, edgeRouter := range edgeRoutersForSvc.EdgeRouters {
+		state := ae.Broker.GetEdgeRouterState(edgeRouter.Id)
+
+		syncStatus := string(state.SyncStatus)
+		cost := int64(edgeRouter.Cost)
+		er := &rest_model.CommonEdgeRouterProperties{
+			Hostname:           &state.Hostname,
+			IsOnline:           &state.IsOnline,
+			Name:               &edgeRouter.Name,
+			SupportedProtocols: state.Protocols,
+			SyncStatus:         &syncStatus,
+			Cost:               &cost,
+			NoTraversal:        &edgeRouter.NoTraversal,
+			Disabled:           &edgeRouter.Disabled,
+		}
+		edgeRouters.EdgeRouters = append(edgeRouters.EdgeRouters, er)
+	}
+
+	return edgeRouters, nil
+}

controller/internal/routes/session_router.go

@@ -18,14 +18,16 @@ package routes
 
 import (
 	"github.com/go-openapi/runtime/middleware"
+	"github.com/michaelquigley/pfxlog"
 	"github.com/openziti/edge-api/rest_client_api_server/operations/session"
 	clientSession "github.com/openziti/edge-api/rest_client_api_server/operations/session"
 	managementSession "github.com/openziti/edge-api/rest_management_api_server/operations/session"
 	"github.com/openziti/edge-api/rest_model"
+	"github.com/openziti/metrics"
 	"github.com/openziti/ziti/controller/env"
 	"github.com/openziti/ziti/controller/internal/permissions"
 	"github.com/openziti/ziti/controller/response"
-	"github.com/openziti/metrics"
+	"net/http"
 	"time"
 )
 
@@ -123,10 +125,45 @@ func (r *SessionRouter) Delete(ae *env.AppEnv, rc *response.RequestContext) {
 
 func (r *SessionRouter) Create(ae *env.AppEnv, rc *response.RequestContext, params session.CreateSessionParams) {
 	start := time.Now()
-	responder := &SessionRequestResponder{ae: ae, Responder: rc}
-	CreateWithResponder(rc, responder, SessionLinkFactory, func() (string, error) {
-		return ae.Managers.Session.Create(MapCreateSessionToModel(rc.Identity.Id, rc.ApiSession.Id, params.Session), rc.NewChangeContext())
-	})
+	if rc.Claims != nil {
+		request := MapCreateSessionToModel(rc.Claims.Subject, rc.Claims.ApiSessionId, params.Session)
+		token, err := ae.Managers.Session.CreateJwt(request, rc.NewChangeContext())
+
+		if err != nil {
+			rc.RespondWithError(err)
+			return
+		}
+
+		edgeRouters, err := getSessionEdgeRouters(ae, request)
+		if err != nil {
+			pfxlog.Logger().Errorf("could not render edge routers for jwt session: %s", err)
+		}
+
+		sessionType := rest_model.DialBind(request.Type)
+
+		newSessionEnvelope := &rest_model.SessionCreateEnvelope{
+			Data: &rest_model.SessionDetail{
+				BaseEntity: rest_model.BaseEntity{
+					ID: &request.Id,
+				},
+				APISessionID: &rc.ApiSession.Id,
+				EdgeRouters:  edgeRouters,
+				IdentityID:   &rc.Identity.Id,
+				ServiceID:    &request.ServiceId,
+				Token:        &token,
+				Type:         &sessionType,
+			},
+			Meta: &rest_model.Meta{},
+		}
+
+		rc.Respond(newSessionEnvelope, http.StatusCreated)
+	} else {
+		responder := &SessionRequestResponder{ae: ae, Responder: rc}
+		CreateWithResponder(rc, responder, SessionLinkFactory, func() (string, error) {
+			return ae.Managers.Session.Create(MapCreateSessionToModel(rc.Identity.Id, rc.ApiSession.Id, params.Session), rc.NewChangeContext())
+		})
+	}
+
 	r.createTimer.UpdateSince(start)
 }
 

controller/jwtsigner/jwt.go

@@ -17,30 +17,39 @@
 package jwtsigner
 
 import (
+	"crypto"
 	"github.com/golang-jwt/jwt/v5"
 )
 
 type Signer interface {
-	Generate(string, string, jwt.Claims) (string, error)
+	Generate(jwt.Claims) (string, error)
+	SigningMethod() jwt.SigningMethod
+	KeyId() string
 }
 
-type IdentitySigner struct {
+type SignerImpl struct {
 	signingMethod jwt.SigningMethod
-	issuer        string
-	key           interface{}
+	key           crypto.PrivateKey
 	keyId         string
 }
 
-func New(issuer string, sm jwt.SigningMethod, key interface{}, keyId string) *IdentitySigner {
-	return &IdentitySigner{
+func New(sm jwt.SigningMethod, key crypto.PrivateKey, keyId string) *SignerImpl {
+	return &SignerImpl{
 		signingMethod: sm,
-		issuer:        issuer,
 		key:           key,
 		keyId:         keyId,
 	}
 }
 
-func (j *IdentitySigner) Generate(subj, jti string, claims jwt.Claims) (string, error) {
+func (j *SignerImpl) SigningMethod() jwt.SigningMethod {
+	return j.signingMethod
+}
+
+func (j *SignerImpl) KeyId() string {
+	return j.keyId
+}
+
+func (j *SignerImpl) Generate(claims jwt.Claims) (string, error) {
 	token := jwt.NewWithClaims(j.signingMethod, claims)
 
 	if j.keyId != "" {

controller/model/api_session_certificate_manager.go

@@ -18,7 +18,6 @@ package model
 
 import (
 	"crypto/x509"
-	"fmt"
 	"github.com/openziti/ziti/common/cert"
 	"github.com/openziti/ziti/controller/apierror"
 	"github.com/openziti/ziti/controller/change"
@@ -111,13 +110,18 @@ func (self *ApiSessionCertificateManager) Query(tx *bbolt.Tx, query string) (*Ap
 }
 
 func (self *ApiSessionCertificateManager) ReadByApiSessionId(tx *bbolt.Tx, apiSessionId string) ([]*ApiSessionCertificate, error) {
-	result, err := self.Query(tx, fmt.Sprintf(`apiSession = "%s"`, apiSessionId))
+	var result []*ApiSessionCertificate
 
-	if err != nil {
-		return nil, err
+	certIds := self.env.GetStores().ApiSession.GetRelatedEntitiesIdList(tx, apiSessionId, db.EntityTypeApiSessionCertificates)
+	for _, key := range certIds {
+		apiSessionCert, err := self.readInTx(tx, key)
+		if err != nil {
+			return nil, err
+		}
+		result = append(result, apiSessionCert)
 	}
 
-	return result.ApiSessionCertificates, nil
+	return result, nil
 }
 
 type ApiSessionCertificateListResult struct {

controller/model/api_session_certificate_model.go

@@ -86,7 +86,7 @@ func (entity *ApiSessionCertificate) fillFrom(env Env, tx *bbolt.Tx, boltApiSess
 	entity.PEM = boltApiSessionCertificate.PEM
 	entity.ApiSessionId = boltApiSessionCertificate.ApiSessionId
 
-	boltApiSession, err := env.GetStores().ApiSession.LoadOneById(tx, boltApiSessionCertificate.ApiSessionId)
+	boltApiSession, err := env.GetStores().ApiSession.LoadById(tx, boltApiSessionCertificate.ApiSessionId)
 	if err != nil {
 		return err
 	}

controller/model/api_session_manager.go

@@ -124,6 +124,7 @@ func (self *ApiSessionManager) UpdateWithFieldChecker(apiSession *ApiSession, fi
 
 func (self *ApiSessionManager) MfaCompleted(apiSession *ApiSession, ctx *change.Context) error {
 	apiSession.MfaComplete = true
+
 	return self.updateEntity(apiSession, &OrFieldChecker{NewFieldChecker(db.FieldApiSessionMfaComplete), self}, ctx.NewMutateContext())
 }
 
@@ -295,6 +296,20 @@ func (self *ApiSessionManager) DeleteByIdentityId(identityId string, changeCtx *
 	})
 }
 
+func (self *ApiSessionManager) SetMfaPassed(apiSession *ApiSession, changeCtx *change.Context) error {
+	apiSession.MfaComplete = true
+	apiSession.MfaRequired = true
+
+	if err := self.UpdateWithFieldChecker(apiSession, boltz.MapFieldChecker{db.FieldApiSessionMfaComplete: struct{}{}, db.FieldApiSessionMfaRequired: struct{}{}}, changeCtx); err != nil {
+		pfxlog.Logger().Errorf("could not update API Session with new MFA status: %v", err)
+		return err
+	}
+
+	self.env.GetManagers().PostureResponse.SetMfaPosture(apiSession.IdentityId, apiSession.Id, true)
+
+	return nil
+}
+
 type ApiSessionListResult struct {
 	manager     *ApiSessionManager
 	ApiSessions []*ApiSession

controller/model/api_session_model.go

@@ -82,7 +82,7 @@ func (entity *ApiSession) fillFrom(env Env, tx *bbolt.Tx, boltApiSession *db.Api
 	entity.LastActivityAt = boltApiSession.LastActivityAt
 	entity.AuthenticatorId = boltApiSession.AuthenticatorId
 
-	boltIdentity, err := env.GetStores().Identity.LoadOneById(tx, boltApiSession.IdentityId)
+	boltIdentity, err := env.GetStores().Identity.LoadById(tx, boltApiSession.IdentityId)
 	if err != nil {
 		return err
 	}

controller/model/authenticator_manager.go

@@ -87,7 +87,7 @@ func (self *AuthenticatorManager) ReadFingerprints(authenticatorId string) ([]st
 
 	err := self.env.GetDbProvider().GetDb().View(func(tx *bbolt.Tx) error {
 		var err error
-		authenticator, err = self.authStore.LoadOneById(tx, authenticatorId)
+		authenticator, err = self.authStore.LoadById(tx, authenticatorId)
 		return err
 	})
 
@@ -677,7 +677,7 @@ func getCaId(env Env, auth *AuthenticatorCert) string {
 	caId := ""
 	err := env.GetDbProvider().GetDb().View(func(tx *bbolt.Tx) error {
 		for cursor := env.GetStores().Ca.IterateIds(tx, ast.BoolNodeTrue); cursor.IsValid(); cursor.Next() {
-			ca, err := env.GetStores().Ca.LoadOneById(tx, string(cursor.Current()))
+			ca, err := env.GetStores().Ca.LoadById(tx, string(cursor.Current()))
 			if err != nil {
 				continue
 			}

controller/model/authenticator_mod_ext_jwt.go

@@ -566,7 +566,7 @@ func (a *AuthModuleExtJwt) loadExistingSigners() {
 		}
 
 		for _, id := range ids {
-			signer, err := a.env.GetStores().ExternalJwtSigner.LoadOneById(tx, id)
+			signer, err := a.env.GetStores().ExternalJwtSigner.LoadById(tx, id)
 			if err != nil {
 				return err
 			}

controller/model/controller_manager.go

@@ -0,0 +1,246 @@
+/*
+	Copyright NetFoundry Inc.
+
+	Licensed under the Apache License, Version 2.0 (the "License");
+	you may not use this file except in compliance with the License.
+	You may obtain a copy of the License at
+
+	https://www.apache.org/licenses/LICENSE-2.0
+
+	Unless required by applicable law or agreed to in writing, software
+	distributed under the License is distributed on an "AS IS" BASIS,
+	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+	See the License for the specific language governing permissions and
+	limitations under the License.
+*/
+
+package model
+
+import (
+	"github.com/michaelquigley/pfxlog"
+	nfpem "github.com/openziti/foundation/v2/pem"
+	"github.com/openziti/foundation/v2/stringz"
+	"github.com/openziti/storage/boltz"
+	"github.com/openziti/ziti/common/pb/edge_cmd_pb"
+	"github.com/openziti/ziti/controller/change"
+	"github.com/openziti/ziti/controller/command"
+	"github.com/openziti/ziti/controller/db"
+	"github.com/openziti/ziti/controller/event"
+	"github.com/openziti/ziti/controller/fields"
+	"github.com/openziti/ziti/controller/models"
+	"github.com/openziti/ziti/controller/network"
+	"go.etcd.io/bbolt"
+	"google.golang.org/protobuf/proto"
+	"time"
+)
+
+func NewControllerManager(env Env) *ControllerManager {
+	manager := &ControllerManager{
+		baseEntityManager: newBaseEntityManager[*Controller, *db.Controller](env, env.GetStores().Controller),
+	}
+	manager.impl = manager
+
+	network.RegisterManagerDecoder[*Controller](env.GetHostController().GetNetwork().Managers, manager)
+
+	return manager
+}
+
+type ControllerManager struct {
+	baseEntityManager[*Controller, *db.Controller]
+}
+
+func (self *ControllerManager) newModelEntity() *Controller {
+	return &Controller{}
+}
+
+func (self *ControllerManager) Create(entity *Controller, ctx *change.Context) error {
+	return network.DispatchCreate[*Controller](self, entity, ctx)
+}
+
+func (self *ControllerManager) ApplyCreate(cmd *command.CreateEntityCommand[*Controller], ctx boltz.MutateContext) error {
+	_, err := self.createEntity(cmd.Entity, ctx)
+	return err
+}
+
+func (self *ControllerManager) Update(entity *Controller, checker fields.UpdatedFields, ctx *change.Context) error {
+	return network.DispatchUpdate[*Controller](self, entity, checker, ctx)
+}
+
+func (self *ControllerManager) ApplyUpdate(cmd *command.UpdateEntityCommand[*Controller], ctx boltz.MutateContext) error {
+	return self.updateEntity(cmd.Entity, cmd.UpdatedFields, ctx)
+}
+
+func (self *ControllerManager) Read(id string) (*Controller, error) {
+	modelEntity := &Controller{}
+	if err := self.readEntity(id, modelEntity); err != nil {
+		return nil, err
+	}
+	return modelEntity, nil
+}
+
+func (self *ControllerManager) readInTx(tx *bbolt.Tx, id string) (*Controller, error) {
+	modelEntity := &Controller{}
+	if err := self.readEntityInTx(tx, id, modelEntity); err != nil {
+		return nil, err
+	}
+	return modelEntity, nil
+}
+
+func (self *ControllerManager) ReadByName(name string) (*Controller, error) {
+	modelEntity := &Controller{}
+	nameIndex := self.env.GetStores().Controller.GetNameIndex()
+	if err := self.readEntityWithIndex("name", []byte(name), nameIndex, modelEntity); err != nil {
+		return nil, err
+	}
+	return modelEntity, nil
+}
+
+func (self *ControllerManager) MapControllerNamesToIds(values []string, identityId string) map[string]struct{} {
+	var result []string
+	if stringz.Contains(values, "all") {
+		result = []string{"all"}
+	} else {
+		for _, val := range values {
+			if Controller, _ := self.Read(val); Controller != nil {
+				result = append(result, val)
+			} else if Controller, _ := self.ReadByName(val); Controller != nil {
+				result = append(result, Controller.Id)
+			} else {
+				pfxlog.Logger().Debugf("user %v submitted %v as a config type of interest, but no matching records found", identityId, val)
+			}
+		}
+	}
+	return stringz.SliceToSet(result)
+}
+
+func (self *ControllerManager) Marshall(entity *Controller) ([]byte, error) {
+	msg := &edge_cmd_pb.Controller{
+		Id:           entity.Id,
+		Name:         entity.Name,
+		Address:      entity.Address,
+		CertPem:      entity.CertPem,
+		Fingerprint:  entity.Fingerprint,
+		IsOnline:     entity.IsOnline,
+		LastJoinedAt: timePtrToPb(entity.LastJoinedAt),
+	}
+
+	return proto.Marshal(msg)
+}
+
+func (self *ControllerManager) Unmarshall(bytes []byte) (*Controller, error) {
+	msg := &edge_cmd_pb.Controller{}
+
+	if err := proto.Unmarshal(bytes, msg); err != nil {
+		return nil, err
+	}
+
+	return &Controller{
+		BaseEntity: models.BaseEntity{
+			Id:   msg.Id,
+			Tags: edge_cmd_pb.DecodeTags(msg.Tags),
+		},
+		Name:         msg.Name,
+		Address:      msg.Address,
+		CertPem:      msg.CertPem,
+		Fingerprint:  msg.Fingerprint,
+		IsOnline:     msg.IsOnline,
+		LastJoinedAt: pbTimeToTimePtr(msg.LastJoinedAt),
+	}, nil
+}
+
+func (self *ControllerManager) PeersConnected(peers []*event.ClusterPeer) {
+	var controllerIds []string
+	err := self.ListWithHandler("", func(tx *bbolt.Tx, ids []string, qmd *models.QueryMetaData) error {
+		controllerIds = ids
+		return nil
+	})
+
+	changeCtx := change.New()
+	changeCtx.SetSourceType("raft.peers.connected").
+		SetChangeAuthorType(change.AuthorTypeController)
+
+	if err != nil {
+		pfxlog.Logger().WithError(err).Error("could not list controllers to handle new peer(s) connection")
+		return
+	}
+
+	connectFields := fields.UpdatedFieldsMap{
+		db.FieldControllerLastJoinedAt: struct{}{},
+		db.FieldControllerCertPem:      struct{}{},
+		db.FieldControllerFingerprint:  struct{}{},
+		db.FieldControllerIsOnline:     struct{}{},
+		db.FieldControllerAddress:      struct{}{},
+	}
+
+	now := time.Now()
+
+	for _, peer := range peers {
+		if stringz.Contains(controllerIds, peer.Id) {
+			existing, err := self.Read(peer.Id)
+
+			if err != nil {
+				pfxlog.Logger().WithError(err).Error("could not handle new peer(s) connection, existing controller could not be read")
+				continue
+			}
+
+			existing.Address = peer.Addr
+			existing.IsOnline = true
+			existing.LastJoinedAt = &now
+
+			if len(peer.ServerCert) > 0 {
+				existing.CertPem = nfpem.EncodeToString(peer.ServerCert[0])
+				existing.Fingerprint = nfpem.FingerprintFromCertificate(peer.ServerCert[0])
+			}
+
+			if err := self.Update(existing, connectFields, changeCtx); err != nil {
+				pfxlog.Logger().WithError(err).Error("could not update controller during peer(s) connection")
+			}
+		} else {
+			if len(peer.ServerCert) == 0 {
+				pfxlog.Logger().Error("could not create controller during peer(s) connection, no server certificate provided")
+				continue
+			}
+
+			newController := &Controller{
+				BaseEntity: models.BaseEntity{
+					Id: peer.Id,
+				},
+				Address:      peer.Addr,
+				IsOnline:     true,
+				LastJoinedAt: &now,
+			}
+
+			newController.Name = peer.ServerCert[0].Subject.CommonName
+			newController.CertPem = nfpem.EncodeToString(peer.ServerCert[0])
+			newController.Fingerprint = nfpem.FingerprintFromCertificate(peer.ServerCert[0])
+
+			if err := self.Create(newController, changeCtx); err != nil {
+				pfxlog.Logger().WithError(err).Error("could not create controller during peer(s) connection")
+				continue
+			}
+		}
+	}
+}
+
+func (self *ControllerManager) PeersDisconnected(peers []*event.ClusterPeer) {
+	changeCtx := change.New()
+	changeCtx.SetSourceType("raft.peers.disconnected").
+		SetChangeAuthorType(change.AuthorTypeController)
+
+	disconnectFields := fields.UpdatedFieldsMap{
+		db.FieldControllerLastJoinedAt: struct{}{},
+		db.FieldControllerIsOnline:     struct{}{},
+	}
+	for _, peer := range peers {
+		controller := &Controller{
+			BaseEntity: models.BaseEntity{
+				Id: peer.Id,
+			},
+			IsOnline: false,
+		}
+
+		if err := self.Update(controller, disconnectFields, changeCtx); err != nil {
+			pfxlog.Logger().WithError(err).Error("could not update controller during peer(s) disconnection")
+		}
+	}
+}

controller/model/controller_model.go

@@ -0,0 +1,69 @@
+/*
+	Copyright NetFoundry Inc.
+
+	Licensed under the Apache License, Version 2.0 (the "License");
+	you may not use this file except in compliance with the License.
+	You may obtain a copy of the License at
+
+	https://www.apache.org/licenses/LICENSE-2.0
+
+	Unless required by applicable law or agreed to in writing, software
+	distributed under the License is distributed on an "AS IS" BASIS,
+	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+	See the License for the specific language governing permissions and
+	limitations under the License.
+*/
+
+package model
+
+import (
+	"github.com/openziti/storage/boltz"
+	"github.com/openziti/ziti/controller/db"
+	"github.com/openziti/ziti/controller/models"
+	"go.etcd.io/bbolt"
+	"time"
+)
+
+type Controller struct {
+	models.BaseEntity
+	Name         string
+	Address      string
+	CertPem      string
+	Fingerprint  string
+	IsOnline     bool
+	LastJoinedAt *time.Time
+}
+
+func (entity *Controller) toBoltEntity(tx *bbolt.Tx, env Env) (*db.Controller, error) {
+	boltEntity := &db.Controller{
+		BaseExtEntity: *boltz.NewExtEntity(entity.Id, entity.Tags),
+		Name:          entity.Name,
+		Address:       entity.Address,
+		CertPem:       entity.CertPem,
+		Fingerprint:   entity.Fingerprint,
+		IsOnline:      entity.IsOnline,
+		LastJoinedAt:  entity.LastJoinedAt,
+	}
+
+	return boltEntity, nil
+}
+
+func (entity *Controller) toBoltEntityForCreate(tx *bbolt.Tx, env Env) (*db.Controller, error) {
+	return entity.toBoltEntity(tx, env)
+}
+
+func (entity *Controller) toBoltEntityForUpdate(tx *bbolt.Tx, env Env, _ boltz.FieldChecker) (*db.Controller, error) {
+	return entity.toBoltEntity(tx, env)
+}
+
+func (entity *Controller) fillFrom(env Env, tx *bbolt.Tx, boltController *db.Controller) error {
+	entity.FillCommon(boltController)
+	entity.Name = boltController.Name
+	entity.Address = boltController.Address
+	entity.CertPem = boltController.CertPem
+	entity.Fingerprint = boltController.Fingerprint
+	entity.IsOnline = boltController.IsOnline
+	entity.LastJoinedAt = boltController.LastJoinedAt
+
+	return nil
+}

controller/model/create_terminator_cmd.go

@@ -93,7 +93,7 @@ func (self *CreateEdgeTerminatorCmd) getTerminatorSession(tx *bbolt.Tx, terminat
 	}
 
 	if session.ApiSession == nil {
-		apiSession, err := self.Env.GetStores().ApiSession.LoadOneById(tx, session.ApiSessionId)
+		apiSession, err := self.Env.GetStores().ApiSession.LoadById(tx, session.ApiSessionId)
 		if err != nil {
 			return nil, err
 		}

controller/model/edge_service_manager.go

@@ -287,7 +287,7 @@ func (self *EdgeServiceManager) mergeConfigs(tx *bbolt.Tx, configTypes map[strin
 	if len(configTypes) > 0 && len(service.Configs) > 0 {
 		configStore := self.env.GetStores().Config
 		for _, configId := range service.Configs {
-			config, _ := configStore.LoadOneById(tx, configId)
+			config, _ := configStore.LoadById(tx, configId)
 			if config != nil {
 				_, wantsConfig := configTypes[config.Type]
 				if wantsAll || wantsConfig {
@@ -340,6 +340,10 @@ func (self *EdgeServiceManager) GetPolicyPostureChecks(identityId, serviceId str
 	policyTypeSymbol := self.env.GetStores().ServicePolicy.GetSymbol(db.FieldServicePolicyType)
 
 	_ = self.GetDb().View(func(tx *bbolt.Tx) error {
+		if !self.env.GetStores().PostureCheck.IterateIds(tx, ast.BoolNodeTrue).IsValid() {
+			return nil
+		}
+
 		policyCursor := self.env.GetStores().Identity.GetRelatedEntitiesCursor(tx, identityId, db.EntityTypeServicePolicies, true)
 		policyCursor = ast.NewFilteredCursor(policyCursor, func(policyId []byte) bool {
 			return serviceLinks.IsLinked(tx, policyId, []byte(serviceId))

controller/model/edge_service_model.go

@@ -63,14 +63,14 @@ func (entity *Service) validateConfigs(tx *bbolt.Tx, env Env) error {
 	typeMap := map[string]*db.Config{}
 	configStore := env.GetStores().Config
 	for _, id := range entity.Configs {
-		config, _ := configStore.LoadOneById(tx, id)
+		config, _ := configStore.LoadById(tx, id)
 		if config == nil {
 			return boltz.NewNotFoundError(db.EntityTypeConfigs, "id", id)
 		}
 		conflictConfig, found := typeMap[config.Type]
 		if found {
 			configTypeName := "<not found>"
-			if configType, _ := env.GetStores().ConfigType.LoadOneById(tx, config.Type); configType != nil {
+			if configType, _ := env.GetStores().ConfigType.LoadById(tx, config.Type); configType != nil {
 				configTypeName = configType.Name
 			}
 			msg := fmt.Sprintf("duplicate configs named %v and %v found for config type %v. Only one config of a given typed is allowed per service ",

controller/model/enrollment_model.go

@@ -59,8 +59,15 @@ func (entity *Enrollment) FillJwtInfoWithExpiresAt(env Env, subject string, expi
 		entity.Token = uuid.New().String()
 	}
 
+	peerControllers := env.GetPeerControllerAddresses()
+
+	for i, addr := range peerControllers {
+		peerControllers[i] = "https://" + addr
+	}
+
 	enrollmentClaims := &ziti.EnrollmentClaims{
 		EnrollmentMethod: entity.Method,
+		Controllers:      peerControllers,
 		RegisteredClaims: jwt.RegisteredClaims{
 			Audience:  []string{""},
 			ExpiresAt: &jwt.NumericDate{Time: expiresAt},
@@ -70,7 +77,7 @@ func (entity *Enrollment) FillJwtInfoWithExpiresAt(env Env, subject string, expi
 		},
 	}
 
-	signedJwt, err := env.GetJwtSigner().Generate(subject, entity.Id, enrollmentClaims)
+	signedJwt, err := env.GetServerJwtSigner().Generate(enrollmentClaims)
 
 	if err != nil {
 		return err

controller/model/env.go

@@ -22,6 +22,7 @@ import (
 	"github.com/golang-jwt/jwt/v5"
 	"github.com/openziti/identity"
 	"github.com/openziti/metrics"
+	"github.com/openziti/ziti/common"
 	"github.com/openziti/ziti/common/cert"
 	"github.com/openziti/ziti/controller/config"
 	"github.com/openziti/ziti/controller/db"
@@ -33,7 +34,6 @@ import (
 type Env interface {
 	GetManagers() *Managers
 	GetConfig() *config.Config
-	GetJwtSigner() jwtsigner.Signer
 	GetDbProvider() network.DbProvider
 	GetStores() *db.Stores
 	GetAuthRegistry() AuthRegistry
@@ -46,8 +46,17 @@ type Env interface {
 	GetMetricsRegistry() metrics.Registry
 	GetFingerprintGenerator() cert.FingerprintGenerator
 	HandleServiceUpdatedEventForIdentityId(identityId string)
+
+	GetServerJwtSigner() jwtsigner.Signer
 	GetServerCert() (*tls.Certificate, string, jwt.SigningMethod)
 	JwtSignerKeyFunc(token *jwt.Token) (interface{}, error)
+	GetPeerControllerAddresses() []string
+
+	ValidateAccessToken(token string) (*common.AccessClaims, error)
+	ValidateServiceAccessToken(token string, apiSessionId *string) (*common.ServiceAccessClaims, error)
+
+	OidcIssuer() string
+	RootIssuer() string
 }
 
 type HostController interface {
@@ -57,6 +66,7 @@ type HostController interface {
 	IsRaftEnabled() bool
 	Identity() identity.Identity
 	GetPeerSigners() []*x509.Certificate
+	GetRaftIndex() uint64
 }
 
 type Schemas interface {

controller/model/managers.go

@@ -36,6 +36,7 @@ type Managers struct {
 	Ca                      *CaManager
 	Config                  *ConfigManager
 	ConfigType              *ConfigTypeManager
+	Controller              *ControllerManager
 	EdgeRouter              *EdgeRouterManager
 	EdgeRouterPolicy        *EdgeRouterPolicyManager
 	EdgeService             *EdgeServiceManager
@@ -72,6 +73,7 @@ func InitEntityManagers(env Env) *Managers {
 	managers.Ca = NewCaManager(env)
 	managers.Config = NewConfigManager(env)
 	managers.ConfigType = NewConfigTypeManager(env)
+	managers.Controller = NewControllerManager(env)
 	managers.EdgeRouter = NewEdgeRouterManager(env)
 	managers.EdgeRouterPolicy = NewEdgeRouterPolicyManager(env)
 	managers.EdgeService = NewEdgeServiceManager(env)

controller/model/mfa_manager.go

@@ -21,6 +21,7 @@ import (
 	"encoding/base32"
 	"fmt"
 	"github.com/dgryski/dgoogauth"
+	"github.com/michaelquigley/pfxlog"
 	"github.com/openziti/foundation/v2/errorz"
 	"github.com/openziti/storage/boltz"
 	"github.com/openziti/ziti/common/pb/edge_cmd_pb"
@@ -61,6 +62,16 @@ func (self *MfaManager) newModelEntity() *Mfa {
 	return &Mfa{}
 }
 
+func (self *MfaManager) CreateForIdentityId(identityId string, ctx *change.Context) (string, error) {
+	identity, err := self.env.GetManagers().Identity.Read(identityId)
+
+	if err != nil {
+		return "", err
+	}
+
+	return self.CreateForIdentity(identity, ctx)
+}
+
 func (self *MfaManager) CreateForIdentity(identity *Identity, ctx *change.Context) (string, error) {
 	secretBytes := make([]byte, 10)
 	_, _ = rand.Read(secretBytes)
@@ -303,6 +314,37 @@ func (self *MfaManager) DeleteAllForIdentity(id string, ctx *change.Context) err
 	})
 }
 
+func (self *MfaManager) CompleteTotpEnrollment(identityId string, code string, changeCtx *change.Context) error {
+	mfa, err := self.ReadOneByIdentityId(identityId)
+
+	if err != nil {
+		return err
+	}
+
+	if mfa == nil || mfa.IsVerified {
+		return errorz.NewNotFound()
+	}
+
+	ok, err := self.VerifyTOTP(mfa, code)
+
+	if err != nil {
+		pfxlog.Logger().WithError(err).Error("could not verify TOTP code")
+		return apierror.NewInvalidMfaTokenError()
+	}
+
+	if !ok {
+		return apierror.NewInvalidMfaTokenError()
+	}
+
+	mfa.IsVerified = true
+	if err := self.Update(mfa, nil, changeCtx); err != nil {
+		pfxlog.Logger().Errorf("could not update MFA with new MFA status: %v", err)
+		return errors.New("could not update MFA status")
+	}
+
+	return nil
+}
+
 type MfaListResult struct {
 	manager *MfaManager
 	Mfas    []*Mfa

controller/model/mfa_model.go

@@ -68,7 +68,7 @@ func (entity *Mfa) fillFrom(env Env, tx *bbolt.Tx, boltMfa *db.Mfa) error {
 	entity.IdentityId = boltMfa.IdentityId
 	entity.RecoveryCodes = boltMfa.RecoveryCodes
 	entity.Secret = boltMfa.Secret
-	boltIdentity, err := env.GetStores().Identity.LoadOneById(tx, boltMfa.IdentityId)
+	boltIdentity, err := env.GetStores().Identity.LoadById(tx, boltMfa.IdentityId)
 	if err != nil {
 		return err
 	}

controller/model/policy_advisor.go

@@ -82,7 +82,7 @@ func (advisor *PolicyAdvisor) getServicePermissions(identityId, serviceId string
 
 	servicePolicyStore := advisor.env.GetStores().ServicePolicy
 	servicePolicyIterator := func(tx *bbolt.Tx, servicePolicyId string) error {
-		servicePolicy, err := servicePolicyStore.LoadOneById(tx, servicePolicyId)
+		servicePolicy, err := servicePolicyStore.LoadById(tx, servicePolicyId)
 		if err != nil {
 			return err
 		}

controller/model/posture_response_manager.go

@@ -289,7 +289,7 @@ func (self *PostureResponseManager) GetEndpointStateChangeAffectedServices(timeS
 				cursor := self.env.GetStores().PostureCheck.IterateIds(tx, query)
 
 				for cursor.IsValid() {
-					if check, err := self.env.GetStores().PostureCheck.LoadOneById(tx, string(cursor.Current())); err == nil {
+					if check, err := self.env.GetStores().PostureCheck.LoadById(tx, string(cursor.Current())); err == nil {
 						if mfaCheck, ok := check.SubType.(*db.PostureCheckMfa); ok {
 							if shouldPostureCheckTimeoutBeAltered(mfaCheck, timeSinceLastMfa, gracePeriod, onWake, onUnlock) {
 								affectedChecks[check.Id] = mfaCheck.TimeoutSeconds
@@ -321,7 +321,7 @@ func (self *PostureResponseManager) GetEndpointStateChangeAffectedServices(timeS
 
 					for serviceCursor.IsValid() {
 						if _, ok := services[string(serviceCursor.Current())]; !ok {
-							service, err := self.env.GetStores().EdgeService.LoadOneById(tx, string(serviceCursor.Current()))
+							service, err := self.env.GetStores().EdgeService.LoadById(tx, string(serviceCursor.Current()))
 							if err == nil {
 								modelService := &Service{}
 								if err := modelService.fillFrom(self.env, tx, service); err == nil {

controller/model/posture_response_model.go

@@ -122,7 +122,7 @@ func (pc *PostureCache) evaluate() {
 			}
 
 			for cursor.IsValid() && len(sessions) < maxScanPerTx {
-				if session, _ := pc.env.GetStores().Session.LoadOneById(tx, string(cursor.Current())); session != nil {
+				if session, _ := pc.env.GetStores().Session.LoadById(tx, string(cursor.Current())); session != nil {
 					sessions = append(sessions, session)
 				}
 				lastId = cursor.Current()

controller/model/service_policy_manager.go

@@ -25,6 +25,7 @@ import (
 	"github.com/openziti/ziti/controller/fields"
 	"github.com/openziti/ziti/controller/models"
 	"github.com/openziti/ziti/controller/network"
+	"go.etcd.io/bbolt"
 	"google.golang.org/protobuf/proto"
 )
 
@@ -103,3 +104,17 @@ func (self *ServicePolicyManager) Unmarshall(bytes []byte) (*ServicePolicy, erro
 		PolicyType:        msg.PolicyType,
 	}, nil
 }
+
+type AssociatedIdsResult struct {
+	ServiceIds      []string
+	IdentityIds     []string
+	PostureCheckIds []string
+}
+
+func (self *ServicePolicyManager) ListAssociatedIds(tx *bbolt.Tx, id string) *AssociatedIdsResult {
+	return &AssociatedIdsResult{
+		IdentityIds:     self.env.GetStores().ServicePolicy.GetRelatedEntitiesIdList(tx, id, db.EntityTypeIdentities),
+		ServiceIds:      self.env.GetStores().ServicePolicy.GetRelatedEntitiesIdList(tx, id, db.EntityTypeServicePolicies),
+		PostureCheckIds: self.env.GetStores().ServicePolicy.GetRelatedEntitiesIdList(tx, id, db.EntityTypePostureChecks),
+	}
+}

controller/model/session_manager.go

@@ -18,11 +18,14 @@ package model
 
 import (
 	"fmt"
+	"github.com/golang-jwt/jwt/v5"
+	"github.com/google/uuid"
 	"github.com/lucsky/cuid"
 	"github.com/openziti/foundation/v2/errorz"
 	"github.com/openziti/foundation/v2/stringz"
 	"github.com/openziti/storage/ast"
 	"github.com/openziti/storage/boltz"
+	"github.com/openziti/ziti/common"
 	"github.com/openziti/ziti/controller/apierror"
 	fabricApiError "github.com/openziti/ziti/controller/apierror"
 	"github.com/openziti/ziti/controller/change"
@@ -151,6 +154,61 @@ func (self *SessionManager) EvaluatePostureForService(identityId, apiSessionId,
 	}
 }
 
+func (self *SessionManager) CreateJwt(entity *Session, ctx *change.Context) (string, error) {
+	entity.Id = uuid.New().String()
+
+	service, err := self.GetEnv().GetManagers().EdgeService.ReadForIdentity(entity.ServiceId, entity.IdentityId, nil)
+	if err != nil {
+		return "", err
+	}
+
+	if entity.Type == "" {
+		entity.Type = db.SessionTypeDial
+	}
+
+	if db.SessionTypeDial == entity.Type && !stringz.Contains(service.Permissions, db.PolicyTypeDialName) {
+		return "", errorz.NewFieldError("service not found", "ServiceId", entity.ServiceId)
+	}
+
+	if db.SessionTypeBind == entity.Type && !stringz.Contains(service.Permissions, db.PolicyTypeBindName) {
+		return "", errorz.NewFieldError("service not found", "ServiceId", entity.ServiceId)
+	}
+
+	policyResult := self.EvaluatePostureForService(entity.IdentityId, entity.ApiSessionId, entity.Type, service.Id, service.Name)
+
+	if !policyResult.Passed {
+		self.env.GetManagers().PostureResponse.postureCache.AddSessionRequestFailure(entity.IdentityId, policyResult.Failure)
+		return "", apierror.NewInvalidPosture(policyResult.Cause)
+	}
+
+	edgeRouterAvailable, err := self.GetEnv().GetManagers().EdgeRouter.IsSharedEdgeRouterPresent(entity.IdentityId, entity.ServiceId)
+	if err != nil {
+		return "", err
+	}
+
+	if !edgeRouterAvailable {
+		return "", apierror.NewNoEdgeRoutersAvailable()
+	}
+	entity.ServicePolicies = policyResult.PassingPolicyIds
+
+	claims := common.ServiceAccessClaims{
+		RegisteredClaims: jwt.RegisteredClaims{
+			Issuer:    self.env.RootIssuer(),
+			Subject:   entity.ServiceId,
+			Audience:  jwt.ClaimStrings{common.ClaimAudienceOpenZiti},
+			IssuedAt:  &jwt.NumericDate{Time: time.Now()},
+			ID:        entity.Id,
+			ExpiresAt: &jwt.NumericDate{Time: time.Now().AddDate(1, 0, 0)}, //bound by API Session
+		},
+		ApiSessionId: entity.ApiSessionId,
+		IdentityId:   entity.IdentityId,
+		Type:         entity.Type,
+		TokenType:    common.TokenTypeServiceAccess,
+	}
+
+	return self.env.GetServerJwtSigner().Generate(claims)
+}
+
 func (self *SessionManager) Create(entity *Session, ctx *change.Context) (string, error) {
 	if self.getExistingSessionEntity(entity) {
 		return entity.Id, nil
@@ -194,6 +252,7 @@ func (self *SessionManager) Create(entity *Session, ctx *change.Context) (string
 	if err != nil {
 		return "", err
 	}
+
 	if !edgeRouterAvailable {
 		return "", apierror.NewNoEdgeRoutersAvailable()
 	}

controller/model/session_model.go

@@ -35,7 +35,7 @@ type Session struct {
 }
 
 func (entity *Session) toBoltEntityForCreate(tx *bbolt.Tx, env Env) (*db.Session, error) {
-	apiSession, err := env.GetStores().ApiSession.LoadOneById(tx, entity.ApiSessionId)
+	apiSession, err := env.GetStores().ApiSession.LoadById(tx, entity.ApiSessionId)
 	if err != nil {
 		return nil, err
 	}

controller/model/testing.go

@@ -19,23 +19,26 @@ package model
 import (
 	"crypto/tls"
 	"crypto/x509"
+	"testing"
+	"time"
+
 	"github.com/golang-jwt/jwt/v5"
 	"github.com/google/uuid"
 	"github.com/openziti/foundation/v2/versions"
 	"github.com/openziti/identity"
 	"github.com/openziti/metrics"
 	"github.com/openziti/storage/boltz"
+	"github.com/openziti/ziti/common"
 	"github.com/openziti/ziti/common/cert"
 	"github.com/openziti/ziti/common/eid"
 	"github.com/openziti/ziti/controller/change"
 	"github.com/openziti/ziti/controller/command"
 	"github.com/openziti/ziti/controller/config"
+	edgeconfig "github.com/openziti/ziti/controller/config"
 	"github.com/openziti/ziti/controller/db"
 	"github.com/openziti/ziti/controller/event"
 	"github.com/openziti/ziti/controller/jwtsigner"
 	"github.com/openziti/ziti/controller/network"
-	"testing"
-	"time"
 )
 
 var _ Env = &TestContext{}
@@ -47,6 +50,10 @@ type testHostController struct {
 	ctx         *TestContext
 }
 
+func (self *testHostController) GetRaftIndex() uint64 {
+	return 0
+}
+
 func (self *testHostController) GetPeerSigners() []*x509.Certificate {
 	return nil
 }
@@ -79,7 +86,7 @@ type TestContext struct {
 	*db.TestContext
 	n               *network.Network
 	managers        *Managers
-	config          *config.Config
+	config          *edgeconfig.Config
 	metricsRegistry metrics.Registry
 	hostController  *testHostController
 }
@@ -88,6 +95,38 @@ func (ctx *TestContext) GetDbProvider() network.DbProvider {
 	return ctx.n
 }
 
+func (ctx *TestContext) ValidateAccessToken(token string) (*common.AccessClaims, error) {
+	//TODO implement me
+	panic("implement me")
+}
+
+func (ctx *TestContext) ValidateServiceAccessToken(token string, apiSessionId *string) (*common.ServiceAccessClaims, error) {
+	//TODO implement me
+	panic("implement me")
+}
+
+func (ctx *TestContext) OidcIssuer() string {
+	//TODO implement me
+	panic("implement me")
+}
+
+func (ctx *TestContext) RootIssuer() string {
+	//TODO implement me
+	panic("implement me")
+}
+
+func (ctx *TestContext) GetPeerControllerAddresses() []string {
+	return nil
+}
+
+func (ctx *TestContext) SigningMethod() jwt.SigningMethod {
+	return nil
+}
+
+func (ctx *TestContext) KeyId() string {
+	return "123-test-context"
+}
+
 func (ctx *TestContext) JwtSignerKeyFunc(*jwt.Token) (interface{}, error) {
 	tlsCert, _, _ := ctx.GetServerCert()
 	return tlsCert.Leaf.PublicKey, nil
@@ -99,7 +138,7 @@ func (ctx *TestContext) GetServerCert() (*tls.Certificate, string, jwt.SigningMe
 
 func (ctx *TestContext) HandleServiceUpdatedEventForIdentityId(string) {}
 
-func (ctx *TestContext) Generate(string, string, jwt.Claims) (string, error) {
+func (ctx *TestContext) Generate(jwt.Claims) (string, error) {
 	return "I'm a very legitimate claim", nil
 }
 
@@ -107,11 +146,11 @@ func (ctx *TestContext) GetManagers() *Managers {
 	return ctx.managers
 }
 
-func (ctx *TestContext) GetConfig() *config.Config {
+func (ctx *TestContext) GetConfig() *edgeconfig.Config {
 	return ctx.config
 }
 
-func (ctx *TestContext) GetJwtSigner() jwtsigner.Signer {
+func (ctx *TestContext) GetServerJwtSigner() jwtsigner.Signer {
 	return ctx
 }
 

controller/network/network.go

@@ -594,14 +594,14 @@ func (network *Network) CreateCircuit(params CreateCircuitParams) (*Circuit, err
 				// revert successful routes
 				logger.Warnf("circuit creation failed after [%d] attempts, sending cleanup unroutes", network.options.CreateCircuitRetries)
 				for cleanupRId := range allCleanups {
-					if r, err := network.GetRouter(cleanupRId); err == nil {
+					if r := network.GetConnectedRouter(cleanupRId); r != nil {
 						if err := sendUnroute(r, circuitId, true); err == nil {
 							logger.WithField("routerId", cleanupRId).Debug("sent cleanup unroute for circuit")
 						} else {
 							logger.WithField("routerId", cleanupRId).Error("error sending cleanup unroute for circuit")
 						}
 					} else {
-						logger.WithField("routerId", cleanupRId).Error("missing router for circuit cleanup")
+						logger.WithField("routerId", cleanupRId).Error("router for circuit cleanup not connected")
 					}
 				}
 
@@ -618,14 +618,14 @@ func (network *Network) CreateCircuit(params CreateCircuitParams) (*Circuit, err
 		for cleanupRId := range allCleanups {
 			if _, found := usedRouters[cleanupRId]; !found {
 				cleanupCount++
-				if r, err := network.GetRouter(cleanupRId); err == nil {
+				if r := network.GetConnectedRouter(cleanupRId); r != nil {
 					if err := sendUnroute(r, circuitId, true); err == nil {
 						logger.WithField("routerId", cleanupRId).Debug("sent abandoned cleanup unroute for circuit to router")
 					} else {
 						logger.WithField("routerId", cleanupRId).WithError(err).Error("error sending abandoned cleanup unroute for circuit to router")
 					}
 				} else {
-					logger.WithField("routerId", cleanupRId).Error("missing router for circuit, abandoned cleanup")
+					logger.WithField("routerId", cleanupRId).Error("router not connected for circuit, abandoned cleanup")
 				}
 			}
 		}