Merge pull request #257 from openziti/model-performance

Model performance

controller/model/edge_router_handlers.go

@@ -147,7 +147,7 @@ func (handler *EdgeRouterHandler) ListForIdentityAndServiceWithTx(tx *bbolt.Tx,
 		return nil, errors.Errorf("no service with id %v found", serviceId)
 	}
 
-	query := fmt.Sprintf(`anyOf(edgeRouterPolicies.identities) = "%v" and anyOf(serviceEdgeRouterPolicies.services) = "%v"`, identityId, service.Id)
+	query := fmt.Sprintf(`anyOf(identities) = "%v" and anyOf(services) = "%v"`, identityId, service.Id)
 
 	if limit != nil {
 		query += " limit " + strconv.Itoa(*limit)

controller/model/policy_advisor.go

@@ -87,7 +87,7 @@ func (advisor *PolicyAdvisor) getServicePermissions(identityId, serviceId string
 		if err != nil {
 			return err
 		}
-		if servicePolicyStore.IsEntityRelated(tx, servicePolicyId, persistence.EntityTypeServices, serviceId) {
+		if servicePolicyStore.IsEntityRelated(tx, servicePolicyId, db.EntityTypeServices, serviceId) {
 			if !stringz.Contains(permissions, servicePolicy.GetPolicyTypeName()) {
 				permissions = append(permissions, servicePolicy.GetPolicyTypeName())
 			}
@@ -243,7 +243,7 @@ func (advisor *PolicyAdvisor) getServicePolicies(identityId, serviceId string) (
 		if err != nil {
 			return err
 		}
-		if policyStore.IsEntityRelated(tx, policyId, persistence.EntityTypeServices, serviceId) {
+		if policyStore.IsEntityRelated(tx, policyId, db.EntityTypeServices, serviceId) {
 			result = append(result, policy)
 		}
 		return nil

controller/model/service_handlers.go

@@ -96,19 +96,14 @@ func (handler *EdgeServiceHandler) ReadForIdentityInTx(tx *bbolt.Tx, id string,
 }
 
 func (handler *EdgeServiceHandler) ReadForNonAdminIdentityInTx(tx *bbolt.Tx, id string, identityId string) (*ServiceDetail, error) {
-	query := `id = "%v" and not isEmpty(from servicePolicies where (type = %v and anyOf(identities.id) = "%v"))`
+	edgeServiceStore := handler.env.GetStores().EdgeService
+	isBindable := edgeServiceStore.IsBindableByIdentity(tx, id, identityId)
+	isDialable := edgeServiceStore.IsDialableByIdentity(tx, id, identityId)
 
-	dialQuery := fmt.Sprintf(query, id, persistence.PolicyTypeDial, identityId)
-	_, dialCount, err := handler.GetStore().QueryIds(tx, dialQuery)
-	if err != nil {
-		return nil, err
+	if !isBindable && !isDialable {
+		return nil, boltz.NewNotFoundError(handler.GetStore().GetSingularEntityType(), "id", id)
 	}
 
-	bindQuery := fmt.Sprintf(query, id, persistence.PolicyTypeBind, identityId)
-	_, bindCount, err := handler.GetStore().QueryIds(tx, bindQuery)
-	if err != nil {
-		return nil, err
-	}
 	result, err := handler.readInTx(tx, id)
 	if err != nil {
 		return nil, err
@@ -116,13 +111,10 @@ func (handler *EdgeServiceHandler) ReadForNonAdminIdentityInTx(tx *bbolt.Tx, id
 	if result == nil {
 		return nil, boltz.NewNotFoundError(handler.GetStore().GetSingularEntityType(), "id", id)
 	}
-	if bindCount > 0 {
+	if isBindable {
 		result.Permissions = append(result.Permissions, persistence.PolicyTypeBindName)
-	} else if dialCount == 0 {
-		return nil, boltz.NewNotFoundError(handler.GetStore().GetSingularEntityType(), "id", id)
 	}
-
-	if dialCount > 0 {
+	if isDialable {
 		result.Permissions = append(result.Permissions, persistence.PolicyTypeDialName)
 	}
 	return result, nil
@@ -148,7 +140,7 @@ func (handler *EdgeServiceHandler) PublicQueryForIdentity(sessionIdentity *Ident
 }
 
 func (handler *EdgeServiceHandler) QueryForIdentity(identityId string, configTypes map[string]struct{}, query ast.Query) (*ServiceListResult, error) {
-	idFilterQueryString := fmt.Sprintf(`not isEmpty(from servicePolicies where anyOf(identities) = "%v")`, identityId)
+	idFilterQueryString := fmt.Sprintf(`(anyOf(dialIdentities) = "%v" or anyOf(bindIdentities) = "%v")`, identityId, identityId)
 	idFilterQuery, err := ast.Parse(handler.Store, idFilterQueryString)
 	if err != nil {
 		return nil, err

controller/persistence/base_entity.go

@@ -16,7 +16,10 @@
 
 package persistence
 
-import "strings"
+import (
+	"github.com/openziti/foundation/storage/boltz"
+	"strings"
+)
 
 const (
 	EntityTypeApiSessions               = "apiSessions"
@@ -28,7 +31,6 @@ const (
 	EntityTypeGeoRegions                = "geoRegions"
 	EntityTypeIdentities                = "identities"
 	EntityTypeIdentityTypes             = "identityTypes"
-	EntityTypeServices                  = "services"
 	EntityTypeServicePolicies           = "servicePolicies"
 	EntityTypeServiceEdgeRouterPolicies = "serviceEdgeRouterPolicies"
 	EntityTypeSessions                  = "sessions"
@@ -75,3 +77,8 @@ func toStringInterfaceMap(m map[string]string) map[string]interface{} {
 	}
 	return result
 }
+
+type Policy interface {
+	boltz.NamedExtEntity
+	GetSemantic() string
+}

controller/persistence/base_store.go

@@ -17,15 +17,12 @@
 package persistence
 
 import (
-	"fmt"
 	"github.com/openziti/fabric/controller/db"
 	"github.com/openziti/fabric/controller/network"
 	"github.com/openziti/foundation/storage/ast"
 	"github.com/openziti/foundation/storage/boltz"
 	"github.com/openziti/foundation/util/errorz"
-	"github.com/openziti/foundation/util/stringz"
 	"github.com/openziti/foundation/validation"
-	"github.com/pkg/errors"
 	"go.etcd.io/bbolt"
 	"strings"
 )
@@ -81,11 +78,6 @@ func (store *baseStore) addUniqueNameField() boltz.ReadIndex {
 	return store.AddUniqueIndex(symbolName)
 }
 
-func (store *baseStore) addRoleAttributesField() boltz.SetReadIndex {
-	symbol := store.AddSetSymbol(FieldRoleAttributes, ast.NodeTypeString)
-	return store.AddSetIndex(symbol)
-}
-
 func (store *baseStore) initializeIndexes(tx *bbolt.Tx, errorHolder errorz.ErrorHolder) {
 	store.InitializeIndexes(tx, errorHolder)
 }
@@ -102,7 +94,6 @@ func (store *baseStore) baseLoadOneById(tx *bbolt.Tx, id string, entity boltz.En
 }
 
 func (store *baseStore) deleteEntityReferences(tx *bbolt.Tx, entity boltz.NamedExtEntity, rolesSymbol boltz.EntitySetSymbol) error {
-	// If the entity name is the same as entity ID, don't remove any of those references as id references take precedence
 	idRef := entityRef(entity.GetId())
 
 	for _, policyHolderId := range store.GetRelatedEntitiesIdList(tx, entity.GetId(), rolesSymbol.GetStore().GetEntityType()) {
@@ -118,91 +109,6 @@ func (store *baseStore) deleteEntityReferences(tx *bbolt.Tx, entity boltz.NamedE
 	return nil
 }
 
-func (store *baseStore) getEntityIdsForRoleSet(tx *bbolt.Tx, field string, roleSet []string, semantic string, index boltz.SetReadIndex, targetStore NameIndexedStore) ([]string, error) {
-	entityStore := index.GetSymbol().GetStore()
-	if stringz.Contains(roleSet, AllRole) {
-		ids, _, err := entityStore.QueryIds(tx, "true")
-		if err != nil {
-			return nil, err
-		}
-		return ids, nil
-	}
-
-	roles, ids, err := splitRolesAndIds(roleSet)
-	if err != nil {
-		return nil, err
-	}
-	if err := validateEntityIds(tx, targetStore, field, ids); err != nil {
-		return nil, err
-	}
-	var roleIds []string
-	if strings.EqualFold(semantic, SemanticAllOf) {
-		roleIds = store.FindMatching(tx, index, roles)
-	} else if strings.EqualFold(semantic, SemanticAnyOf) {
-		roleIds = store.FindMatchingAnyOf(tx, index, roles)
-	} else {
-		return nil, errors.Errorf("unsupported policy semantic %v", semantic)
-	}
-
-	for _, id := range ids {
-		if entityStore.IsEntityPresent(tx, id) {
-			roleIds = append(roleIds, id)
-		}
-	}
-	return roleIds, nil
-}
-
-func validateEntityIds(tx *bbolt.Tx, store NameIndexedStore, field string, ids []string) error {
-	var invalid []string
-	for _, val := range ids {
-		if !store.IsEntityPresent(tx, val) {
-			invalid = append(invalid, val)
-		}
-	}
-	if len(invalid) > 0 {
-		return validation.NewFieldError(fmt.Sprintf("no %v found with the given ids", store.GetEntityType()), field, invalid)
-	}
-	return nil
-}
-
-func UpdateRelatedRoles(tx *bbolt.Tx, entityId string, roleSymbol boltz.EntitySetSymbol,
-	linkCollection boltz.LinkCollection, new []boltz.FieldTypeAndValue, holder errorz.ErrorHolder, semanticSymbol boltz.EntitySymbol) {
-	ids, _, err := roleSymbol.GetStore().QueryIds(tx, "true")
-	holder.SetError(err)
-
-	entityRoles := FieldValuesToIds(new)
-
-	for _, id := range ids {
-		roleSet := roleSymbol.EvalStringList(tx, []byte(id))
-		roles, ids, err := splitRolesAndIds(roleSet)
-		if err != nil {
-			holder.SetError(err)
-			return
-		}
-		semantic := SemanticAllOf
-		if semanticSymbol != nil {
-			if _, semanticValue := semanticSymbol.Eval(tx, []byte(id)); semanticValue != nil {
-				semantic = string(semanticValue)
-			}
-		}
-
-		if stringz.Contains(ids, entityId) || stringz.Contains(roles, "all") {
-			err = linkCollection.AddLinks(tx, id, entityId)
-		} else if strings.EqualFold(semantic, SemanticAllOf) && len(roles) > 0 && stringz.ContainsAll(entityRoles, roles...) {
-			err = linkCollection.AddLinks(tx, id, entityId)
-		} else if strings.EqualFold(semantic, SemanticAnyOf) && len(roles) > 0 && stringz.ContainsAny(entityRoles, roles...) {
-			err = linkCollection.AddLinks(tx, id, entityId)
-		} else {
-			err = linkCollection.RemoveLinks(tx, id, entityId)
-		}
-
-		holder.SetError(err)
-		if holder.HasError() {
-			return
-		}
-	}
-}
-
 type NameIndexedStore interface {
 	Store
 	GetNameIndex() boltz.ReadIndex

controller/persistence/config_store.go

@@ -18,6 +18,7 @@ package persistence
 
 import (
 	"github.com/openziti/edge/eid"
+	"github.com/openziti/fabric/controller/db"
 	"github.com/openziti/foundation/storage/ast"
 	"github.com/openziti/foundation/storage/boltz"
 	"go.etcd.io/bbolt"
@@ -100,7 +101,7 @@ func (store *configStoreImpl) initializeLocal() {
 	store.indexName = store.addUniqueNameField()
 	store.symbolType = store.AddFkSymbol(FieldConfigType, store.stores.configType)
 	store.AddMapSymbol(FieldConfigData, ast.NodeTypeAnyType, FieldConfigData)
-	store.symbolServices = store.AddFkSetSymbol(EntityTypeServices, store.stores.edgeService)
+	store.symbolServices = store.AddFkSetSymbol(db.EntityTypeServices, store.stores.edgeService)
 	store.symbolIdentityServices = store.AddSetSymbol(FieldConfigIdentityService, ast.NodeTypeOther)
 	store.identityServicesLinks = &boltz.LinkedSetSymbol{EntitySymbol: store.symbolIdentityServices}
 }

controller/persistence/edge_router_policy_store.go

@@ -1,8 +1,9 @@
 package persistence
 
 import (
-	"github.com/openziti/fabric/controller/db"
+	"fmt"
 	"github.com/openziti/edge/eid"
+	"github.com/openziti/fabric/controller/db"
 	"github.com/openziti/foundation/storage/ast"
 	"github.com/openziti/foundation/storage/boltz"
 	"github.com/openziti/foundation/util/stringz"
@@ -30,6 +31,10 @@ func (entity *EdgeRouterPolicy) GetName() string {
 	return entity.Name
 }
 
+func (entity *EdgeRouterPolicy) GetSemantic() string {
+	return entity.Semantic
+}
+
 func (entity *EdgeRouterPolicy) LoadValues(_ boltz.CrudStore, bucket *boltz.TypedBucket) {
 	entity.LoadBaseValues(bucket)
 	entity.Name = bucket.GetStringOrError(FieldName)
@@ -83,6 +88,7 @@ type EdgeRouterPolicyStore interface {
 	NameIndexedStore
 	LoadOneById(tx *bbolt.Tx, id string) (*EdgeRouterPolicy, error)
 	LoadOneByName(tx *bbolt.Tx, id string) (*EdgeRouterPolicy, error)
+	CheckIntegrity(tx *bbolt.Tx, fix bool, errorSink func(error)) error
 }
 
 func newEdgeRouterPolicyStore(stores *stores) *edgeRouterPolicyStoreImpl {
@@ -154,16 +160,55 @@ Optimizations
    and just add/remove/ignore
 3. Related entity deletes should be handled automatically by FK Indexes on those entities (need to verify the reverse as well/deleting policy)
 */
-func (store *edgeRouterPolicyStoreImpl) edgeRouterRolesUpdated(ctx *boltz.PersistContext, policy *EdgeRouterPolicy) {
-	roleIds, err := store.getEntityIdsForRoleSet(ctx.Bucket.Tx(), "edgeRouterRoles", policy.EdgeRouterRoles, policy.Semantic, store.stores.edgeRouter.indexRoleAttributes, store.stores.edgeRouter)
-	if !ctx.Bucket.SetError(err) {
-		ctx.Bucket.SetError(store.edgeRouterCollection.SetLinks(ctx.Bucket.Tx(), policy.Id, roleIds))
+func (store *edgeRouterPolicyStoreImpl) edgeRouterRolesUpdated(persistCtx *boltz.PersistContext, policy *EdgeRouterPolicy) {
+	ctx := &roleAttributeChangeContext{
+		tx:                    persistCtx.Bucket.Tx(),
+		rolesSymbol:           store.symbolEdgeRouterRoles,
+		linkCollection:        store.edgeRouterCollection,
+		relatedLinkCollection: store.identityCollection,
+		denormLinkCollection:  store.stores.edgeRouter.identitiesCollection,
+		ErrorHolder:           persistCtx.Bucket,
 	}
+	EvaluatePolicy(ctx, policy, store.stores.edgeRouter.symbolRoleAttributes)
 }
 
-func (store *edgeRouterPolicyStoreImpl) identityRolesUpdated(ctx *boltz.PersistContext, policy *EdgeRouterPolicy) {
-	roleIds, err := store.getEntityIdsForRoleSet(ctx.Bucket.Tx(), "identityRoles", policy.IdentityRoles, policy.Semantic, store.stores.identity.indexRoleAttributes, store.stores.identity)
-	if !ctx.Bucket.SetError(err) {
-		ctx.Bucket.SetError(store.identityCollection.SetLinks(ctx.Bucket.Tx(), policy.Id, roleIds))
+func (store *edgeRouterPolicyStoreImpl) identityRolesUpdated(persistCtx *boltz.PersistContext, policy *EdgeRouterPolicy) {
+	ctx := &roleAttributeChangeContext{
+		tx:                    persistCtx.Bucket.Tx(),
+		rolesSymbol:           store.symbolIdentityRoles,
+		linkCollection:        store.identityCollection,
+		relatedLinkCollection: store.edgeRouterCollection,
+		denormLinkCollection:  store.stores.identity.edgeRoutersCollection,
+		ErrorHolder:           persistCtx.Bucket,
 	}
+	EvaluatePolicy(ctx, policy, store.stores.identity.symbolRoleAttributes)
+}
+
+func (store *edgeRouterPolicyStoreImpl) DeleteById(ctx boltz.MutateContext, id string) error {
+	policy, err := store.LoadOneById(ctx.Tx(), id)
+	if err != nil {
+		return err
+	}
+	policy.EdgeRouterRoles = nil
+	policy.IdentityRoles = nil
+	err = store.Update(ctx, policy, nil)
+	if err != nil {
+		return fmt.Errorf("failure while clearing policy before delete: %w", err)
+	}
+	return store.BaseStore.DeleteById(ctx, id)
+}
+
+func (store *edgeRouterPolicyStoreImpl) CheckIntegrity(tx *bbolt.Tx, fix bool, errorSink func(error)) error {
+	ctx := &denormCheckCtx{
+		tx:                     tx,
+		sourceStore:            store.stores.identity,
+		targetStore:            store.stores.edgeRouter,
+		policyStore:            store,
+		sourceCollection:       store.identityCollection,
+		targetCollection:       store.edgeRouterCollection,
+		targetDenormCollection: store.stores.identity.edgeRoutersCollection,
+		repair:                 fix,
+		errorSink:              errorSink,
+	}
+	return validatePolicyDenormalization(ctx)
 }

controller/persistence/edge_router_policy_store_test.go

@@ -5,6 +5,7 @@ import (
 	"github.com/openziti/edge/eid"
 	"github.com/openziti/fabric/controller/db"
 	"github.com/openziti/foundation/storage/boltz"
+	"github.com/openziti/foundation/util/errorz"
 	"github.com/openziti/foundation/util/stringz"
 	"go.etcd.io/bbolt"
 	"sort"
@@ -116,7 +117,7 @@ func (ctx *TestContext) testEdgeRouterPolicyUpdateDeleteRefs(t *testing.T) {
 
 	policy.IdentityRoles = []string{entityRef(identity.Id)}
 	ctx.RequireCreate(policy)
-	ctx.validateEdgeRouterPolicyIdentities([]*Identity{identity}, []*EdgeRouterPolicy{policy})
+	ctx.validateEdgeRouterPolicies([]*Identity{identity}, nil, []*EdgeRouterPolicy{policy})
 	ctx.RequireDelete(identity)
 	ctx.RequireReload(policy)
 	ctx.Equal(0, len(policy.IdentityRoles), "identity id should have been removed from identity roles")
@@ -182,14 +183,14 @@ func (ctx *TestContext) testEdgeRouterPolicyRoleEvaluation(t *testing.T) {
 
 	identityTypeId := ctx.getIdentityTypeId()
 
-	var identities []*Identity
+	identities := make([]*Identity, 0) // goland complains of potential nil panic if we use var identities []*Identities
 	for i := 0; i < 5; i++ {
 		identity := newIdentity(eid.New(), identityTypeId)
 		ctx.RequireCreate(identity)
 		identities = append(identities, identity)
 	}
 
-	var edgeRouters []*EdgeRouter
+	edgeRouters := make([]*EdgeRouter, 0)
 	for i := 0; i < 5; i++ {
 		edgeRouter := newEdgeRouter(eid.New())
 		ctx.RequireCreate(edgeRouter)
@@ -255,8 +256,7 @@ func (ctx *TestContext) testEdgeRouterPolicyRoleEvaluation(t *testing.T) {
 		edgeRouters = append(edgeRouters, edgeRouter)
 	})
 
-	ctx.validateEdgeRouterPolicyIdentities(identities, policies)
-	ctx.validateEdgeRouterPolicyEdgeRouters(edgeRouters, policies)
+	ctx.validateEdgeRouterPolicies(identities, edgeRouters, policies)
 
 	for _, identity := range identities {
 		ctx.RequireDelete(identity)
@@ -285,8 +285,7 @@ func (ctx *TestContext) testEdgeRouterPolicyRoleEvaluation(t *testing.T) {
 		edgeRouters = append(edgeRouters, edgeRouter)
 	})
 
-	ctx.validateEdgeRouterPolicyIdentities(identities, policies)
-	ctx.validateEdgeRouterPolicyEdgeRouters(edgeRouters, policies)
+	ctx.validateEdgeRouterPolicies(identities, edgeRouters, policies)
 
 	// ensure policies get cleaned up
 	for _, policy := range policies {
@@ -296,8 +295,7 @@ func (ctx *TestContext) testEdgeRouterPolicyRoleEvaluation(t *testing.T) {
 	// test with policies created after identities/edge routers
 	policies = ctx.createEdgeRouterPolicies(identityRoles, edgeRouterRoles, identities, edgeRouters, true)
 
-	ctx.validateEdgeRouterPolicyIdentities(identities, policies)
-	ctx.validateEdgeRouterPolicyEdgeRouters(edgeRouters, policies)
+	ctx.validateEdgeRouterPolicies(identities, edgeRouters, policies)
 
 	for _, policy := range policies {
 		ctx.RequireDelete(policy)
@@ -306,8 +304,7 @@ func (ctx *TestContext) testEdgeRouterPolicyRoleEvaluation(t *testing.T) {
 	// test with policies created after identities/edge routers and roles added after created
 	policies = ctx.createEdgeRouterPolicies(identityRoles, edgeRouterRoles, identities, edgeRouters, false)
 
-	ctx.validateEdgeRouterPolicyIdentities(identities, policies)
-	ctx.validateEdgeRouterPolicyEdgeRouters(edgeRouters, policies)
+	ctx.validateEdgeRouterPolicies(identities, edgeRouters, policies)
 
 	for _, identity := range identities {
 		if len(identity.RoleAttributes) > 0 {
@@ -333,8 +330,7 @@ func (ctx *TestContext) testEdgeRouterPolicyRoleEvaluation(t *testing.T) {
 		ctx.RequireUpdate(policy)
 	}
 
-	ctx.validateEdgeRouterPolicyIdentities(identities, policies)
-	ctx.validateEdgeRouterPolicyEdgeRouters(edgeRouters, policies)
+	ctx.validateEdgeRouterPolicies(identities, edgeRouters, policies)
 }
 
 func (ctx *TestContext) createEdgeRouterPolicies(identityRoles, edgeRouterRoles []string, identities []*Identity, edgeRouters []*EdgeRouter, oncreate bool) []*EdgeRouterPolicy {
@@ -391,6 +387,12 @@ func (ctx *TestContext) createEdgeRouterPolicies(identityRoles, edgeRouterRoles
 	return policies
 }
 
+func (ctx *TestContext) validateEdgeRouterPolicies(identities []*Identity, edgeRouters []*EdgeRouter, policies []*EdgeRouterPolicy) {
+	ctx.validateEdgeRouterPolicyIdentities(identities, policies)
+	ctx.validateEdgeRouterPolicyEdgeRouters(edgeRouters, policies)
+	ctx.validateEdgeRouterPolicyDenormalization()
+}
+
 func (ctx *TestContext) validateEdgeRouterPolicyIdentities(identities []*Identity, policies []*EdgeRouterPolicy) {
 	for _, policy := range policies {
 		count := 0
@@ -413,6 +415,16 @@ func (ctx *TestContext) validateEdgeRouterPolicyIdentities(identities []*Identit
 	}
 }
 
+func (ctx *TestContext) validateEdgeRouterPolicyDenormalization() {
+	errorHolder := &errorz.ErrorHolderImpl{}
+	errorHolder.SetError(ctx.db.View(func(tx *bbolt.Tx) error {
+		return ctx.stores.EdgeRouterPolicy.CheckIntegrity(tx, false, func(err error) {
+			errorHolder.SetError(err)
+		})
+	}))
+	ctx.NoError(errorHolder.GetError())
+}
+
 func (ctx *TestContext) validateEdgeRouterPolicyEdgeRouters(edgeRouters []*EdgeRouter, policies []*EdgeRouterPolicy) {
 	for _, policy := range policies {
 		count := 0

controller/persistence/edge_router_store.go

@@ -120,9 +120,16 @@ type edgeRouterStoreImpl struct {
 	indexName           boltz.ReadIndex
 	indexRoleAttributes boltz.SetReadIndex
 
+	symbolRoleAttributes            boltz.EntitySetSymbol
 	symbolEdgeRouterPolicies        boltz.EntitySetSymbol
 	symbolServiceEdgeRouterPolicies boltz.EntitySetSymbol
 	symbolEnrollments               boltz.EntitySetSymbol
+
+	symbolIdentities boltz.EntitySetSymbol
+	symbolServices   boltz.EntitySetSymbol
+
+	identitiesCollection boltz.RefCountedLinkCollection
+	servicesCollection   boltz.RefCountedLinkCollection
 }
 
 func (store *edgeRouterStoreImpl) NewStoreEntity() boltz.Entity {
@@ -136,35 +143,53 @@ func (store *edgeRouterStoreImpl) GetRoleAttributesIndex() boltz.SetReadIndex {
 func (store *edgeRouterStoreImpl) initializeLocal() {
 	store.GetParentStore().GrantSymbols(store)
 
+	store.symbolRoleAttributes = store.AddSetSymbol(FieldRoleAttributes, ast.NodeTypeString)
+
 	store.indexName = store.GetParentStore().(db.RouterStore).GetNameIndex()
-	store.indexRoleAttributes = store.addRoleAttributesField()
+	store.indexRoleAttributes = store.AddSetIndex(store.symbolRoleAttributes)
 
 	store.AddSymbol(FieldEdgeRouterIsVerified, ast.NodeTypeBool)
 	store.symbolEnrollments = store.AddFkSetSymbol(FieldEdgeRouterEnrollments, store.stores.enrollment)
 	store.symbolEdgeRouterPolicies = store.AddFkSetSymbol(EntityTypeEdgeRouterPolicies, store.stores.edgeRouterPolicy)
 	store.symbolServiceEdgeRouterPolicies = store.AddFkSetSymbol(EntityTypeServiceEdgeRouterPolicies, store.stores.serviceEdgeRouterPolicy)
 
+	store.symbolIdentities = store.AddFkSetSymbol(EntityTypeIdentities, store.stores.identity)
+	store.symbolServices = store.AddFkSetSymbol(db.EntityTypeServices, store.stores.edgeService)
+
 	store.indexRoleAttributes.AddListener(store.rolesChanged)
 }
 
-func (store *edgeRouterStoreImpl) rolesChanged(tx *bbolt.Tx, rowId []byte, _ []boltz.FieldTypeAndValue, new []boltz.FieldTypeAndValue, holder errorz.ErrorHolder) {
-	// Recalculate edge router policy links
-	rolesSymbol := store.stores.edgeRouterPolicy.symbolEdgeRouterRoles
-	linkCollection := store.stores.edgeRouterPolicy.edgeRouterCollection
-	semanticSymbol := store.stores.edgeRouterPolicy.symbolSemantic
-	UpdateRelatedRoles(tx, string(rowId), rolesSymbol, linkCollection, new, holder, semanticSymbol)
-
-	// Recalculate service edge router policy links
-	rolesSymbol = store.stores.serviceEdgeRouterPolicy.symbolEdgeRouterRoles
-	linkCollection = store.stores.serviceEdgeRouterPolicy.edgeRouterCollection
-	semanticSymbol = store.stores.serviceEdgeRouterPolicy.symbolSemantic
-	UpdateRelatedRoles(tx, string(rowId), rolesSymbol, linkCollection, new, holder, semanticSymbol)
-}
-
 func (store *edgeRouterStoreImpl) initializeLinked() {
 	store.AddLinkCollection(store.symbolEdgeRouterPolicies, store.stores.edgeRouterPolicy.symbolEdgeRouters)
 	store.AddLinkCollection(store.symbolServiceEdgeRouterPolicies, store.stores.serviceEdgeRouterPolicy.symbolEdgeRouters)
 	store.AddLinkCollection(store.symbolEnrollments, store.stores.enrollment.symbolEdgeRouter)
+
+	store.identitiesCollection = store.AddRefCountedLinkCollection(store.symbolIdentities, store.stores.identity.symbolEdgeRouters)
+	store.servicesCollection = store.AddRefCountedLinkCollection(store.symbolServices, store.stores.edgeService.symbolEdgeRouters)
+}
+
+func (store *edgeRouterStoreImpl) rolesChanged(tx *bbolt.Tx, rowId []byte, _ []boltz.FieldTypeAndValue, new []boltz.FieldTypeAndValue, holder errorz.ErrorHolder) {
+	// Recalculate edge router policy links
+	ctx := &roleAttributeChangeContext{
+		tx:                    tx,
+		rolesSymbol:           store.stores.edgeRouterPolicy.symbolEdgeRouterRoles,
+		linkCollection:        store.stores.edgeRouterPolicy.edgeRouterCollection,
+		relatedLinkCollection: store.stores.edgeRouterPolicy.identityCollection,
+		denormLinkCollection:  store.identitiesCollection,
+		ErrorHolder:           holder,
+	}
+	UpdateRelatedRoles(ctx, rowId, new, store.stores.edgeRouterPolicy.symbolSemantic)
+
+	// Recalculate service edge router policy links
+	ctx = &roleAttributeChangeContext{
+		tx:                    tx,
+		rolesSymbol:           store.stores.serviceEdgeRouterPolicy.symbolEdgeRouterRoles,
+		linkCollection:        store.stores.serviceEdgeRouterPolicy.edgeRouterCollection,
+		relatedLinkCollection: store.stores.serviceEdgeRouterPolicy.serviceCollection,
+		denormLinkCollection:  store.servicesCollection,
+		ErrorHolder:           holder,
+	}
+	UpdateRelatedRoles(ctx, rowId, new, store.stores.serviceEdgeRouterPolicy.symbolSemantic)
 }
 
 func (store *edgeRouterStoreImpl) GetNameIndex() boltz.ReadIndex {

controller/persistence/edge_service_store.go

@@ -27,6 +27,11 @@ import (
 	"reflect"
 )
 
+const (
+	FieldEdgeServiceDialIdentities = "dialIdentities"
+	FieldEdgeServiceBindIdentities = "bindIdentities"
+)
+
 type EdgeService struct {
 	db.Service
 	RoleAttributes []string
@@ -75,7 +80,8 @@ type EdgeServiceStore interface {
 
 	LoadOneById(tx *bbolt.Tx, id string) (*EdgeService, error)
 	LoadOneByName(tx *bbolt.Tx, id string) (*EdgeService, error)
-
+	IsBindableByIdentity(tx *bbolt.Tx, id string, identityId string) bool
+	IsDialableByIdentity(tx *bbolt.Tx, id string, identityId string) bool
 	GetRoleAttributesIndex() boltz.SetReadIndex
 	GetRoleAttributesCursorProvider(values []string, semantic string) (ast.SetCursorProvider, error)
 }
@@ -93,12 +99,21 @@ type edgeServiceStoreImpl struct {
 
 	indexName           boltz.ReadIndex
 	indexRoleAttributes boltz.SetReadIndex
-	symbolClusters      boltz.EntitySetSymbol
-	symbolSessions      boltz.EntitySetSymbol
+
+	symbolRoleAttributes boltz.EntitySetSymbol
+	symbolSessions       boltz.EntitySetSymbol
+	symbolConfigs        boltz.EntitySetSymbol
 
 	symbolServicePolicies           boltz.EntitySetSymbol
 	symbolServiceEdgeRouterPolicies boltz.EntitySetSymbol
-	symbolConfigs                   boltz.EntitySetSymbol
+
+	symbolDialIdentities boltz.EntitySetSymbol
+	symbolBindIdentities boltz.EntitySetSymbol
+	symbolEdgeRouters    boltz.EntitySetSymbol
+
+	bindIdentitiesCollection boltz.RefCountedLinkCollection
+	dialIdentitiesCollection boltz.RefCountedLinkCollection
+	edgeRoutersCollection    boltz.RefCountedLinkCollection
 }
 
 func (store *edgeServiceStoreImpl) NewStoreEntity() boltz.Entity {
@@ -112,36 +127,32 @@ func (store *edgeServiceStoreImpl) GetRoleAttributesIndex() boltz.SetReadIndex {
 func (store *edgeServiceStoreImpl) initializeLocal() {
 	store.GetParentStore().GrantSymbols(store)
 
+	store.symbolRoleAttributes = store.AddSetSymbol(FieldRoleAttributes, ast.NodeTypeString)
+
 	store.indexName = store.GetParentStore().(db.ServiceStore).GetNameIndex()
-	store.indexRoleAttributes = store.addRoleAttributesField()
+	store.indexRoleAttributes = store.AddSetIndex(store.symbolRoleAttributes)
 
 	store.symbolSessions = store.AddFkSetSymbol(EntityTypeSessions, store.stores.session)
 	store.symbolServiceEdgeRouterPolicies = store.AddFkSetSymbol(EntityTypeServiceEdgeRouterPolicies, store.stores.serviceEdgeRouterPolicy)
 	store.symbolServicePolicies = store.AddFkSetSymbol(EntityTypeServicePolicies, store.stores.servicePolicy)
 	store.symbolConfigs = store.AddFkSetSymbol(EntityTypeConfigs, store.stores.config)
 
+	store.symbolBindIdentities = store.AddFkSetSymbol(FieldEdgeServiceBindIdentities, store.stores.identity)
+	store.symbolDialIdentities = store.AddFkSetSymbol(FieldEdgeServiceDialIdentities, store.stores.identity)
+	store.symbolEdgeRouters = store.AddFkSetSymbol(migrationEntityTypeEdgeRouters, store.stores.edgeRouter)
+
 	store.indexRoleAttributes.AddListener(store.rolesChanged)
 }
 
-func (store *edgeServiceStoreImpl) rolesChanged(tx *bbolt.Tx, rowId []byte, _ []boltz.FieldTypeAndValue, new []boltz.FieldTypeAndValue, holder errorz.ErrorHolder) {
-	// Recalculate service policy links
-	rolesSymbol := store.stores.servicePolicy.symbolServiceRoles
-	linkCollection := store.stores.servicePolicy.serviceCollection
-	semanticSymbol := store.stores.servicePolicy.symbolSemantic
-	UpdateRelatedRoles(tx, string(rowId), rolesSymbol, linkCollection, new, holder, semanticSymbol)
-
-	// Recalculate service edge router policy links
-	rolesSymbol = store.stores.serviceEdgeRouterPolicy.symbolServiceRoles
-	linkCollection = store.stores.serviceEdgeRouterPolicy.serviceCollection
-	semanticSymbol = store.stores.serviceEdgeRouterPolicy.symbolSemantic
-	UpdateRelatedRoles(tx, string(rowId), rolesSymbol, linkCollection, new, holder, semanticSymbol)
-}
-
 func (store *edgeServiceStoreImpl) initializeLinked() {
 	store.AddLinkCollection(store.symbolServiceEdgeRouterPolicies, store.stores.serviceEdgeRouterPolicy.symbolServices)
 	store.AddLinkCollection(store.symbolServicePolicies, store.stores.servicePolicy.symbolServices)
 	store.AddLinkCollection(store.symbolConfigs, store.stores.config.symbolServices)
 
+	store.bindIdentitiesCollection = store.AddRefCountedLinkCollection(store.symbolBindIdentities, store.stores.identity.symbolBindServices)
+	store.dialIdentitiesCollection = store.AddRefCountedLinkCollection(store.symbolDialIdentities, store.stores.identity.symbolDialServices)
+	store.edgeRoutersCollection = store.AddRefCountedLinkCollection(store.symbolEdgeRouters, store.stores.edgeRouter.symbolServices)
+
 	store.EventEmmiter.AddListener(boltz.EventUpdate, func(i ...interface{}) {
 		if len(i) != 1 {
 			return
@@ -156,6 +167,29 @@ func (store *edgeServiceStoreImpl) initializeLinked() {
 	})
 }
 
+func (store *edgeServiceStoreImpl) rolesChanged(tx *bbolt.Tx, rowId []byte, _ []boltz.FieldTypeAndValue, new []boltz.FieldTypeAndValue, holder errorz.ErrorHolder) {
+	// Recalculate service policy links
+	ctx := &roleAttributeChangeContext{
+		tx:                    tx,
+		rolesSymbol:           store.stores.servicePolicy.symbolServiceRoles,
+		linkCollection:        store.stores.servicePolicy.serviceCollection,
+		relatedLinkCollection: store.stores.servicePolicy.identityCollection,
+		ErrorHolder:           holder,
+	}
+	store.updateServicePolicyRelatedRoles(ctx, rowId, new)
+
+	// Recalculate service edge router policy links
+	ctx = &roleAttributeChangeContext{
+		tx:                    tx,
+		rolesSymbol:           store.stores.serviceEdgeRouterPolicy.symbolServiceRoles,
+		linkCollection:        store.stores.serviceEdgeRouterPolicy.serviceCollection,
+		relatedLinkCollection: store.stores.serviceEdgeRouterPolicy.edgeRouterCollection,
+		denormLinkCollection:  store.edgeRoutersCollection,
+		ErrorHolder:           holder,
+	}
+	UpdateRelatedRoles(ctx, rowId, new, store.stores.serviceEdgeRouterPolicy.symbolSemantic)
+}
+
 func (store *edgeServiceStoreImpl) GetNameIndex() boltz.ReadIndex {
 	return store.indexName
 }
@@ -209,3 +243,13 @@ func (store *edgeServiceStoreImpl) DeleteById(ctx boltz.MutateContext, id string
 func (store *edgeServiceStoreImpl) GetRoleAttributesCursorProvider(values []string, semantic string) (ast.SetCursorProvider, error) {
 	return store.getRoleAttributesCursorProvider(store.indexRoleAttributes, values, semantic)
 }
+
+func (store *edgeServiceStoreImpl) IsBindableByIdentity(tx *bbolt.Tx, id string, identityId string) bool {
+	linkCount := store.bindIdentitiesCollection.GetLinkCount(tx, []byte(id), []byte(identityId))
+	return linkCount != nil && *linkCount > 0
+}
+
+func (store *edgeServiceStoreImpl) IsDialableByIdentity(tx *bbolt.Tx, id string, identityId string) bool {
+	linkCount := store.dialIdentitiesCollection.GetLinkCount(tx, []byte(id), []byte(identityId))
+	return linkCount != nil && *linkCount > 0
+}

controller/persistence/identity_store.go

@@ -19,6 +19,7 @@ package persistence
 import (
 	"github.com/michaelquigley/pfxlog"
 	"github.com/openziti/edge/eid"
+	"github.com/openziti/fabric/controller/db"
 	"github.com/openziti/foundation/storage/ast"
 	"github.com/openziti/foundation/storage/boltz"
 	"github.com/openziti/foundation/util/errorz"
@@ -43,6 +44,9 @@ const (
 	FieldIdentitySdkInfoRevision  = "sdkInfoRevision"
 	FieldIdentitySdkInfoType      = "sdkInfoType"
 	FieldIdentitySdkInfoVersion   = "sdkInfoVersion"
+
+	FieldIdentityBindServices = "bindServices"
+	FieldIdentityDialServices = "dialServices"
 )
 
 func newIdentity(name string, identityTypeId string, roleAttributes ...string) *Identity {
@@ -184,12 +188,21 @@ type identityStoreImpl struct {
 	indexName           boltz.ReadIndex
 	indexRoleAttributes boltz.SetReadIndex
 
-	symbolApiSessions        boltz.EntitySetSymbol
-	symbolAuthenticators     boltz.EntitySetSymbol
+	symbolRoleAttributes boltz.EntitySetSymbol
+	symbolApiSessions    boltz.EntitySetSymbol
+	symbolAuthenticators boltz.EntitySetSymbol
+	symbolIdentityTypeId boltz.EntitySymbol
+	symbolEnrollments    boltz.EntitySetSymbol
+
 	symbolEdgeRouterPolicies boltz.EntitySetSymbol
-	symbolEnrollments        boltz.EntitySetSymbol
 	symbolServicePolicies    boltz.EntitySetSymbol
-	symbolIdentityTypeId     boltz.EntitySymbol
+	symbolEdgeRouters        boltz.EntitySetSymbol
+	symbolBindServices       boltz.EntitySetSymbol
+	symbolDialServices       boltz.EntitySetSymbol
+
+	edgeRoutersCollection  boltz.RefCountedLinkCollection
+	bindServicesCollection boltz.RefCountedLinkCollection
+	dialServicesCollection boltz.RefCountedLinkCollection
 }
 
 func (store *identityStoreImpl) NewStoreEntity() boltz.Entity {
@@ -202,10 +215,15 @@ func (store *identityStoreImpl) GetRoleAttributesIndex() boltz.SetReadIndex {
 
 func (store *identityStoreImpl) initializeLocal() {
 	store.AddExtEntitySymbols()
-	store.indexRoleAttributes = store.addRoleAttributesField()
+
+	store.symbolRoleAttributes = store.AddSetSymbol(FieldRoleAttributes, ast.NodeTypeString)
+	store.indexRoleAttributes = store.AddSetIndex(store.symbolRoleAttributes)
 
 	store.indexName = store.addUniqueNameField()
 	store.symbolApiSessions = store.AddFkSetSymbol(FieldIdentityApiSessions, store.stores.apiSession)
+	store.symbolEdgeRouters = store.AddFkSetSymbol(db.EntityTypeRouters, store.stores.edgeRouter)
+	store.symbolBindServices = store.AddFkSetSymbol(FieldIdentityBindServices, store.stores.edgeService)
+	store.symbolDialServices = store.AddFkSetSymbol(FieldIdentityDialServices, store.stores.edgeService)
 	store.symbolEdgeRouterPolicies = store.AddFkSetSymbol(EntityTypeEdgeRouterPolicies, store.stores.edgeRouterPolicy)
 	store.symbolServicePolicies = store.AddFkSetSymbol(EntityTypeServicePolicies, store.stores.servicePolicy)
 	store.symbolEnrollments = store.AddFkSetSymbol(FieldIdentityEnrollments, store.stores.enrollment)
@@ -219,23 +237,36 @@ func (store *identityStoreImpl) initializeLocal() {
 	store.indexRoleAttributes.AddListener(store.rolesChanged)
 }
 
-func (store *identityStoreImpl) rolesChanged(tx *bbolt.Tx, rowId []byte, _ []boltz.FieldTypeAndValue, new []boltz.FieldTypeAndValue, holder errorz.ErrorHolder) {
-	rolesSymbol := store.stores.edgeRouterPolicy.symbolIdentityRoles
-	linkCollection := store.stores.edgeRouterPolicy.identityCollection
-	semanticSymbol := store.stores.edgeRouterPolicy.symbolSemantic
-	UpdateRelatedRoles(tx, string(rowId), rolesSymbol, linkCollection, new, holder, semanticSymbol)
-
-	rolesSymbol = store.stores.servicePolicy.symbolIdentityRoles
-	linkCollection = store.stores.servicePolicy.identityCollection
-	semanticSymbol = store.stores.servicePolicy.symbolSemantic
-	UpdateRelatedRoles(tx, string(rowId), rolesSymbol, linkCollection, new, holder, semanticSymbol)
-}
-
 func (store *identityStoreImpl) initializeLinked() {
 	store.AddLinkCollection(store.symbolAuthenticators, store.stores.authenticator.symbolIdentityId)
 	store.AddLinkCollection(store.symbolEnrollments, store.stores.enrollment.symbolIdentity)
 	store.AddLinkCollection(store.symbolEdgeRouterPolicies, store.stores.edgeRouterPolicy.symbolIdentities)
 	store.AddLinkCollection(store.symbolServicePolicies, store.stores.servicePolicy.symbolIdentities)
+
+	store.edgeRoutersCollection = store.AddRefCountedLinkCollection(store.symbolEdgeRouters, store.stores.edgeRouter.symbolIdentities)
+	store.bindServicesCollection = store.AddRefCountedLinkCollection(store.symbolBindServices, store.stores.edgeService.symbolBindIdentities)
+	store.dialServicesCollection = store.AddRefCountedLinkCollection(store.symbolDialServices, store.stores.edgeService.symbolDialIdentities)
+}
+
+func (store *identityStoreImpl) rolesChanged(tx *bbolt.Tx, rowId []byte, _ []boltz.FieldTypeAndValue, new []boltz.FieldTypeAndValue, holder errorz.ErrorHolder) {
+	ctx := &roleAttributeChangeContext{
+		tx:                    tx,
+		rolesSymbol:           store.stores.edgeRouterPolicy.symbolIdentityRoles,
+		linkCollection:        store.stores.edgeRouterPolicy.identityCollection,
+		relatedLinkCollection: store.stores.edgeRouterPolicy.edgeRouterCollection,
+		denormLinkCollection:  store.edgeRoutersCollection,
+		ErrorHolder:           holder,
+	}
+	UpdateRelatedRoles(ctx, rowId, new, store.stores.edgeRouterPolicy.symbolSemantic)
+
+	ctx = &roleAttributeChangeContext{
+		tx:                    tx,
+		rolesSymbol:           store.stores.servicePolicy.symbolIdentityRoles,
+		linkCollection:        store.stores.servicePolicy.identityCollection,
+		relatedLinkCollection: store.stores.servicePolicy.serviceCollection,
+		ErrorHolder:           holder,
+	}
+	store.updateServicePolicyRelatedRoles(ctx, rowId, new)
 }
 
 func (store *identityStoreImpl) GetNameIndex() boltz.ReadIndex {

controller/persistence/migration_v8.go

@@ -0,0 +1,34 @@
+package persistence
+
+import (
+	"github.com/michaelquigley/pfxlog"
+	"github.com/openziti/foundation/storage/boltz"
+)
+
+func (m *Migrations) denormalizePolicies(step *boltz.MigrationStep) {
+	log := pfxlog.Logger()
+
+	err := m.stores.EdgeRouterPolicy.CheckIntegrity(step.Ctx.Tx(), true, func(err error) {
+		log.WithError(err).Debug("updating edge router policies")
+	})
+
+	if step.SetError(err) {
+		return
+	}
+
+	err = m.stores.ServiceEdgeRouterPolicy.CheckIntegrity(step.Ctx.Tx(), true, func(err error) {
+		log.WithError(err).Debug("updating service edge router policies")
+	})
+
+	if step.SetError(err) {
+		return
+	}
+
+	err = m.stores.ServicePolicy.CheckIntegrity(step.Ctx.Tx(), true, func(err error) {
+		log.WithError(err).Debug("updating service policies")
+	})
+
+	if step.SetError(err) {
+		return
+	}
+}

controller/persistence/migrations.go

@@ -22,7 +22,7 @@ import (
 )
 
 const (
-	CurrentDbVersion = 7
+	CurrentDbVersion = 8
 	FieldVersion     = "version"
 )
 
@@ -78,6 +78,10 @@ func (m *Migrations) migrate(step *boltz.MigrationStep) int {
 		m.fixAuthenticatorCertFingerprints(step)
 	}
 
+	if step.CurrentVersion < 8 {
+		m.denormalizePolicies(step)
+	}
+
 	// current version
 	if step.CurrentVersion <= CurrentDbVersion {
 		return CurrentDbVersion

controller/persistence/policy_common.go

@@ -0,0 +1,239 @@
+package persistence
+
+import (
+	"bytes"
+	"fmt"
+	"github.com/openziti/foundation/storage/ast"
+	"github.com/openziti/foundation/storage/boltz"
+	"github.com/openziti/foundation/util/errorz"
+	"github.com/openziti/foundation/util/stringz"
+	"github.com/openziti/foundation/validation"
+	"github.com/pkg/errors"
+	"go.etcd.io/bbolt"
+	"strings"
+)
+
+type roleAttributeChangeContext struct {
+	tx                    *bbolt.Tx
+	rolesSymbol           boltz.EntitySetSymbol
+	linkCollection        boltz.LinkCollection
+	relatedLinkCollection boltz.LinkCollection
+	denormLinkCollection  boltz.RefCountedLinkCollection
+	errorz.ErrorHolder
+}
+
+func (store *baseStore) updateServicePolicyRelatedRoles(ctx *roleAttributeChangeContext, entityId []byte, newRoleAttributes []boltz.FieldTypeAndValue) {
+	cursor := ctx.rolesSymbol.GetStore().IterateIds(ctx.tx, ast.BoolNodeTrue)
+
+	entityRoles := FieldValuesToIds(newRoleAttributes)
+
+	semanticSymbol := store.stores.servicePolicy.symbolSemantic
+	policyTypeSymbol := store.stores.servicePolicy.symbolPolicyType
+
+	isServices := ctx.rolesSymbol == store.stores.servicePolicy.symbolServiceRoles
+
+	for ; cursor.IsValid(); cursor.Next() {
+		policyId := cursor.Current()
+		roleSet := ctx.rolesSymbol.EvalStringList(ctx.tx, policyId)
+		roles, ids, err := splitRolesAndIds(roleSet)
+		if err != nil {
+			ctx.SetError(err)
+			return
+		}
+
+		semantic := SemanticAllOf
+		if _, semanticValue := semanticSymbol.Eval(ctx.tx, policyId); semanticValue != nil {
+			semantic = string(semanticValue)
+		}
+		policyType := PolicyTypeDial
+		if fieldType, policyTypeValue := policyTypeSymbol.Eval(ctx.tx, policyId); fieldType == boltz.TypeInt32 {
+			policyType = *boltz.BytesToInt32(policyTypeValue)
+		}
+		if policyType == PolicyTypeDial {
+			if isServices {
+				ctx.denormLinkCollection = store.stores.edgeService.dialIdentitiesCollection
+			} else {
+				ctx.denormLinkCollection = store.stores.identity.dialServicesCollection
+			}
+		} else if isServices {
+			ctx.denormLinkCollection = store.stores.edgeService.bindIdentitiesCollection
+		} else {
+			ctx.denormLinkCollection = store.stores.identity.bindServicesCollection
+		}
+		evaluatePolicyAgainstEntity(ctx, semantic, entityId, policyId, ids, roles, entityRoles)
+	}
+}
+
+func EvaluatePolicy(ctx *roleAttributeChangeContext, policy Policy, roleAttributesSymbol boltz.EntitySetSymbol) {
+	policyId := []byte(policy.GetId())
+
+	roleSet := ctx.rolesSymbol.EvalStringList(ctx.tx, policyId)
+	roles, ids, err := splitRolesAndIds(roleSet)
+	if err != nil {
+		ctx.SetError(err)
+		return
+	}
+
+	if err := validateEntityIds(ctx.tx, ctx.linkCollection.GetLinkedSymbol().GetStore(), ctx.rolesSymbol.GetName(), ids); err != nil {
+		ctx.SetError(err)
+		return
+	}
+
+	cursor := roleAttributesSymbol.GetStore().IterateIds(ctx.tx, ast.BoolNodeTrue)
+	for ; cursor.IsValid(); cursor.Next() {
+		entityId := cursor.Current()
+		entityRoleAttributes := roleAttributesSymbol.EvalStringList(ctx.tx, entityId)
+		evaluatePolicyAgainstEntity(ctx, policy.GetSemantic(), entityId, policyId, ids, roles, entityRoleAttributes)
+	}
+}
+
+func validateEntityIds(tx *bbolt.Tx, store boltz.ListStore, field string, ids []string) error {
+	var invalid []string
+	for _, val := range ids {
+		if !store.IsEntityPresent(tx, val) {
+			invalid = append(invalid, val)
+		}
+	}
+	if len(invalid) > 0 {
+		return validation.NewFieldError(fmt.Sprintf("no %v found with the given ids", store.GetEntityType()), field, invalid)
+	}
+	return nil
+}
+
+func UpdateRelatedRoles(ctx *roleAttributeChangeContext, entityId []byte, newRoleAttributes []boltz.FieldTypeAndValue, semanticSymbol boltz.EntitySymbol) {
+	cursor := ctx.rolesSymbol.GetStore().IterateIds(ctx.tx, ast.BoolNodeTrue)
+
+	entityRoles := FieldValuesToIds(newRoleAttributes)
+
+	for ; cursor.IsValid(); cursor.Next() {
+		policyId := cursor.Current()
+		roleSet := ctx.rolesSymbol.EvalStringList(ctx.tx, policyId)
+		roles, ids, err := splitRolesAndIds(roleSet)
+		if err != nil {
+			ctx.SetError(err)
+			return
+		}
+
+		semantic := SemanticAllOf
+		if _, semanticValue := semanticSymbol.Eval(ctx.tx, policyId); semanticValue != nil {
+			semantic = string(semanticValue)
+		}
+		evaluatePolicyAgainstEntity(ctx, semantic, entityId, policyId, ids, roles, entityRoles)
+	}
+}
+
+func evaluatePolicyAgainstEntity(ctx *roleAttributeChangeContext, semantic string, entityId, policyId []byte, ids, roles, roleAttributes []string) {
+	if stringz.Contains(ids, string(entityId)) || stringz.Contains(roles, "all") ||
+		(strings.EqualFold(semantic, SemanticAllOf) && len(roles) > 0 && stringz.ContainsAll(roleAttributes, roles...)) ||
+		(strings.EqualFold(semantic, SemanticAnyOf) && len(roles) > 0 && stringz.ContainsAny(roleAttributes, roles...)) {
+		ProcessEntityPolicyMatched(ctx, entityId, policyId)
+	} else {
+		ProcessEntityPolicyUnmatched(ctx, entityId, policyId)
+	}
+}
+
+func ProcessEntityPolicyMatched(ctx *roleAttributeChangeContext, entityId, policyId []byte) {
+	if added, err := ctx.linkCollection.AddLink(ctx.tx, policyId, entityId); ctx.SetError(err) || !added {
+		return
+	}
+	cursor := ctx.relatedLinkCollection.IterateLinks(ctx.tx, policyId)
+	for ; cursor.IsValid(); cursor.Next() {
+		relatedEntityId := cursor.Current()
+		_, err := ctx.denormLinkCollection.IncrementLinkCount(ctx.tx, entityId, relatedEntityId)
+		if ctx.SetError(err) {
+			return
+		}
+	}
+}
+
+func ProcessEntityPolicyUnmatched(ctx *roleAttributeChangeContext, entityId, policyId []byte) {
+	if removed, err := ctx.linkCollection.RemoveLink(ctx.tx, policyId, entityId); ctx.SetError(err) || !removed {
+		return
+	}
+
+	cursor := ctx.relatedLinkCollection.IterateLinks(ctx.tx, policyId)
+	for ; cursor.IsValid(); cursor.Next() {
+		relatedEntityId := cursor.Current()
+		_, err := ctx.denormLinkCollection.DecrementLinkCount(ctx.tx, entityId, relatedEntityId)
+		if ctx.SetError(err) {
+			return
+		}
+	}
+}
+
+type denormCheckCtx struct {
+	tx                     *bbolt.Tx
+	sourceStore            boltz.CrudStore
+	targetStore            boltz.CrudStore
+	policyStore            boltz.CrudStore
+	sourceCollection       boltz.LinkCollection
+	targetCollection       boltz.LinkCollection
+	targetDenormCollection boltz.RefCountedLinkCollection
+	policyFilter           func(policyId []byte) bool
+	errorSink              func(err error)
+	repair                 bool
+}
+
+func validatePolicyDenormalization(ctx *denormCheckCtx) error {
+	for sourceCursor := ctx.sourceStore.IterateIds(ctx.tx, ast.BoolNodeTrue); sourceCursor.IsValid(); sourceCursor.Next() {
+		sourceEntityId := sourceCursor.Current()
+		for targetCursor := ctx.targetStore.IterateIds(ctx.tx, ast.BoolNodeTrue); targetCursor.IsValid(); targetCursor.Next() {
+			targetEntityId := targetCursor.Current()
+
+			var relatedPolicies []string
+
+			for policyCursor := ctx.policyStore.IterateIds(ctx.tx, ast.BoolNodeTrue); policyCursor.IsValid(); policyCursor.Next() {
+				policyId := policyCursor.Current()
+				if ctx.policyFilter == nil || ctx.policyFilter(policyId) {
+					sourceRelated := isRelatedByLinkCollection(ctx.tx, ctx.sourceCollection, policyId, sourceEntityId)
+					targetRelated := isRelatedByLinkCollection(ctx.tx, ctx.targetCollection, policyId, targetEntityId)
+					if sourceRelated && targetRelated {
+						relatedPolicies = append(relatedPolicies, string(policyId))
+					}
+				}
+			}
+			linkCount := len(relatedPolicies)
+			var sourceLinkCount, targetLinkCount *int32
+			var err error
+			if ctx.repair {
+				sourceLinkCount, targetLinkCount, err = ctx.targetDenormCollection.SetLinkCount(ctx.tx, sourceEntityId, targetEntityId, linkCount)
+			} else {
+				sourceLinkCount, targetLinkCount = ctx.targetDenormCollection.GetLinkCounts(ctx.tx, sourceEntityId, targetEntityId)
+			}
+			if err != nil {
+				return err
+			}
+			logDiscrepencies(ctx, linkCount, sourceEntityId, targetEntityId, sourceLinkCount, targetLinkCount)
+		}
+	}
+	return nil
+}
+
+func logDiscrepencies(ctx *denormCheckCtx, count int, sourceId, targetId []byte, sourceLinkCount, targetLinkCount *int32) {
+	oldValuesMatch := (sourceLinkCount == nil && targetLinkCount == nil) || (sourceLinkCount != nil && targetLinkCount != nil && *sourceLinkCount == *targetLinkCount)
+	if !oldValuesMatch {
+		err := errors.Errorf("fixed mismatched link counts. was %v %v (%v) <-> %v %v (%v), now both are %v",
+			ctx.sourceStore.GetSingularEntityType(), string(sourceId), sourceLinkCount,
+			ctx.targetStore.GetSingularEntityType(), string(targetId), targetLinkCount, count)
+		ctx.errorSink(err)
+	}
+
+	if ((sourceLinkCount == nil || *sourceLinkCount == 0) && count != 0) ||
+		(sourceLinkCount != nil && *sourceLinkCount != int32(count)) {
+		sourceCount := int32(0)
+		if sourceLinkCount != nil {
+			sourceCount = *sourceLinkCount
+		}
+		err := errors.Errorf("fixed incorrect link counts for %v %v <-> %v %v was %v, is now %v",
+			ctx.sourceStore.GetSingularEntityType(), string(sourceId),
+			ctx.targetStore.GetSingularEntityType(), string(targetId),
+			sourceCount, count)
+		ctx.errorSink(err)
+	}
+}
+
+func isRelatedByLinkCollection(tx *bbolt.Tx, linkCollection boltz.LinkCollection, entityId, relatedId []byte) bool {
+	cursor := linkCollection.IterateLinks(tx, entityId)
+	cursor.Seek(relatedId)
+	return bytes.Equal(cursor.Current(), relatedId)
+}

controller/persistence/service_edge_router_policy_store.go

@@ -1,8 +1,9 @@
 package persistence
 
 import (
-	"github.com/openziti/fabric/controller/db"
+	"fmt"
 	"github.com/openziti/edge/eid"
+	"github.com/openziti/fabric/controller/db"
 	"github.com/openziti/foundation/storage/ast"
 	"github.com/openziti/foundation/storage/boltz"
 	"github.com/openziti/foundation/util/stringz"
@@ -30,6 +31,10 @@ func (entity *ServiceEdgeRouterPolicy) GetName() string {
 	return entity.Name
 }
 
+func (entity *ServiceEdgeRouterPolicy) GetSemantic() string {
+	return entity.Semantic
+}
+
 func (entity *ServiceEdgeRouterPolicy) LoadValues(_ boltz.CrudStore, bucket *boltz.TypedBucket) {
 	entity.LoadBaseValues(bucket)
 	entity.Name = bucket.GetStringOrError(FieldName)
@@ -83,6 +88,7 @@ type ServiceEdgeRouterPolicyStore interface {
 	NameIndexedStore
 	LoadOneById(tx *bbolt.Tx, id string) (*ServiceEdgeRouterPolicy, error)
 	LoadOneByName(tx *bbolt.Tx, id string) (*ServiceEdgeRouterPolicy, error)
+	CheckIntegrity(tx *bbolt.Tx, fix bool, errorSink func(error)) error
 }
 
 func newServiceEdgeRouterPolicyStore(stores *stores) *serviceEdgeRouterPolicyStoreImpl {
@@ -122,7 +128,7 @@ func (store *serviceEdgeRouterPolicyStoreImpl) initializeLocal() {
 	store.symbolSemantic = store.AddSymbol(FieldSemantic, ast.NodeTypeString)
 	store.symbolServiceRoles = store.AddSetSymbol(FieldServiceRoles, ast.NodeTypeString)
 	store.symbolEdgeRouterRoles = store.AddSetSymbol(FieldEdgeRouterRoles, ast.NodeTypeString)
-	store.symbolServices = store.AddFkSetSymbol(EntityTypeServices, store.stores.edgeService)
+	store.symbolServices = store.AddFkSetSymbol(db.EntityTypeServices, store.stores.edgeService)
 	store.symbolEdgeRouters = store.AddFkSetSymbol(db.EntityTypeRouters, store.stores.edgeRouter)
 }
 
@@ -154,16 +160,55 @@ Optimizations
    and just add/remove/ignore
 3. Related entity deletes should be handled automatically by FK Indexes on those entities (need to verify the reverse as well/deleting policy)
 */
-func (store *serviceEdgeRouterPolicyStoreImpl) edgeRouterRolesUpdated(ctx *boltz.PersistContext, policy *ServiceEdgeRouterPolicy) {
-	roleIds, err := store.getEntityIdsForRoleSet(ctx.Bucket.Tx(), FieldEdgeRouterRoles, policy.EdgeRouterRoles, policy.Semantic, store.stores.edgeRouter.indexRoleAttributes, store.stores.edgeRouter)
-	if !ctx.Bucket.SetError(err) {
-		ctx.Bucket.SetError(store.edgeRouterCollection.SetLinks(ctx.Bucket.Tx(), policy.Id, roleIds))
+func (store *serviceEdgeRouterPolicyStoreImpl) edgeRouterRolesUpdated(persistCtx *boltz.PersistContext, policy *ServiceEdgeRouterPolicy) {
+	ctx := &roleAttributeChangeContext{
+		tx:                    persistCtx.Bucket.Tx(),
+		rolesSymbol:           store.symbolEdgeRouterRoles,
+		linkCollection:        store.edgeRouterCollection,
+		relatedLinkCollection: store.serviceCollection,
+		denormLinkCollection:  store.stores.edgeRouter.servicesCollection,
+		ErrorHolder:           persistCtx.Bucket,
 	}
+	EvaluatePolicy(ctx, policy, store.stores.edgeRouter.symbolRoleAttributes)
 }
 
-func (store *serviceEdgeRouterPolicyStoreImpl) serviceRolesUpdated(ctx *boltz.PersistContext, policy *ServiceEdgeRouterPolicy) {
-	roleIds, err := store.getEntityIdsForRoleSet(ctx.Bucket.Tx(), FieldServiceRoles, policy.ServiceRoles, policy.Semantic, store.stores.edgeService.indexRoleAttributes, store.stores.edgeService)
-	if !ctx.Bucket.SetError(err) {
-		ctx.Bucket.SetError(store.serviceCollection.SetLinks(ctx.Bucket.Tx(), policy.Id, roleIds))
+func (store *serviceEdgeRouterPolicyStoreImpl) serviceRolesUpdated(persistCtx *boltz.PersistContext, policy *ServiceEdgeRouterPolicy) {
+	ctx := &roleAttributeChangeContext{
+		tx:                    persistCtx.Bucket.Tx(),
+		rolesSymbol:           store.symbolServiceRoles,
+		linkCollection:        store.serviceCollection,
+		relatedLinkCollection: store.edgeRouterCollection,
+		denormLinkCollection:  store.stores.edgeService.edgeRoutersCollection,
+		ErrorHolder:           persistCtx.Bucket,
 	}
+	EvaluatePolicy(ctx, policy, store.stores.edgeService.symbolRoleAttributes)
+}
+
+func (store *serviceEdgeRouterPolicyStoreImpl) DeleteById(ctx boltz.MutateContext, id string) error {
+	policy, err := store.LoadOneById(ctx.Tx(), id)
+	if err != nil {
+		return err
+	}
+	policy.EdgeRouterRoles = nil
+	policy.ServiceRoles = nil
+	err = store.Update(ctx, policy, nil)
+	if err != nil {
+		return fmt.Errorf("failure while clearing policy before delete: %w", err)
+	}
+	return store.BaseStore.DeleteById(ctx, id)
+}
+
+func (store *serviceEdgeRouterPolicyStoreImpl) CheckIntegrity(tx *bbolt.Tx, fix bool, errorSink func(error)) error {
+	ctx := &denormCheckCtx{
+		tx:                     tx,
+		sourceStore:            store.stores.edgeService,
+		targetStore:            store.stores.edgeRouter,
+		policyStore:            store,
+		sourceCollection:       store.serviceCollection,
+		targetCollection:       store.edgeRouterCollection,
+		targetDenormCollection: store.stores.edgeService.edgeRoutersCollection,
+		errorSink:              errorSink,
+		repair:                 fix,
+	}
+	return validatePolicyDenormalization(ctx)
 }

controller/persistence/service_edge_router_policy_store_test.go

@@ -2,8 +2,9 @@ package persistence
 
 import (
 	"fmt"
-	"github.com/openziti/fabric/controller/db"
 	"github.com/openziti/edge/eid"
+	"github.com/openziti/fabric/controller/db"
+	"github.com/openziti/foundation/util/errorz"
 	"github.com/openziti/foundation/util/stringz"
 	"go.etcd.io/bbolt"
 	"sort"
@@ -30,7 +31,7 @@ func (ctx *TestContext) testCreateServiceEdgeRouterPolicy(_ *testing.T) {
 
 	err := ctx.GetDb().View(func(tx *bbolt.Tx) error {
 		ctx.Equal(0, len(ctx.stores.ServiceEdgeRouterPolicy.GetRelatedEntitiesIdList(tx, policy.Id, db.EntityTypeRouters)))
-		ctx.Equal(0, len(ctx.stores.ServiceEdgeRouterPolicy.GetRelatedEntitiesIdList(tx, policy.Id, EntityTypeServices)))
+		ctx.Equal(0, len(ctx.stores.ServiceEdgeRouterPolicy.GetRelatedEntitiesIdList(tx, policy.Id, db.EntityTypeServices)))
 
 		testPolicy, err := ctx.stores.ServiceEdgeRouterPolicy.LoadOneByName(tx, policy.Name)
 		ctx.NoError(err)
@@ -205,7 +206,7 @@ func (ctx *TestContext) testServiceEdgeRouterPolicyRoleEvaluation(_ *testing.T)
 
 	for i := 0; i < 9; i++ {
 		relatedEdgeRouters := ctx.getRelatedIds(policies[i], db.EntityTypeRouters)
-		relatedServices := ctx.getRelatedIds(policies[i], EntityTypeServices)
+		relatedServices := ctx.getRelatedIds(policies[i], db.EntityTypeServices)
 		if i == 3 {
 			ctx.Equal([]string{edgeRouters[0].Id}, relatedEdgeRouters)
 			ctx.Equal([]string{services[0].Id}, relatedServices)
@@ -245,8 +246,7 @@ func (ctx *TestContext) testServiceEdgeRouterPolicyRoleEvaluation(_ *testing.T)
 		edgeRouters = append(edgeRouters, edgeRouter)
 	})
 
-	ctx.validateServiceEdgeRouterPolicyServices(services, policies)
-	ctx.validateServiceEdgeRouterPolicyEdgeRouters(edgeRouters, policies)
+	ctx.validateServiceEdgeRouterPolicies(services, edgeRouters, policies)
 
 	for _, service := range services {
 		ctx.RequireDelete(service)
@@ -275,8 +275,7 @@ func (ctx *TestContext) testServiceEdgeRouterPolicyRoleEvaluation(_ *testing.T)
 		edgeRouters = append(edgeRouters, edgeRouter)
 	})
 
-	ctx.validateServiceEdgeRouterPolicyServices(services, policies)
-	ctx.validateServiceEdgeRouterPolicyEdgeRouters(edgeRouters, policies)
+	ctx.validateServiceEdgeRouterPolicies(services, edgeRouters, policies)
 
 	// ensure policies get cleaned up
 	for _, policy := range policies {
@@ -286,8 +285,7 @@ func (ctx *TestContext) testServiceEdgeRouterPolicyRoleEvaluation(_ *testing.T)
 	// test with policies created after services/edge routers
 	policies = ctx.createServiceEdgeRouterPolicies(serviceRoles, edgeRouterRoles, services, edgeRouters, true)
 
-	ctx.validateServiceEdgeRouterPolicyServices(services, policies)
-	ctx.validateServiceEdgeRouterPolicyEdgeRouters(edgeRouters, policies)
+	ctx.validateServiceEdgeRouterPolicies(services, edgeRouters, policies)
 
 	for _, policy := range policies {
 		ctx.RequireDelete(policy)
@@ -296,8 +294,7 @@ func (ctx *TestContext) testServiceEdgeRouterPolicyRoleEvaluation(_ *testing.T)
 	// test with policies created after services/edge routers and roles added after created
 	policies = ctx.createServiceEdgeRouterPolicies(serviceRoles, edgeRouterRoles, services, edgeRouters, false)
 
-	ctx.validateServiceEdgeRouterPolicyServices(services, policies)
-	ctx.validateServiceEdgeRouterPolicyEdgeRouters(edgeRouters, policies)
+	ctx.validateServiceEdgeRouterPolicies(services, edgeRouters, policies)
 
 	for _, service := range services {
 		if len(service.RoleAttributes) > 0 {
@@ -323,8 +320,7 @@ func (ctx *TestContext) testServiceEdgeRouterPolicyRoleEvaluation(_ *testing.T)
 		ctx.RequireUpdate(policy)
 	}
 
-	ctx.validateServiceEdgeRouterPolicyServices(services, policies)
-	ctx.validateServiceEdgeRouterPolicyEdgeRouters(edgeRouters, policies)
+	ctx.validateServiceEdgeRouterPolicies(services, edgeRouters, policies)
 }
 
 func (ctx *TestContext) createServiceEdgeRouterPolicies(serviceRoles, edgeRouterRoles []string, services []*EdgeService, edgeRouters []*EdgeRouter, oncreate bool) []*ServiceEdgeRouterPolicy {
@@ -381,10 +377,16 @@ func (ctx *TestContext) createServiceEdgeRouterPolicies(serviceRoles, edgeRouter
 	return policies
 }
 
+func (ctx *TestContext) validateServiceEdgeRouterPolicies(services []*EdgeService, edgeRouters []*EdgeRouter, policies []*ServiceEdgeRouterPolicy) {
+	ctx.validateServiceEdgeRouterPolicyServices(services, policies)
+	ctx.validateServiceEdgeRouterPolicyEdgeRouters(edgeRouters, policies)
+	ctx.validateServiceEdgeRouterPolicyDenormalization()
+}
+
 func (ctx *TestContext) validateServiceEdgeRouterPolicyServices(services []*EdgeService, policies []*ServiceEdgeRouterPolicy) {
 	for _, policy := range policies {
 		count := 0
-		relatedServices := ctx.getRelatedIds(policy, EntityTypeServices)
+		relatedServices := ctx.getRelatedIds(policy, db.EntityTypeServices)
 		for _, service := range services {
 			relatedPolicies := ctx.getRelatedIds(service, EntityTypeServiceEdgeRouterPolicies)
 			shouldContain := ctx.policyShouldMatch(policy.Semantic, policy.ServiceRoles, service, service.RoleAttributes)
@@ -423,3 +425,13 @@ func (ctx *TestContext) validateServiceEdgeRouterPolicyEdgeRouters(edgeRouters [
 		ctx.Equal(count, len(relatedEdgeRouters))
 	}
 }
+
+func (ctx *TestContext) validateServiceEdgeRouterPolicyDenormalization() {
+	errorHolder := &errorz.ErrorHolderImpl{}
+	errorHolder.SetError(ctx.db.View(func(tx *bbolt.Tx) error {
+		return ctx.stores.ServiceEdgeRouterPolicy.CheckIntegrity(tx, false, func(err error) {
+			errorHolder.SetError(err)
+		})
+	}))
+	ctx.NoError(errorHolder.GetError())
+}

controller/persistence/service_policy_store.go

@@ -1,20 +1,24 @@
 package persistence
 
 import (
+	"fmt"
 	"github.com/openziti/edge/eid"
+	"github.com/openziti/fabric/controller/db"
 	"github.com/openziti/foundation/storage/ast"
 	"github.com/openziti/foundation/storage/boltz"
 	"github.com/openziti/foundation/util/stringz"
 	"github.com/openziti/foundation/validation"
 	"go.etcd.io/bbolt"
+	"math/rand"
 	"sort"
 )
 
 const (
 	FieldServicePolicyType = "type"
 
-	PolicyTypeDialName = "Dial"
-	PolicyTypeBindName = "Bind"
+	PolicyTypeInvalidName = "Invalid"
+	PolicyTypeDialName    = "Dial"
+	PolicyTypeBindName    = "Bind"
 
 	PolicyTypeInvalid int32 = iota
 	PolicyTypeDial
@@ -22,9 +26,14 @@ const (
 )
 
 func newServicePolicy(name string) *ServicePolicy {
+	policyType := PolicyTypeDial
+	if rand.Int()%2 == 0 {
+		policyType = PolicyTypeBind
+	}
 	return &ServicePolicy{
 		BaseExtEntity: boltz.BaseExtEntity{Id: eid.New()},
 		Name:          name,
+		PolicyType:    policyType,
 	}
 }
 
@@ -41,6 +50,10 @@ func (entity *ServicePolicy) GetName() string {
 	return entity.Name
 }
 
+func (entity *ServicePolicy) GetSemantic() string {
+	return entity.Semantic
+}
+
 func (entity *ServicePolicy) LoadValues(_ boltz.CrudStore, bucket *boltz.TypedBucket) {
 	entity.LoadBaseValues(bucket)
 	entity.Name = bucket.GetStringOrError(FieldName)
@@ -55,6 +68,10 @@ func (entity *ServicePolicy) SetValues(ctx *boltz.PersistContext) {
 		entity.Semantic = SemanticAllOf
 	}
 
+	if entity.PolicyType != PolicyTypeBind {
+		entity.PolicyType = PolicyTypeDial
+	}
+
 	if err := validateRolesAndIds(FieldIdentityRoles, entity.IdentityRoles); err != nil {
 		ctx.Bucket.SetError(err)
 	}
@@ -91,17 +108,25 @@ func (entity *ServicePolicy) GetEntityType() string {
 	return EntityTypeServicePolicies
 }
 
-func (entity *ServicePolicy) GetPolicyTypeName() string {
-	if entity.PolicyType == PolicyTypeBind {
+func getPolicyTypeName(policyType int32) string {
+	if policyType == PolicyTypeBind {
 		return PolicyTypeBindName
 	}
-	return PolicyTypeDialName
+	if policyType == PolicyTypeDial {
+		return PolicyTypeDialName
+	}
+	return PolicyTypeInvalidName
+}
+
+func (entity *ServicePolicy) GetPolicyTypeName() string {
+	return getPolicyTypeName(entity.PolicyType)
 }
 
 type ServicePolicyStore interface {
 	NameIndexedStore
 	LoadOneById(tx *bbolt.Tx, id string) (*ServicePolicy, error)
 	LoadOneByName(tx *bbolt.Tx, id string) (*ServicePolicy, error)
+	CheckIntegrity(tx *bbolt.Tx, fix bool, errorSink func(error)) error
 }
 
 func newServicePolicyStore(stores *stores) *servicePolicyStoreImpl {
@@ -116,6 +141,7 @@ type servicePolicyStoreImpl struct {
 	*baseStore
 
 	indexName           boltz.ReadIndex
+	symbolPolicyType    boltz.EntitySymbol
 	symbolSemantic      boltz.EntitySymbol
 	symbolIdentityRoles boltz.EntitySetSymbol
 	symbolServiceRoles  boltz.EntitySetSymbol
@@ -138,12 +164,12 @@ func (store *servicePolicyStoreImpl) initializeLocal() {
 	store.AddExtEntitySymbols()
 
 	store.indexName = store.addUniqueNameField()
-	store.AddSymbol(FieldServicePolicyType, ast.NodeTypeInt64)
+	store.symbolPolicyType = store.AddSymbol(FieldServicePolicyType, ast.NodeTypeInt64)
 	store.symbolSemantic = store.AddSymbol(FieldSemantic, ast.NodeTypeString)
 	store.symbolIdentityRoles = store.AddSetSymbol(FieldIdentityRoles, ast.NodeTypeString)
 	store.symbolServiceRoles = store.AddSetSymbol(FieldServiceRoles, ast.NodeTypeString)
 	store.symbolIdentities = store.AddFkSetSymbol(EntityTypeIdentities, store.stores.identity)
-	store.symbolServices = store.AddFkSetSymbol(EntityTypeServices, store.stores.edgeService)
+	store.symbolServices = store.AddFkSetSymbol(db.EntityTypeServices, store.stores.edgeService)
 }
 
 func (store *servicePolicyStoreImpl) initializeLinked() {
@@ -174,16 +200,95 @@ Optimizations
    and just add/remove/ignore
 3. Related entity deletes should be handled automatically by FK Indexes on those entities (need to verify the reverse as well/deleting policy)
 */
-func (store *servicePolicyStoreImpl) serviceRolesUpdated(ctx *boltz.PersistContext, policy *ServicePolicy) {
-	roleIds, err := store.getEntityIdsForRoleSet(ctx.Bucket.Tx(), "serviceRoles", policy.ServiceRoles, policy.Semantic, store.stores.edgeService.indexRoleAttributes, store.stores.edgeService)
-	if !ctx.Bucket.SetError(err) {
-		ctx.Bucket.SetError(store.serviceCollection.SetLinks(ctx.Bucket.Tx(), policy.Id, roleIds))
+func (store *servicePolicyStoreImpl) serviceRolesUpdated(persistCtx *boltz.PersistContext, policy *ServicePolicy) {
+	ctx := &roleAttributeChangeContext{
+		tx:                    persistCtx.Bucket.Tx(),
+		rolesSymbol:           store.symbolServiceRoles,
+		linkCollection:        store.serviceCollection,
+		relatedLinkCollection: store.identityCollection,
+		ErrorHolder:           persistCtx.Bucket,
 	}
+	if policy.PolicyType == PolicyTypeDial {
+		ctx.denormLinkCollection = store.stores.edgeService.dialIdentitiesCollection
+	} else {
+		ctx.denormLinkCollection = store.stores.edgeService.bindIdentitiesCollection
+	}
+	EvaluatePolicy(ctx, policy, store.stores.edgeService.symbolRoleAttributes)
 }
 
-func (store *servicePolicyStoreImpl) identityRolesUpdated(ctx *boltz.PersistContext, policy *ServicePolicy) {
-	roleIds, err := store.getEntityIdsForRoleSet(ctx.Bucket.Tx(), "identityRoles", policy.IdentityRoles, policy.Semantic, store.stores.identity.indexRoleAttributes, store.stores.identity)
-	if !ctx.Bucket.SetError(err) {
-		ctx.Bucket.SetError(store.identityCollection.SetLinks(ctx.Bucket.Tx(), policy.Id, roleIds))
+func (store *servicePolicyStoreImpl) identityRolesUpdated(persistCtx *boltz.PersistContext, policy *ServicePolicy) {
+	ctx := &roleAttributeChangeContext{
+		tx:                    persistCtx.Bucket.Tx(),
+		rolesSymbol:           store.symbolIdentityRoles,
+		linkCollection:        store.identityCollection,
+		relatedLinkCollection: store.serviceCollection,
+		ErrorHolder:           persistCtx.Bucket,
 	}
+
+	if policy.PolicyType == PolicyTypeDial {
+		ctx.denormLinkCollection = store.stores.identity.dialServicesCollection
+	} else {
+		ctx.denormLinkCollection = store.stores.identity.bindServicesCollection
+	}
+
+	EvaluatePolicy(ctx, policy, store.stores.identity.symbolRoleAttributes)
+}
+
+func (store *servicePolicyStoreImpl) DeleteById(ctx boltz.MutateContext, id string) error {
+	policy, err := store.LoadOneById(ctx.Tx(), id)
+	if err != nil {
+		return err
+	}
+	policy.IdentityRoles = nil
+	policy.ServiceRoles = nil
+	err = store.Update(ctx, policy, nil)
+	if err != nil {
+		return fmt.Errorf("failure while clearing policy before delete: %w", err)
+	}
+	return store.BaseStore.DeleteById(ctx, id)
+}
+
+func (store *servicePolicyStoreImpl) CheckIntegrity(tx *bbolt.Tx, fix bool, errorSink func(error)) error {
+	ctx := &denormCheckCtx{
+		tx:                     tx,
+		sourceStore:            store.stores.identity,
+		targetStore:            store.stores.edgeService,
+		policyStore:            store,
+		sourceCollection:       store.identityCollection,
+		targetCollection:       store.serviceCollection,
+		targetDenormCollection: store.stores.identity.bindServicesCollection,
+		errorSink:              errorSink,
+		repair:                 fix,
+		policyFilter: func(policyId []byte) bool {
+			policyType := PolicyTypeInvalid
+			if result := boltz.FieldToInt32(store.symbolPolicyType.Eval(tx, policyId)); result != nil {
+				policyType = *result
+			}
+			return policyType == PolicyTypeBind
+		},
+	}
+	if err := validatePolicyDenormalization(ctx); err != nil {
+		return err
+	}
+
+	ctx = &denormCheckCtx{
+		tx:                     tx,
+		sourceStore:            store.stores.identity,
+		targetStore:            store.stores.edgeService,
+		policyStore:            store,
+		sourceCollection:       store.identityCollection,
+		targetCollection:       store.serviceCollection,
+		targetDenormCollection: store.stores.identity.dialServicesCollection,
+		errorSink:              errorSink,
+		repair:                 fix,
+		policyFilter: func(policyId []byte) bool {
+			policyType := PolicyTypeInvalid
+			if result := boltz.FieldToInt32(store.symbolPolicyType.Eval(tx, policyId)); result != nil {
+				policyType = *result
+			}
+			return policyType == PolicyTypeDial
+		},
+	}
+
+	return validatePolicyDenormalization(ctx)
 }

controller/persistence/service_policy_store_test.go

@@ -3,6 +3,8 @@ package persistence
 import (
 	"fmt"
 	"github.com/openziti/edge/eid"
+	"github.com/openziti/fabric/controller/db"
+	"github.com/openziti/foundation/util/errorz"
 	"github.com/openziti/foundation/util/stringz"
 	"go.etcd.io/bbolt"
 	"sort"
@@ -28,7 +30,7 @@ func (ctx *TestContext) testCreateServicePolicy(_ *testing.T) {
 
 	err := ctx.GetDb().View(func(tx *bbolt.Tx) error {
 		ctx.ValidateBaseline(policy)
-		ctx.Equal(0, len(ctx.stores.ServicePolicy.GetRelatedEntitiesIdList(tx, policy.Id, EntityTypeServices)))
+		ctx.Equal(0, len(ctx.stores.ServicePolicy.GetRelatedEntitiesIdList(tx, policy.Id, db.EntityTypeServices)))
 		ctx.Equal(0, len(ctx.stores.ServicePolicy.GetRelatedEntitiesIdList(tx, policy.Id, EntityTypeIdentities)))
 
 		testPolicy, err := ctx.stores.ServicePolicy.LoadOneByName(tx, policy.Name)
@@ -207,7 +209,7 @@ func (ctx *TestContext) testServicePolicyRoleEvaluation(_ *testing.T) {
 	policies := ctx.createServicePolicies(identityRoles, serviceRoles, identities, services, true)
 
 	for i := 0; i < 9; i++ {
-		relatedServices := ctx.getRelatedIds(policies[i], EntityTypeServices)
+		relatedServices := ctx.getRelatedIds(policies[i], db.EntityTypeServices)
 		relatedIdentities := ctx.getRelatedIds(policies[i], EntityTypeIdentities)
 		if i == 3 {
 			ctx.Equal([]string{services[0].Id}, relatedServices)
@@ -248,8 +250,7 @@ func (ctx *TestContext) testServicePolicyRoleEvaluation(_ *testing.T) {
 		services = append(services, service)
 	})
 
-	ctx.validateServicePolicyIdentities(identities, policies)
-	ctx.validateServicePolicyServices(services, policies)
+	ctx.validateServicePolicies(identities, services, policies)
 
 	for _, identity := range identities {
 		ctx.RequireDelete(identity)
@@ -278,8 +279,7 @@ func (ctx *TestContext) testServicePolicyRoleEvaluation(_ *testing.T) {
 		services = append(services, service)
 	})
 
-	ctx.validateServicePolicyIdentities(identities, policies)
-	ctx.validateServicePolicyServices(services, policies)
+	ctx.validateServicePolicies(identities, services, policies)
 
 	// ensure policies get cleaned up
 	for _, policy := range policies {
@@ -289,8 +289,7 @@ func (ctx *TestContext) testServicePolicyRoleEvaluation(_ *testing.T) {
 	// test with policies created after identities/edge routers
 	policies = ctx.createServicePolicies(identityRoles, serviceRoles, identities, services, true)
 
-	ctx.validateServicePolicyIdentities(identities, policies)
-	ctx.validateServicePolicyServices(services, policies)
+	ctx.validateServicePolicies(identities, services, policies)
 
 	for _, policy := range policies {
 		ctx.RequireDelete(policy)
@@ -299,8 +298,7 @@ func (ctx *TestContext) testServicePolicyRoleEvaluation(_ *testing.T) {
 	// test with policies created after identities/edge routers and roles added after created
 	policies = ctx.createServicePolicies(identityRoles, serviceRoles, identities, services, false)
 
-	ctx.validateServicePolicyIdentities(identities, policies)
-	ctx.validateServicePolicyServices(services, policies)
+	ctx.validateServicePolicies(identities, services, policies)
 
 	for _, identity := range identities {
 		if len(identity.RoleAttributes) > 0 {
@@ -326,8 +324,7 @@ func (ctx *TestContext) testServicePolicyRoleEvaluation(_ *testing.T) {
 		ctx.RequireUpdate(policy)
 	}
 
-	ctx.validateServicePolicyIdentities(identities, policies)
-	ctx.validateServicePolicyServices(services, policies)
+	ctx.validateServicePolicies(identities, services, policies)
 }
 
 func (ctx *TestContext) createServicePolicies(identityRoles, serviceRoles []string, identities []*Identity, services []*EdgeService, oncreate bool) []*ServicePolicy {
@@ -382,6 +379,12 @@ func (ctx *TestContext) createServicePolicies(identityRoles, serviceRoles []stri
 	return policies
 }
 
+func (ctx *TestContext) validateServicePolicies(identities []*Identity, services []*EdgeService, policies []*ServicePolicy) {
+	ctx.validateServicePolicyIdentities(identities, policies)
+	ctx.validateServicePolicyServices(services, policies)
+	ctx.validateServicePolicyDenormalization()
+}
+
 func (ctx *TestContext) validateServicePolicyIdentities(identities []*Identity, policies []*ServicePolicy) {
 	for _, policy := range policies {
 		count := 0
@@ -407,7 +410,7 @@ func (ctx *TestContext) validateServicePolicyIdentities(identities []*Identity,
 func (ctx *TestContext) validateServicePolicyServices(services []*EdgeService, policies []*ServicePolicy) {
 	for _, policy := range policies {
 		count := 0
-		relatedServices := ctx.getRelatedIds(policy, EntityTypeServices)
+		relatedServices := ctx.getRelatedIds(policy, db.EntityTypeServices)
 		for _, service := range services {
 			relatedPolicies := ctx.getRelatedIds(service, EntityTypeServicePolicies)
 			shouldContain := ctx.policyShouldMatch(policy.Semantic, policy.ServiceRoles, service, service.RoleAttributes)
@@ -424,3 +427,13 @@ func (ctx *TestContext) validateServicePolicyServices(services []*EdgeService, p
 		ctx.Equal(count, len(relatedServices))
 	}
 }
+
+func (ctx *TestContext) validateServicePolicyDenormalization() {
+	errorHolder := &errorz.ErrorHolderImpl{}
+	errorHolder.SetError(ctx.db.View(func(tx *bbolt.Tx) error {
+		return ctx.stores.ServicePolicy.CheckIntegrity(tx, false, func(err error) {
+			errorHolder.SetError(err)
+		})
+	}))
+	ctx.NoError(errorHolder.GetError())
+}

controller/persistence/testing.go

@@ -159,12 +159,15 @@ func (ctx *TestContext) cleanupAll() {
 	stores := []boltz.CrudStore{
 		ctx.stores.Session,
 		ctx.stores.ApiSession,
-		ctx.stores.EdgeRouterPolicy,
 		ctx.stores.Service,
 		ctx.stores.EdgeService,
 		ctx.stores.Identity,
 		ctx.stores.EdgeRouter,
 		ctx.stores.Config,
+		ctx.stores.Identity,
+		ctx.stores.EdgeRouterPolicy,
+		ctx.stores.ServicePolicy,
+		ctx.stores.ServiceEdgeRouterPolicy,
 	}
 	_ = ctx.GetDb().Update(func(tx *bbolt.Tx) error {
 		mutateContext := boltz.NewMutateContext(tx)

eid/id.go

@@ -1,9 +1,10 @@
 package eid
 
-import "github.com/teris-io/shortid"
+import (
+	"github.com/teris-io/shortid"
+)
 
 func New() string {
 	id, _ := shortid.GetDefault().Generate()
-
 	return id
 }

go.mod

@@ -31,10 +31,11 @@ require (
 	github.com/michaelquigley/pfxlog v0.0.0-20190813191113-2be43bd0dccc
 	github.com/miekg/dns v1.1.30
 	github.com/mitchellh/mapstructure v1.3.3
-	github.com/openziti/fabric v0.11.53
-	github.com/openziti/foundation v0.11.16
-	github.com/openziti/sdk-golang v0.13.26
+	github.com/openziti/fabric v0.11.56
+	github.com/openziti/foundation v0.11.15
+	github.com/openziti/sdk-golang v0.13.24
 	github.com/pkg/errors v0.9.1
+	github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0
 	github.com/sirupsen/logrus v1.6.0
 	github.com/spf13/cobra v0.0.7
 	github.com/stretchr/testify v1.6.1

go.sum

@@ -179,8 +179,6 @@ github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w=
-github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.1 h1:JFrFEBb2xKufg6XkJsJr+WbKb4FQlURi5RUcBveYu9k=
 github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@@ -284,14 +282,12 @@ github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs=
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
-github.com/openziti/fabric v0.11.53 h1:OVT71Hmbgf5gqsO7fz4iwsyP/T/ZzUfgOD0Bp39z/3U=
-github.com/openziti/fabric v0.11.53/go.mod h1:d0EueCQD4CIC3TLAMdw0EoYb+GsmwPrZz3BwRgtU28c=
-github.com/openziti/foundation v0.11.11 h1:GPASO0aTtzuzwQbfvu1OMOPdAKzi/DTkxKJ0y2NTzXU=
-github.com/openziti/foundation v0.11.11/go.mod h1:Qdd7eCuTrpRwEeDbvYy89sFbEY4+J7OuX8UygxNBsvY=
-github.com/openziti/foundation v0.11.16 h1:8pWIEHGM6Stmc9QxW/R+saRzxkzDsTDlAcyO21qrh4k=
-github.com/openziti/foundation v0.11.16/go.mod h1:4Jgs3HGDiP1aVk5nNzXJqwJwWhrjOYy1F+4Xtxcgxvo=
-github.com/openziti/sdk-golang v0.13.26 h1:WOGdmkzqNTh/3dT1dU6Bg0yZ7IX3drAhQju3WO/+kJs=
-github.com/openziti/sdk-golang v0.13.26/go.mod h1:leIaRUZzYSt+AHnwXlzH/XSX0hOhcjjJ/rMDTkieV/8=
+github.com/openziti/fabric v0.11.56 h1:u76PWSwbIgATtlF3mp3J1ioDwCyr7geaHFr0mZaiqEk=
+github.com/openziti/fabric v0.11.56/go.mod h1:9hvRAwTCRJanUb+1iF/NQ6HFfull7maHa9tCqklsNxk=
+github.com/openziti/foundation v0.11.15 h1:+LpphupxZvABFT2lZoS+Nk5EPBe09aXE935Jyfb5sQ8=
+github.com/openziti/foundation v0.11.15/go.mod h1:4Jgs3HGDiP1aVk5nNzXJqwJwWhrjOYy1F+4Xtxcgxvo=
+github.com/openziti/sdk-golang v0.13.24 h1:gu9R7QaGwAjgBigbotZz8Qwe0jiCkCVSzMuBWNIc9FE=
+github.com/openziti/sdk-golang v0.13.24/go.mod h1:hPRLQHARp9OuRViYFsZ6icwn1LjudXQBJ/USswKQyTY=
 github.com/orcaman/concurrent-map v0.0.0-20190826125027-8c72a8bb44f6 h1:lNCW6THrCKBiJBpz8kbVGjC7MgdCGKwuvBgc7LoD6sw=
 github.com/orcaman/concurrent-map v0.0.0-20190826125027-8c72a8bb44f6/go.mod h1:Lu3tH6HLW3feq74c2GC+jIMS/K2CFcDWnWD9XkenwhI=
 github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=

tests/ats-ctrl.yml

@@ -7,7 +7,7 @@ v: 3
 #  memory:
 #    path: ctrl.memprof
 
-db:                     testdata/ctrl.db
+db:                     testdata/${ZITI_TEST_DB}.db
 
 identity:
   cert:                 testdata/ca/intermediate/certs/ctrl-client.cert.pem

tests/authenticate.go

@@ -323,6 +323,12 @@ func (request *authenticatedRequests) requireNewService(roleAttributes, configs
 	return service
 }
 
+func (request *authenticatedRequests) newServiceBulk(roleAttributes, configs []string) *service {
+	service := request.testContext.newService(roleAttributes, configs)
+	request.requireCreateEntity(service)
+	return service
+}
+
 func (request *authenticatedRequests) RequireNewServiceAccessibleToAll(terminatorStrategy string) *service {
 	request.requireNewServicePolicy("Dial", s("#all"), s("#all"))
 	request.requireNewServicePolicy("Bind", s("#all"), s("#all"))
@@ -411,6 +417,16 @@ func (request *authenticatedRequests) requireCreateEntity(entity entity) string
 	return id
 }
 
+func (request *authenticatedRequests) createEntityBulk(entity entity) string {
+	resp := request.createEntity(entity)
+	if http.StatusCreated != resp.StatusCode() {
+		panic(errors.Errorf("expected error code %v", resp.StatusCode()))
+	}
+	id := request.testContext.getEntityId(resp.Body())
+	entity.setId(id)
+	return id
+}
+
 func (request *authenticatedRequests) requireDeleteEntity(entity entity) {
 	resp := request.deleteEntityOfType(entity.getEntityType(), entity.getId())
 	standardJsonResponseTests(resp, http.StatusOK, request.testContext.testing)

tests/context.go

@@ -189,17 +189,26 @@ func (ctx *TestContext) NewClientComponents() (*resty.Client, *http.Client, *htt
 }
 
 func (ctx *TestContext) StartServer() {
+	ctx.StartServerFor("default", true)
+}
+
+func (ctx *TestContext) StartServerFor(test string, clean bool) {
 	log := pfxlog.Logger()
 	_ = os.Mkdir("testdata", os.FileMode(0755))
-	err := filepath.Walk("testdata", func(path string, info os.FileInfo, err error) error {
-		if err == nil {
-			if !info.IsDir() && strings.HasPrefix(info.Name(), "ctrl.db") {
-				pfxlog.Logger().Infof("removing test bolt file or backup: %v", path)
-				err = os.Remove(path)
+	if clean {
+		err := filepath.Walk("testdata", func(path string, info os.FileInfo, err error) error {
+			if err == nil {
+				if !info.IsDir() && strings.HasPrefix(info.Name(), test+".db") {
+					pfxlog.Logger().Infof("removing test bolt file or backup: %v", path)
+					err = os.Remove(path)
+				}
 			}
-		}
-		return err
-	})
+			return err
+		})
+		ctx.Req.NoError(err)
+	}
+
+	err := os.Setenv("ZITI_TEST_DB", test)
 	ctx.Req.NoError(err)
 
 	log.Info("loading config")

tests/model_sdk_performance_test.go

@@ -0,0 +1,552 @@
+// +build apitests,perftests
+
+package tests
+
+import (
+	"fmt"
+	"github.com/google/uuid"
+	"github.com/openziti/edge/controller/model"
+	"github.com/openziti/edge/controller/persistence"
+	"github.com/openziti/edge/eid"
+	"github.com/openziti/fabric/controller/models"
+	idloader "github.com/openziti/foundation/identity/identity"
+	"github.com/openziti/foundation/util/errorz"
+	"github.com/openziti/sdk-golang/ziti/config"
+	"github.com/openziti/sdk-golang/ziti/edge"
+	"github.com/openziti/sdk-golang/ziti/edge/api"
+	"github.com/openziti/sdk-golang/ziti/sdkinfo"
+	"github.com/rcrowley/go-metrics"
+	"io"
+	"net/url"
+	"os"
+	"testing"
+	"time"
+)
+
+type modelPerf struct {
+	*TestContext
+}
+
+func Test_SpecLarge(t *testing.T) {
+	spec := &perfScenarioSpec{
+		name:                         "large",
+		serviceCount:                 200,
+		identityCount:                100_000,
+		edgeRouterCount:              500,
+		servicePolicyCount:           250,
+		edgeRouterPolicyCount:        250,
+		serviceEdgeRouterPolicyCount: 100,
+	}
+
+	p := &modelPerf{TestContext: NewTestContext(t)}
+	p.runScenario(spec)
+}
+
+func Test_SpecMedium(t *testing.T) {
+	spec := &perfScenarioSpec{
+		name:                         "medium",
+		serviceCount:                 100,
+		identityCount:                5000,
+		edgeRouterCount:              100,
+		servicePolicyCount:           50,
+		edgeRouterPolicyCount:        50,
+		serviceEdgeRouterPolicyCount: 25,
+	}
+
+	p := &modelPerf{TestContext: NewTestContext(t)}
+	p.runScenario(spec)
+}
+
+func Test_SpecSmall(t *testing.T) {
+	spec := &perfScenarioSpec{
+		name:                         "small",
+		serviceCount:                 20,
+		identityCount:                100,
+		edgeRouterCount:              10,
+		servicePolicyCount:           10,
+		edgeRouterPolicyCount:        10,
+		serviceEdgeRouterPolicyCount: 10,
+	}
+
+	p := &modelPerf{TestContext: NewTestContext(t)}
+	p.runScenario(spec)
+}
+
+func Test_SpecBaseline(t *testing.T) {
+	spec := &perfScenarioSpec{
+		name:                         "baseline",
+		serviceCount:                 1,
+		identityCount:                1,
+		edgeRouterCount:              1,
+		servicePolicyCount:           1,
+		edgeRouterPolicyCount:        1,
+		serviceEdgeRouterPolicyCount: 1,
+	}
+
+	p := &modelPerf{TestContext: NewTestContext(t)}
+	p.runScenario(spec)
+}
+
+type perfScenarioSpec struct {
+	name string
+
+	serviceCount    int
+	identityCount   int
+	edgeRouterCount int
+
+	servicePolicyCount           int
+	edgeRouterPolicyCount        int
+	serviceEdgeRouterPolicyCount int
+
+	serviceAttrs      [][]string
+	serviceAttrSet    []string
+	identityAttrs     [][]string
+	identityAttrSet   []string
+	edgeRouterAttrs   [][]string
+	edgeRouterAttrSet []string
+
+	services    []*model.Service
+	identities  []*model.Identity
+	edgeRouters []*model.EdgeRouter
+
+	config *config.Config
+}
+
+func (spec *perfScenarioSpec) generateAllRoleAttributes() {
+	spec.serviceAttrs, spec.serviceAttrSet = spec.generateRoleAttributes(spec.serviceCount)
+	spec.identityAttrs, spec.identityAttrSet = spec.generateRoleAttributes(spec.identityCount)
+	spec.edgeRouterAttrs, spec.edgeRouterAttrSet = spec.generateRoleAttributes(spec.edgeRouterCount)
+}
+
+func (spec *perfScenarioSpec) generateRoleAttributes(count int) ([][]string, []string) {
+	result := make([][]string, count)
+	if count < 2 {
+		return result, nil
+	}
+	var attrs []string
+
+	set := result[1:]
+
+	// assign 8 role attributes
+	offset := 2
+	index := 0
+
+	counts := []int{count / 2, count / 2, count / 3, count / 4, count / 5, count / 8, count / 10, count / 10}
+	attr := eid.New()
+	attrs = append(attrs, "#"+attr)
+	for len(counts) > 0 {
+		set[index] = append(set[index], attr)
+		counts[0]--
+		for len(counts) > 0 && counts[0] <= 0 {
+			counts = counts[1:]
+			if len(counts) > 0 && counts[0] > 0 {
+				if len(counts) < 4 {
+					offset++
+				}
+				attr = eid.New()
+				attrs = append(attrs, "#"+attr)
+			}
+		}
+		index = (index + offset) % len(set)
+	}
+
+	return result, attrs
+}
+
+func (ctx *modelPerf) runScenario(spec *perfScenarioSpec) {
+	shutdown := false
+	defer func() {
+		if !shutdown {
+			ctx.Teardown()
+		}
+	}()
+
+	ctx.StartServer()
+	ctx.RequireAdminLogin()
+
+	ctx.CreateEnrollAndStartEdgeRouter()
+
+	ctx.createScenario(spec)
+
+	stats := newPerfStats(ctx.TestContext, spec.config, spec.name, spec.services[0].Id, edge.SessionDial)
+	stats.collectStats(100)
+
+	stats = newPerfStats(ctx.TestContext, spec.config, spec.name, spec.services[0].Id, edge.SessionDial)
+	stats.collectStats(100)
+
+	ctx.Teardown()
+	shutdown = true
+	time.Sleep(time.Second)
+
+	stats.dumpStatsToStdOut()
+}
+
+// For policies, want the following, using service policies as an example
+// a. 1 service policy which has 1 id and all services
+// b. 1 service policy which has all ids and 1 service
+// c. 1 identity with all policies (which must be the same as the one in line a)
+// d. 1 service with all policies (which must the same as the one in line b)
+//
+// randomly assign role attributes  to 1/2, 1/4, 1/8, then do permutations of those with both anyof and allof
+// When more groups are needed, create more sets of 1/2, 1/4, 1/8 and 1/16
+func (ctx *modelPerf) createScenario(spec *perfScenarioSpec) {
+	spec.generateAllRoleAttributes()
+	ctx.createServices(spec)
+	ctx.createIdentities(spec)
+	ctx.createEdgeRouters(spec)
+	ctx.createPolicies(spec)
+}
+
+func (ctx *modelPerf) createServices(spec *perfScenarioSpec) {
+	serviceHandler := ctx.EdgeController.AppEnv.Handlers.EdgeService
+	for i := 0; i < spec.serviceCount; i++ {
+		id := eid.New()
+		if i == 0 {
+			id = "zzzzzzzzzzzzzz"
+		}
+		service := &model.Service{
+			BaseEntity: models.BaseEntity{
+				Id: id,
+			},
+			Name:           id,
+			RoleAttributes: spec.serviceAttrs[i],
+		}
+		_, err := serviceHandler.Create(service)
+		ctx.Req.NoError(err)
+		service.Id = id
+		spec.services = append(spec.services, service)
+		if (i+1)%100 == 0 {
+			fmt.Printf("created %v services\n", i)
+		}
+	}
+	fmt.Printf("finished creating %v services\n", spec.serviceCount)
+}
+
+func (ctx *modelPerf) createIdentities(spec *perfScenarioSpec) {
+	identityHandler := ctx.EdgeController.AppEnv.Handlers.Identity
+	for i := 0; i < spec.identityCount; i++ {
+		id := eid.New()
+		if i == 0 {
+			id = "zzzzzzzzzzzzzz"
+		}
+		identity := &model.Identity{
+			BaseEntity: models.BaseEntity{
+				Id: id,
+			},
+			Name:           id,
+			IdentityTypeId: "User",
+			IsDefaultAdmin: false,
+			IsAdmin:        false,
+			RoleAttributes: spec.identityAttrs[i],
+		}
+
+		enrollments := []*model.Enrollment{
+			{
+				BaseEntity: models.BaseEntity{},
+				Method:     persistence.MethodEnrollOtt,
+				Token:      uuid.New().String(),
+			},
+		}
+
+		_, _, err := identityHandler.CreateWithEnrollments(identity, enrollments)
+		ctx.Req.NoError(err)
+		spec.identities = append(spec.identities, identity)
+
+		if i == 0 {
+			spec.config = ctx.EnrollIdentity(identity.Id)
+		}
+		if (i+1)%100 == 0 {
+			fmt.Printf("created %v identities\n", i+1)
+		}
+	}
+	fmt.Printf("finished creating %v identities\n", spec.identityCount)
+}
+
+func (ctx *modelPerf) createEdgeRouters(spec *perfScenarioSpec) {
+	edgeRouterHandler := ctx.EdgeController.AppEnv.Handlers.EdgeRouter
+	for i := 0; i < spec.edgeRouterCount; i++ {
+		id := eid.New()
+		if i == 0 {
+			id = "zzzzzzzzzzzzzz"
+		}
+		edgeRouter := &model.EdgeRouter{
+			BaseEntity:     models.BaseEntity{Id: id},
+			Name:           id,
+			RoleAttributes: spec.edgeRouterAttrs[i],
+			IsVerified:     false,
+		}
+		_, err := edgeRouterHandler.Create(edgeRouter)
+		ctx.Req.NoError(err)
+		spec.edgeRouters = append(spec.edgeRouters, edgeRouter)
+		if (i+1)%100 == 0 {
+			fmt.Printf("created %v edge routers\n", i+1)
+		}
+	}
+	fmt.Printf("finished creating %v edge routers\n", spec.edgeRouterCount)
+}
+
+func (ctx *modelPerf) createPolicies(spec *perfScenarioSpec) {
+	ctx.createServicePolicy("Dial", s("@"+spec.services[0].Id), s("#all"))
+	ctx.createServicePolicy("Dial", s("#all"), s("@"+spec.identities[0].Id))
+
+	serviceRoles := ctx.firstNPermuations(spec.servicePolicyCount, spec.serviceAttrSet)
+	identityRoles := ctx.firstNPermuations(spec.servicePolicyCount, spec.identityAttrSet)
+	for i := 2; i < spec.servicePolicyCount; i++ {
+		serviceRoles[i] = append(serviceRoles[i], "@"+spec.services[0].Id)
+		identityRoles[i] = append(identityRoles[i], "@"+spec.identities[0].Id)
+		ctx.createServicePolicy("Dial", serviceRoles[i], identityRoles[i])
+	}
+
+	ctx.createEdgeRouterPolicy(s("@"+spec.edgeRouters[0].Id), s("#all"))
+	ctx.createEdgeRouterPolicy(s("#all"), s("@"+spec.identities[0].Id))
+
+	edgeRouterRoles := ctx.firstNPermuations(spec.edgeRouterPolicyCount, spec.edgeRouterAttrSet)
+	identityRoles = ctx.firstNPermuations(spec.edgeRouterPolicyCount, spec.identityAttrSet)
+	for i := 2; i < spec.edgeRouterPolicyCount; i++ {
+		edgeRouterRoles[i] = append(edgeRouterRoles[i], "@"+spec.edgeRouters[0].Id)
+		identityRoles[i] = append(identityRoles[i], "@"+spec.identities[0].Id)
+		ctx.createEdgeRouterPolicy(edgeRouterRoles[i], identityRoles[i])
+	}
+
+	ctx.createServiceEdgeRouterPolicy(s("@"+spec.edgeRouters[0].Id), s("#all"))
+	ctx.createServiceEdgeRouterPolicy(s("#all"), s("@"+spec.services[0].Id))
+
+	serviceRoles = ctx.firstNPermuations(spec.serviceEdgeRouterPolicyCount, spec.serviceAttrSet)
+	edgeRouterRoles = ctx.firstNPermuations(spec.serviceEdgeRouterPolicyCount, spec.edgeRouterAttrSet)
+	for i := 2; i < spec.serviceEdgeRouterPolicyCount; i++ {
+		serviceRoles[i] = append(serviceRoles[i], "@"+spec.services[0].Id)
+		edgeRouterRoles[i] = append(edgeRouterRoles[i], "@"+spec.edgeRouters[0].Id)
+		ctx.createServiceEdgeRouterPolicy(edgeRouterRoles[i], serviceRoles[i])
+	}
+}
+
+func (ctx *modelPerf) createServicePolicy(policyType string, identityRoles, serviceRoles []string) {
+	policyHandler := ctx.EdgeController.AppEnv.Handlers.ServicePolicy
+	id := eid.New()
+	policy := &model.ServicePolicy{
+		BaseEntity:    models.BaseEntity{Id: id},
+		Name:          id,
+		PolicyType:    policyType,
+		IdentityRoles: identityRoles,
+		ServiceRoles:  serviceRoles,
+	}
+	_, err := policyHandler.Create(policy)
+	ctx.Req.NoError(err)
+}
+
+func (ctx *modelPerf) createEdgeRouterPolicy(identityRoles, edgeRouterRoles []string) {
+	policyHandler := ctx.EdgeController.AppEnv.Handlers.EdgeRouterPolicy
+	id := eid.New()
+	policy := &model.EdgeRouterPolicy{
+		BaseEntity:      models.BaseEntity{Id: id},
+		Name:            id,
+		IdentityRoles:   identityRoles,
+		EdgeRouterRoles: edgeRouterRoles,
+	}
+	_, err := policyHandler.Create(policy)
+	ctx.Req.NoError(err)
+}
+
+func (ctx *modelPerf) createServiceEdgeRouterPolicy(edgeRouterRoles, serviceRoles []string) {
+	policyHandler := ctx.EdgeController.AppEnv.Handlers.ServiceEdgeRouterPolicy
+	id := eid.New()
+	policy := &model.ServiceEdgeRouterPolicy{
+		BaseEntity:      models.BaseEntity{Id: id},
+		Name:            id,
+		EdgeRouterRoles: edgeRouterRoles,
+		ServiceRoles:    serviceRoles,
+	}
+	_, err := policyHandler.Create(policy)
+	ctx.Req.NoError(err)
+}
+
+func (ctx *modelPerf) firstNPermuations(n int, v []string) [][]string {
+	var result [][]string
+	ctx.permutations(v, func(strings []string) bool {
+		result = append(result, strings)
+		return len(result) < n
+	})
+	return result
+}
+
+func (ctx *modelPerf) permutations(v []string, f func([]string) bool) {
+	ctx.permutationWith([]string{}, v, f)
+}
+
+func (ctx *modelPerf) permutationWith(base, v []string, f func([]string) bool) bool {
+	for i := 0; i < len(v); i++ {
+		var result []string
+		if len(base) > 0 {
+			result = append(result, base...)
+		}
+		result = append(result, v[i])
+		if !f(result) {
+			return false
+		}
+		if !ctx.permutationWith(result, v[i+1:], f) {
+			return false
+		}
+	}
+	return true
+}
+
+func newHistogram() metrics.Histogram {
+	return metrics.NewHistogram(metrics.NewExpDecaySample(128, 0.015))
+}
+
+func newPerfStats(ctx *TestContext, config *config.Config, description string, serviceId string, sessionType edge.SessionType) *perfStats {
+	zitiUrl, err := url.Parse(config.ZtAPI)
+	ctx.Req.NoError(err)
+
+	id, err := idloader.LoadIdentity(config.ID)
+	ctx.Req.NoError(err)
+
+	client, err := api.NewClient(zitiUrl, id.ClientTLSConfig())
+	ctx.Req.NoError(err)
+
+	return &perfStats{
+		TestContext:       ctx,
+		client:            client,
+		description:       description,
+		serviceId:         serviceId,
+		sessionType:       sessionType,
+		createApiSession:  newHistogram(),
+		refreshApiSession: newHistogram(),
+		getServices:       newHistogram(),
+		createSession:     newHistogram(),
+		refreshSession:    newHistogram(),
+	}
+}
+
+type perfStats struct {
+	*TestContext
+	description       string
+	serviceId         string
+	client            api.Client
+	sessionType       edge.SessionType
+	sessionId         string
+	createApiSession  metrics.Histogram
+	refreshApiSession metrics.Histogram
+	getServices       metrics.Histogram
+	createSession     metrics.Histogram
+	refreshSession    metrics.Histogram
+}
+
+func (s *perfStats) dumpStatsToStdOut() {
+	errWriter := &WriterWrapper{
+		Writer: os.Stdout,
+	}
+	s.dumpStats(errWriter)
+	s.Req.NoError(errWriter.GetError())
+}
+
+func (s *perfStats) dumpStats(w ErrorWriter) {
+	w.Println(s.description)
+	w.Println("=======================================")
+	s.logHistogram(w, "Create API Session", s.createApiSession)
+	s.logHistogram(w, "Refresh API Session", s.refreshApiSession)
+	s.logHistogram(w, "Get Services", s.getServices)
+	s.logHistogram(w, "Create Session", s.createSession)
+	s.logHistogram(w, "Refresh Session", s.refreshSession)
+}
+
+func (s *perfStats) logHistogram(w ErrorWriter, name string, h metrics.Histogram) {
+	w.Printf("%v:\n", name)
+	w.Printf("\tMin  : %vms\n", h.Min())
+	w.Printf("\tMax  : %vms\n", h.Max())
+	w.Printf("\tMean : %vms\n", h.Mean())
+	w.Printf("\t95th : %vms\n", h.Percentile(.95))
+}
+
+func (s *perfStats) collectStats(iterations int) {
+	s.repeat(iterations, s.timeCreateApiSession)
+	s.repeat(iterations, s.timeRefreshApiSession)
+	s.repeat(iterations, s.timeGetServices)
+	s.repeat(iterations, s.timeCreateSession)
+	s.repeat(iterations, s.timeRefreshSession)
+}
+
+func (s *perfStats) repeat(n int, f func()) {
+	for i := 0; i < n; i++ {
+		f()
+	}
+}
+
+func (s *perfStats) time(h metrics.Histogram, f func()) {
+	start := time.Now()
+	f()
+	h.Update(time.Now().Sub(start).Milliseconds())
+}
+
+func (s *perfStats) timeCreateApiSession() {
+	s.time(s.createApiSession, func() {
+		info, ok := sdkinfo.GetSdkInfo().(map[string]interface{})
+		s.Req.True(ok)
+		_, err := s.client.Login(info, []string{"all"})
+		s.Req.NoError(err)
+	})
+}
+
+func (s *perfStats) timeRefreshApiSession() {
+	s.time(s.createApiSession, func() {
+		_, err := s.client.Refresh()
+		s.Req.NoError(err)
+	})
+}
+
+func (s *perfStats) timeGetServices() {
+	s.time(s.getServices, func() {
+		_, err := s.client.GetServices()
+		s.Req.NoError(err)
+	})
+}
+
+func (s *perfStats) timeCreateSession() {
+	s.time(s.createSession, func() {
+		session, err := s.client.CreateSession(s.serviceId, s.sessionType)
+		s.Req.NoError(err)
+		s.sessionId = session.Id
+	})
+}
+
+func (s *perfStats) timeRefreshSession() {
+	s.time(s.createSession, func() {
+		_, err := s.client.RefreshSession(s.sessionId)
+		s.Req.NoError(err)
+	})
+}
+
+type ErrorWriter interface {
+	errorz.ErrorHolder
+	Write([]byte) int
+	Print(string)
+	Println(string)
+	Printf(s string, args ...interface{})
+}
+
+type WriterWrapper struct {
+	errorz.ErrorHolderImpl
+	io.Writer
+}
+
+func (w *WriterWrapper) Print(s string) {
+	w.Write([]byte(s))
+}
+
+func (w *WriterWrapper) Println(s string) {
+	w.Write([]byte(s))
+	w.Write([]byte("\n"))
+}
+
+func (w *WriterWrapper) Printf(s string, args ...interface{}) {
+	w.Write([]byte(fmt.Sprintf(s, args...)))
+}
+
+func (w *WriterWrapper) Write(b []byte) int {
+	if !w.HasError() {
+		n, err := w.Writer.Write(b)
+		w.SetError(err)
+		return n
+	}
+	return 0
+}

tests/testdata/.gitignore

@@ -1,3 +1,4 @@
 ctrl.db*
+*.db*
 edge-router/
-transit-router/
+transit-router/