From ad5dc292bbc9fbee601597ee8a3a04be826db42a Mon Sep 17 00:00:00 2001 From: Paul Lorenz Date: Fri, 31 Mar 2023 15:29:56 -0400 Subject: [PATCH 1/2] Improve list performance by using identity indexes rather than service indexes. Fixes #1388 --- controller/internal/routes/query_options.go | 2 +- .../internal/routes/service_api_model.go | 19 ++++++---- controller/internal/routes/service_router.go | 9 +++-- controller/model/base_manager.go | 4 +-- controller/model/edge_service_manager.go | 36 +++++++++---------- controller/model/posture_check_manager.go | 25 +++++++++++++ controller/model/posture_response_manager.go | 4 +++ controller/model/posture_response_model.go | 13 +++++++ controller/persistence/edge_service_store.go | 8 ++--- controller/persistence/identity_store.go | 26 ++++++++++++++ controller/persistence/policy_common.go | 8 ++--- controller/persistence/posture_check_store.go | 6 ++-- go.mod | 2 +- go.sum | 4 +-- tests/list_services_perf_test.go | 2 +- 15 files changed, 120 insertions(+), 48 deletions(-) diff --git a/controller/internal/routes/query_options.go b/controller/internal/routes/query_options.go index adc1db0bd..74475f35c 100644 --- a/controller/internal/routes/query_options.go +++ b/controller/internal/routes/query_options.go @@ -19,9 +19,9 @@ package routes import ( "fmt" "github.com/michaelquigley/pfxlog" + "github.com/openziti/foundation/v2/errorz" "github.com/openziti/storage/ast" "github.com/openziti/storage/boltz" - "github.com/openziti/foundation/v2/errorz" ) const ( diff --git a/controller/internal/routes/service_api_model.go b/controller/internal/routes/service_api_model.go index 98057e985..2ed48764d 100644 --- a/controller/internal/routes/service_api_model.go +++ b/controller/internal/routes/service_api_model.go @@ -191,16 +191,21 @@ func MapServiceToRestModel(ae *env.AppEnv, rc *response.RequestContext, service isCheckPassing, _ = ae.Managers.PostureResponse.Evaluate(rc.Identity.Id, rc.ApiSession.Id, postureCheck) validChecks[postureCheck.Id] = isCheckPassing } - timeout := postureCheck.TimeoutSeconds() - timeoutRemaining := postureCheck.TimeoutRemainingSeconds(rc.ApiSession.Id, ae.Managers.PostureResponse.PostureData(rc.Identity.Id)) - //determine if updatedAt is provided by the source posture check or the posture state - if lastUpdatedAt := postureCheck.LastUpdatedAt(rc.ApiSession.Id, ae.Managers.PostureResponse.PostureData(rc.Identity.Id)); lastUpdatedAt != nil { - if lastUpdatedAt.After(postureCheck.UpdatedAt) { - query.UpdatedAt = DateTimePtrOrNil(lastUpdatedAt) + var timeoutRemaining int64 + + ae.Managers.PostureResponse.WithPostureData(rc.Identity.Id, func(postureData *model.PostureData) { + timeoutRemaining = postureCheck.TimeoutRemainingSeconds(rc.ApiSession.Id, postureData) + + //determine if updatedAt is provided by the source posture check or the posture state + if lastUpdatedAt := postureCheck.LastUpdatedAt(rc.ApiSession.Id, postureData); lastUpdatedAt != nil { + if lastUpdatedAt.After(postureCheck.UpdatedAt) { + query.UpdatedAt = DateTimePtrOrNil(lastUpdatedAt) + } } + }) - } + timeout := postureCheck.TimeoutSeconds() query.IsPassing = &isCheckPassing query.TimeoutRemaining = &timeoutRemaining query.Timeout = &timeout diff --git a/controller/internal/routes/service_router.go b/controller/internal/routes/service_router.go index c09145fcd..9ea34006d 100644 --- a/controller/internal/routes/service_router.go +++ b/controller/internal/routes/service_router.go @@ -17,7 +17,6 @@ package routes import ( - "fmt" "github.com/go-openapi/runtime/middleware" "github.com/michaelquigley/pfxlog" clientService "github.com/openziti/edge-api/rest_client_api_server/operations/service" @@ -122,14 +121,14 @@ func (r *ServiceRouter) ListManagementServices(ae *env.AppEnv, rc *response.Requ List(rc, func(rc *response.RequestContext, queryOptions *PublicQueryOptions) (*QueryResult, error) { identity := rc.Identity if asId := rc.Request.URL.Query().Get("asIdentity"); asId != "" { - var err error - identity, err = ae.Managers.Identity.ReadOneByQuery(fmt.Sprintf(`id = "%v" or name = "%v"`, asId, asId)) - if err != nil { - return nil, err + identity, _ = ae.Managers.Identity.Read(asId) + if identity == nil { + identity, _ = ae.Managers.Identity.ReadByName(asId) } if identity == nil { return nil, boltz.NewNotFoundError("identity", "id or name", asId) } + rc.Identity = identity } // allow overriding config types diff --git a/controller/model/base_manager.go b/controller/model/base_manager.go index 9db55c897..4408f0c7d 100644 --- a/controller/model/base_manager.go +++ b/controller/model/base_manager.go @@ -112,7 +112,7 @@ func (self *baseEntityManager) BasePreparedList(query ast.Query) (*models.Entity func (self *baseEntityManager) BasePreparedListIndexed(cursorProvider ast.SetCursorProvider, query ast.Query) (*models.EntityListResult[models.Entity], error) { result := &models.EntityListResult[models.Entity]{Loader: self} - if err := self.preparedListIndexed(cursorProvider, query, result.Collect); err != nil { + if err := self.PreparedListIndexed(cursorProvider, query, result.Collect); err != nil { return nil, err } return result, nil @@ -124,7 +124,7 @@ func (self *baseEntityManager) PreparedListWithHandler(query ast.Query, resultHa }) } -func (self *baseEntityManager) preparedListIndexed(cursorProvider ast.SetCursorProvider, query ast.Query, resultHandler models.ListResultHandler) error { +func (self *baseEntityManager) PreparedListIndexed(cursorProvider ast.SetCursorProvider, query ast.Query, resultHandler models.ListResultHandler) error { return self.GetDb().View(func(tx *bbolt.Tx) error { return self.PreparedListIndexedWithTx(tx, cursorProvider, query, resultHandler) }) diff --git a/controller/model/edge_service_manager.go b/controller/model/edge_service_manager.go index 6e9f78dd7..858d0b115 100644 --- a/controller/model/edge_service_manager.go +++ b/controller/model/edge_service_manager.go @@ -17,7 +17,6 @@ package model import ( - "fmt" "github.com/michaelquigley/pfxlog" "github.com/openziti/edge/controller/persistence" "github.com/openziti/edge/pb/edge_cmd_pb" @@ -166,13 +165,6 @@ func (self *EdgeServiceManager) PublicQueryForIdentity(sessionIdentity *Identity } func (self *EdgeServiceManager) QueryForIdentity(identityId string, configTypes map[string]struct{}, query ast.Query) (*ServiceListResult, error) { - idFilterQueryString := fmt.Sprintf(`(anyOf(dialIdentities) = "%v" or anyOf(bindIdentities) = "%v")`, identityId, identityId) - idFilterQuery, err := ast.Parse(self.Store, idFilterQueryString) - if err != nil { - return nil, err - } - - query.SetPredicate(ast.NewAndExprNode(query.GetPredicate(), idFilterQuery.GetPredicate())) return self.queryServices(query, identityId, configTypes, false) } @@ -183,9 +175,15 @@ func (self *EdgeServiceManager) queryServices(query ast.Query, identityId string configTypes: configTypes, isAdmin: isAdmin, } - err := self.PreparedListWithHandler(query, result.collect) - if err != nil { - return nil, err + if isAdmin { + if err := self.PreparedListWithHandler(query, result.collect); err != nil { + return nil, err + } + } else { + cursorProvider := self.env.GetStores().Identity.GetIdentityServicesCursorProvider(identityId) + if err := self.PreparedListIndexed(cursorProvider, query, result.collect); err != nil { + return nil, err + } } return result, nil } @@ -326,6 +324,9 @@ func (self *EdgeServiceManager) GetPolicyPostureChecks(identityId, serviceId str postureCheckLinks := servicePolicyStore.GetLinkCollection(persistence.EntityTypePostureChecks) serviceLinks := servicePolicyStore.GetLinkCollection(db.EntityTypeServices) + policyNameSymbol := self.env.GetStores().ServicePolicy.GetSymbol(persistence.FieldName) + policyTypeSymbol := self.env.GetStores().ServicePolicy.GetSymbol(persistence.FieldServicePolicyType) + _ = self.GetDb().View(func(tx *bbolt.Tx) error { policyCursor := self.env.GetStores().Identity.GetRelatedEntitiesCursor(tx, identityId, persistence.EntityTypeServicePolicies, true) policyCursor = ast.NewFilteredCursor(policyCursor, func(policyId []byte) bool { @@ -337,18 +338,17 @@ func (self *EdgeServiceManager) GetPolicyPostureChecks(identityId, serviceId str policyIdStr := string(policyIdBytes) policyCursor.Next() - policy, err := self.env.GetStores().ServicePolicy.LoadOneById(tx, policyIdStr) - - if err != nil { - pfxlog.Logger().Errorf("could not retrieve policy by id [%s] to create posture queries for service id [%s]", policyIdStr, serviceId) - continue + policyName := boltz.FieldToString(policyNameSymbol.Eval(tx, policyIdBytes)) + policyType := persistence.PolicyTypeDial + if fieldType, policyTypeValue := policyTypeSymbol.Eval(tx, policyIdBytes); fieldType == boltz.TypeInt32 { + policyType = persistence.PolicyType(*boltz.BytesToInt32(policyTypeValue)) } //required to provide an entry for policies w/ no checks policyIdToChecks[policyIdStr] = &PolicyPostureChecks{ PostureChecks: []*PostureCheck{}, - PolicyType: policy.PolicyType, - PolicyName: policy.Name, + PolicyType: policyType, + PolicyName: *policyName, } cursor := postureCheckLinks.IterateLinks(tx, policyIdBytes) diff --git a/controller/model/posture_check_manager.go b/controller/model/posture_check_manager.go index 5b0349b9f..e1da8a5ce 100644 --- a/controller/model/posture_check_manager.go +++ b/controller/model/posture_check_manager.go @@ -18,6 +18,7 @@ package model import ( "fmt" + lru "github.com/hashicorp/golang-lru/v2" "github.com/openziti/edge/controller/persistence" "github.com/openziti/edge/pb/edge_cmd_pb" "github.com/openziti/fabric/controller/command" @@ -36,16 +37,31 @@ const ( ) func NewPostureCheckManager(env Env) *PostureCheckManager { + cache, err := lru.New[string, *PostureCheck](256) + if err != nil { + panic(err) + } manager := &PostureCheckManager{ baseEntityManager: newBaseEntityManager(env, env.GetStores().PostureCheck), + cache: cache, } manager.impl = manager network.RegisterManagerDecoder[*PostureCheck](env.GetHostController().GetNetwork().GetManagers(), manager) + + evictF := func(i ...interface{}) { + postureCheck := i[0].(*persistence.PostureCheck) + manager.cache.Remove(postureCheck.Id) + } + + manager.Store.AddListener(boltz.EventUpdate, evictF) + manager.Store.AddListener(boltz.EventDelete, evictF) + return manager } type PostureCheckManager struct { baseEntityManager + cache *lru.Cache[string, *PostureCheck] } func (self *PostureCheckManager) newModelEntity() edgeEntity { @@ -74,6 +90,9 @@ func (self *PostureCheckManager) ApplyUpdate(cmd *command.UpdateEntityCommand[*P } func (self *PostureCheckManager) Read(id string) (*PostureCheck, error) { + if postureCheck, ok := self.cache.Get(id); ok { + return postureCheck, nil + } modelEntity := &PostureCheck{} if err := self.readEntity(id, modelEntity); err != nil { return nil, err @@ -82,10 +101,16 @@ func (self *PostureCheckManager) Read(id string) (*PostureCheck, error) { } func (self *PostureCheckManager) readInTx(tx *bbolt.Tx, id string) (*PostureCheck, error) { + if postureCheck, ok := self.cache.Get(id); ok { + return postureCheck, nil + } + modelEntity := &PostureCheck{} if err := self.readEntityInTx(tx, id, modelEntity); err != nil { return nil, err } + self.cache.Add(id, modelEntity) + return modelEntity, nil } diff --git a/controller/model/posture_response_manager.go b/controller/model/posture_response_manager.go index 6328d463e..e1ba8da07 100644 --- a/controller/model/posture_response_manager.go +++ b/controller/model/posture_response_manager.go @@ -227,6 +227,10 @@ func (self *PostureResponseManager) PostureData(id string) *PostureData { return self.postureCache.PostureData(id) } +func (self *PostureResponseManager) WithPostureData(id string, f func(data *PostureData)) { + self.postureCache.WithPostureData(id, f) +} + func (self *PostureResponseManager) SetSdkInfo(identityId, apiSessionId string, sdkInfo *SdkInfo) { if identityId == "" || apiSessionId == "" || sdkInfo == nil { return diff --git a/controller/model/posture_response_model.go b/controller/model/posture_response_model.go index 6a54cb606..335e9216d 100644 --- a/controller/model/posture_response_model.go +++ b/controller/model/posture_response_model.go @@ -264,6 +264,19 @@ func (pc *PostureCache) PostureData(identityId string) *PostureData { return result } +func (pc *PostureCache) WithPostureData(identityId string, f func(data *PostureData)) { + pc.Upsert(identityId, false, func(exist bool, valueInMap *PostureData, newValue *PostureData) *PostureData { + var pd *PostureData + if exist { + pd = valueInMap + } else { + pd = newValue + } + f(pd) + return pd + }) +} + func (pc *PostureCache) ApiSessionCreated(args ...interface{}) { var apiSession *persistence.ApiSession if len(args) == 1 { diff --git a/controller/persistence/edge_service_store.go b/controller/persistence/edge_service_store.go index ed3883769..b3af6ef10 100644 --- a/controller/persistence/edge_service_store.go +++ b/controller/persistence/edge_service_store.go @@ -230,7 +230,7 @@ func (store *edgeServiceStoreImpl) Update(ctx boltz.MutateContext, entity boltz. // If a service is updated we need to notify everyone who has access to the identity, not just those who // have gained/lost access, since everyone will need to refresh the service. We will generate two events // for identities - cursor := store.dialIdentitiesCollection.IterateLinks(ctx.Tx(), []byte(id)) + cursor := store.dialIdentitiesCollection.IterateLinks(ctx.Tx(), []byte(id), true) for cursor.IsValid() { servicePolicyEvents = append(servicePolicyEvents, &ServiceEvent{ Type: ServiceUpdated, @@ -240,7 +240,7 @@ func (store *edgeServiceStoreImpl) Update(ctx boltz.MutateContext, entity boltz. cursor.Next() } - cursor = store.bindIdentitiesCollection.IterateLinks(ctx.Tx(), []byte(id)) + cursor = store.bindIdentitiesCollection.IterateLinks(ctx.Tx(), []byte(id), true) for cursor.IsValid() { servicePolicyEvents = append(servicePolicyEvents, &ServiceEvent{ Type: ServiceUpdated, @@ -273,7 +273,7 @@ func (store *edgeServiceStoreImpl) cleanupEdgeService(ctx boltz.MutateContext, i var servicePolicyEvents []*ServiceEvent - cursor := store.dialIdentitiesCollection.IterateLinks(ctx.Tx(), []byte(id)) + cursor := store.dialIdentitiesCollection.IterateLinks(ctx.Tx(), []byte(id), true) for cursor.IsValid() { servicePolicyEvents = append(servicePolicyEvents, &ServiceEvent{ Type: ServiceDialAccessLost, @@ -283,7 +283,7 @@ func (store *edgeServiceStoreImpl) cleanupEdgeService(ctx boltz.MutateContext, i cursor.Next() } - cursor = store.bindIdentitiesCollection.IterateLinks(ctx.Tx(), []byte(id)) + cursor = store.bindIdentitiesCollection.IterateLinks(ctx.Tx(), []byte(id), true) for cursor.IsValid() { servicePolicyEvents = append(servicePolicyEvents, &ServiceEvent{ Type: ServiceBindAccessLost, diff --git a/controller/persistence/identity_store.go b/controller/persistence/identity_store.go index 3acb2014a..dfcccd5cb 100644 --- a/controller/persistence/identity_store.go +++ b/controller/persistence/identity_store.go @@ -275,6 +275,7 @@ type IdentityStore interface { RemoveServiceConfigs(tx *bbolt.Tx, identityId string, serviceConfigs ...ServiceConfig) error GetServiceConfigs(tx *bbolt.Tx, identityId string) ([]ServiceConfig, error) LoadServiceConfigsByServiceAndType(tx *bbolt.Tx, identityId string, configTypes map[string]struct{}) map[string]map[string]map[string]interface{} + GetIdentityServicesCursorProvider(identityId string) ast.SetCursorProvider } func newIdentityStore(stores *stores) *identityStoreImpl { @@ -652,3 +653,28 @@ func (store *identityStoreImpl) LoadServiceConfigsByServiceAndType(tx *bbolt.Tx, func (store *identityStoreImpl) GetRoleAttributesCursorProvider(values []string, semantic string) (ast.SetCursorProvider, error) { return store.getRoleAttributesCursorProvider(store.indexRoleAttributes, values, semantic) } + +func (store *identityStoreImpl) GetIdentityServicesCursorProvider(identityId string) ast.SetCursorProvider { + provider := &IdentityServicesCursorProvider{ + store: store, + identityId: identityId, + } + return provider.Cursor +} + +type IdentityServicesCursorProvider struct { + store *identityStoreImpl + identityId string +} + +func (self *IdentityServicesCursorProvider) Cursor(tx *bbolt.Tx, forward bool) ast.SetCursor { + fst := self.store.dialServicesCollection.IterateLinks(tx, []byte(self.identityId), forward) + snd := self.store.bindServicesCollection.IterateLinks(tx, []byte(self.identityId), forward) + if !fst.IsValid() { + return snd + } + if !snd.IsValid() { + return fst + } + return ast.NewUnionSetCursor(fst, snd, forward) +} diff --git a/controller/persistence/policy_common.go b/controller/persistence/policy_common.go index f7145a193..c2a6f7a47 100644 --- a/controller/persistence/policy_common.go +++ b/controller/persistence/policy_common.go @@ -4,10 +4,10 @@ import ( "bytes" "fmt" "github.com/michaelquigley/pfxlog" - "github.com/openziti/storage/ast" - "github.com/openziti/storage/boltz" "github.com/openziti/foundation/v2/errorz" "github.com/openziti/foundation/v2/stringz" + "github.com/openziti/storage/ast" + "github.com/openziti/storage/boltz" "github.com/pkg/errors" "github.com/sirupsen/logrus" "go.etcd.io/bbolt" @@ -19,14 +19,14 @@ type serviceEventHandler struct { } func (self *serviceEventHandler) addServiceUpdatedEvent(store *baseStore, tx *bbolt.Tx, serviceId []byte) { - cursor := store.stores.edgeService.bindIdentitiesCollection.IterateLinks(tx, serviceId) + cursor := store.stores.edgeService.bindIdentitiesCollection.IterateLinks(tx, serviceId, true) for cursor != nil && cursor.IsValid() { self.addServiceEvent(tx, cursor.Current(), serviceId, ServiceUpdated) cursor.Next() } - cursor = store.stores.edgeService.dialIdentitiesCollection.IterateLinks(tx, serviceId) + cursor = store.stores.edgeService.dialIdentitiesCollection.IterateLinks(tx, serviceId, true) for cursor != nil && cursor.IsValid() { self.addServiceEvent(tx, cursor.Current(), serviceId, ServiceUpdated) cursor.Next() diff --git a/controller/persistence/posture_check_store.go b/controller/persistence/posture_check_store.go index b05b0bafd..53d4461eb 100644 --- a/controller/persistence/posture_check_store.go +++ b/controller/persistence/posture_check_store.go @@ -18,9 +18,9 @@ package persistence import ( "github.com/michaelquigley/pfxlog" + "github.com/openziti/foundation/v2/errorz" "github.com/openziti/storage/ast" "github.com/openziti/storage/boltz" - "github.com/openziti/foundation/v2/errorz" "go.etcd.io/bbolt" ) @@ -227,13 +227,13 @@ func (store *postureCheckStoreImpl) Update(ctx boltz.MutateContext, entity boltz func (store *postureCheckStoreImpl) createServiceChangeEvents(tx *bbolt.Tx, id string) { eh := &serviceEventHandler{} - cursor := store.bindServicesCollection.IterateLinks(tx, []byte(id)) + cursor := store.bindServicesCollection.IterateLinks(tx, []byte(id), true) for cursor.IsValid() { eh.addServiceUpdatedEvent(store.baseStore, tx, cursor.Current()) cursor.Next() } - cursor = store.dialServicesCollection.IterateLinks(tx, []byte(id)) + cursor = store.dialServicesCollection.IterateLinks(tx, []byte(id), true) for cursor.IsValid() { eh.addServiceUpdatedEvent(store.baseStore, tx, cursor.Current()) cursor.Next() diff --git a/go.mod b/go.mod index b85ec29ea..01dd30089 100644 --- a/go.mod +++ b/go.mod @@ -36,7 +36,7 @@ require ( github.com/openziti/jwks v1.0.3 github.com/openziti/metrics v1.2.16 github.com/openziti/sdk-golang v0.18.76 - github.com/openziti/storage v0.1.45 + github.com/openziti/storage v0.1.46 github.com/openziti/transport/v2 v2.0.68 github.com/openziti/x509-claims v1.0.3 github.com/openziti/xweb/v2 v2.0.2 diff --git a/go.sum b/go.sum index a04f68de4..4e2cdcd81 100644 --- a/go.sum +++ b/go.sum @@ -469,8 +469,8 @@ github.com/openziti/metrics v1.2.16 h1:2aJCrJbjbu9mMYb1+XLc6GIFbDJEfujs3sF11d1ZQ github.com/openziti/metrics v1.2.16/go.mod h1:OhtLTUKBrhwY5pV17KtQ1OvRPxMqu8K/IaqUuoqUfg0= github.com/openziti/sdk-golang v0.18.76 h1:D+UW1lpGHBBgfVrObpauq9RvJV/TFPdEEshfuPkiMcI= github.com/openziti/sdk-golang v0.18.76/go.mod h1:kw/5rWDLwx52iwW/4a0VRQMUi7GPSI3aETx+G5TvdFQ= -github.com/openziti/storage v0.1.45 h1:0ZBW0Fyx6qbIoHWK9PKJod0+BFeF/TPX6gZJ+cg+4Xo= -github.com/openziti/storage v0.1.45/go.mod h1:RptgJKYVJwEJb0byGpvgZPEZbpLtdJYFGlplvxf6wRQ= +github.com/openziti/storage v0.1.46 h1:2PH1sidrzzbsDwpAM+kpBFV3ANeDKGKWf7C+P5VERUY= +github.com/openziti/storage v0.1.46/go.mod h1:RptgJKYVJwEJb0byGpvgZPEZbpLtdJYFGlplvxf6wRQ= github.com/openziti/transport/v2 v2.0.68 h1:58QTO10nLakqCqGwABC/codq9O1WjtFSByhwlQvF1sc= github.com/openziti/transport/v2 v2.0.68/go.mod h1:dKNuMCulCqNyhkoLWPUzB1CLdhrslSgWsJPKWwrUBTo= github.com/openziti/x509-claims v1.0.3 h1:HNdQ8Nf1agB3lBs1gahcO6zfkeS4S5xoQ2/PkY4HRX0= diff --git a/tests/list_services_perf_test.go b/tests/list_services_perf_test.go index ea9237c7e..46fe30e5d 100644 --- a/tests/list_services_perf_test.go +++ b/tests/list_services_perf_test.go @@ -36,7 +36,7 @@ func TestServicePerf(t *testing.T) { go func() { for i := 0; i < 100; i++ { start := time.Now() - _ = ctx.AdminManagementSession.requireQuery("services?asIdentity=" + identityId) + _ = ctx.AdminManagementSession.requireQuery("services?asIdentity=" + identityId + "&filter=" + url.QueryEscape("true limit 100")) lookupTimer.UpdateSince(start) } close(doneC) From bd9b6a713bb2ebe682511ad26b42fe9ed369750a Mon Sep 17 00:00:00 2001 From: Paul Lorenz Date: Mon, 3 Apr 2023 11:34:37 -0400 Subject: [PATCH 2/2] Pull some tweaks over from denorm branch --- .../internal/routes/service_api_model.go | 5 -- controller/persistence/stores.go | 2 + tests/ats-ctrl.yml | 2 +- tests/context.go | 21 ++---- tests/list_services_perf_test.go | 71 +++++++++++++++++++ 5 files changed, 81 insertions(+), 20 deletions(-) diff --git a/controller/internal/routes/service_api_model.go b/controller/internal/routes/service_api_model.go index 2ed48764d..f0e164257 100644 --- a/controller/internal/routes/service_api_model.go +++ b/controller/internal/routes/service_api_model.go @@ -177,11 +177,6 @@ func MapServiceToRestModel(ae *env.AppEnv, rc *response.RequestContext, service pfxlog.Logger().Errorf("attempting to render API response for policy type [%s] for policy id [%s], unknown type expected dial/bind", policyPostureChecks.PolicyType, policyId) } - if policyPostureChecks == nil { - pfxlog.Logger().Errorf("unexpected nil policyPostureCheck attempting to render posture queries for service [%s]", service.Id) - continue - } - for _, postureCheck := range policyPostureChecks.PostureChecks { query := PostureCheckToQueries(postureCheck) diff --git a/controller/persistence/stores.go b/controller/persistence/stores.go index 373cce669..055d8247e 100644 --- a/controller/persistence/stores.go +++ b/controller/persistence/stores.go @@ -30,6 +30,7 @@ import ( type Stores struct { DbProvider DbProvider EventualEventer EventualEventer + internal *stores // fabric stores Router db.RouterStore @@ -201,6 +202,7 @@ func NewBoltStores(dbProvider DbProvider) (*Stores, error) { externalStores := &Stores{ DbProvider: dbProvider, + internal: internalStores, Terminator: dbProvider.GetStores().Terminator, Router: dbProvider.GetStores().Router, diff --git a/tests/ats-ctrl.yml b/tests/ats-ctrl.yml index 4b8e1c4fa..64303f5d7 100644 --- a/tests/ats-ctrl.yml +++ b/tests/ats-ctrl.yml @@ -7,7 +7,7 @@ v: 3 # memory: # path: ctrl.memprof -db: testdata/${ZITI_TEST_DB}.db +db: ${ZITI_TEST_DB} identity: cert: testdata/ca/intermediate/certs/ctrl-client.cert.pem diff --git a/tests/context.go b/tests/context.go index 592f4ddfe..a48630f75 100644 --- a/tests/context.go +++ b/tests/context.go @@ -51,7 +51,6 @@ import ( "net/http/cookiejar" "net/url" "os" - "path/filepath" "runtime" "strings" "sync" @@ -292,10 +291,10 @@ func (ctx *TestContext) NewClientComponentsWithClientCert(cert *x509.Certificate } func (ctx *TestContext) StartServer() { - ctx.StartServerFor("default", true) + ctx.StartServerFor("testdata/default.db", true) } -func (ctx *TestContext) StartServerFor(test string, clean bool) { +func (ctx *TestContext) StartServerFor(testDb string, clean bool) { if ctx.LogLevel != "" { if level, err := logrus.ParseLevel(ctx.LogLevel); err == nil { logrus.StandardLogger().SetLevel(level) @@ -305,19 +304,13 @@ func (ctx *TestContext) StartServerFor(test string, clean bool) { log := pfxlog.Logger() _ = os.Mkdir("testdata", os.FileMode(0755)) if clean { - err := filepath.Walk("testdata", func(path string, info os.FileInfo, err error) error { - if err == nil { - if !info.IsDir() && strings.HasPrefix(info.Name(), test+".db") { - pfxlog.Logger().Infof("removing test bolt file or backup: %v", path) - err = os.Remove(path) - } - } - return err - }) - ctx.Req.NoError(err) + err := os.Remove(testDb) + if !os.IsNotExist(err) { + ctx.Req.NoError(err) + } } - err := os.Setenv("ZITI_TEST_DB", test) + err := os.Setenv("ZITI_TEST_DB", testDb) ctx.Req.NoError(err) log.Info("loading config") diff --git a/tests/list_services_perf_test.go b/tests/list_services_perf_test.go index 46fe30e5d..270332821 100644 --- a/tests/list_services_perf_test.go +++ b/tests/list_services_perf_test.go @@ -3,10 +3,16 @@ package tests import ( + "encoding/json" "fmt" + "github.com/openziti/edge/controller/model" + "github.com/openziti/edge/controller/persistence" "github.com/openziti/foundation/v2/concurrenz" "github.com/openziti/metrics" + "go.etcd.io/bbolt" "net/url" + "os" + "sort" "testing" "time" ) @@ -54,3 +60,68 @@ func TestServicePerf(t *testing.T) { fmt.Printf("p99: %v\n", time.Duration(lookupTimeSnapshot.P99).String()) fmt.Printf("count: %v\n", lookupTimeSnapshot.Count) } + +func Test_ExportIdentityServicePostureChecks(t *testing.T) { + ctx := NewTestContext(t) + defer ctx.Teardown() + ctx.StartServerFor("/home/plorenz/work/nf/var/db/ctrl.db", false) + + stores := ctx.EdgeController.AppEnv.GetStores() + managers := ctx.EdgeController.AppEnv.Managers + identityManager := managers.Identity + + bindServices := stores.Identity.GetRefCountedLinkCollection(persistence.FieldIdentityBindServices) + dialServices := stores.Identity.GetRefCountedLinkCollection(persistence.FieldIdentityDialServices) + + type identityServicePostureChecks struct { + Id string + Services map[string]map[string]*model.PolicyPostureChecks + } + + outputFile, err := os.Create("/home/plorenz/tmp/posture-checks.json") + ctx.Req.NoError(err) + + err = identityManager.GetDb().View(func(tx *bbolt.Tx) error { + identityIds, _, err := stores.Identity.QueryIds(tx, "limit none") + if err != nil { + return err + } + + for idx, identityId := range identityIds { + fmt.Printf("%v of %v: %v\n", idx, len(identityIds), identityId) + output := &identityServicePostureChecks{ + Id: identityId, + Services: map[string]map[string]*model.PolicyPostureChecks{}, + } + + serviceIdSet := map[string]struct{}{} + for cursor := bindServices.IterateLinks(tx, []byte(identityId), true); cursor.IsValid(); cursor.Next() { + serviceId := string(cursor.Current()) + serviceIdSet[serviceId] = struct{}{} + } + for cursor := dialServices.IterateLinks(tx, []byte(identityId), true); cursor.IsValid(); cursor.Next() { + serviceId := string(cursor.Current()) + serviceIdSet[serviceId] = struct{}{} + } + serviceIds := make([]string, 0, len(serviceIdSet)) + for serviceId := range serviceIdSet { + serviceIds = append(serviceIds, serviceId) + } + sort.Strings(serviceIds) + + for _, serviceId := range serviceIds { + postureChecks := managers.EdgeService.GetPolicyPostureChecks(identityId, serviceId) + output.Services[serviceId] = postureChecks + } + bytes, err := json.Marshal(output) + ctx.Req.NoError(err) + _, err = outputFile.Write(bytes) + ctx.Req.NoError(err) + _, err = outputFile.WriteString("\n") + ctx.Req.NoError(err) + } + + return nil + }) + ctx.Req.NoError(err) +}