From b8f944326ffe77fd4e4eeb7f98c46e805bbdfc7c Mon Sep 17 00:00:00 2001 From: Paul Lorenz Date: Thu, 1 Aug 2024 11:29:28 -0400 Subject: [PATCH] Ensure controller is initialized before raft. Fixes #2279 --- common/concurrency/initialized_notifier.go | 50 +++++++ controller/api_impl/context.go | 2 +- controller/api_impl/responder.go | 8 +- controller/api_impl/routers.go | 8 +- controller/api_impl/wrapper.go | 122 +---------------- controller/controller.go | 45 +++++-- .../internal/routes/controller_api_model.go | 2 +- controller/internal/routes/version_router.go | 22 +-- controller/server/controller.go | 21 --- controller/{server => webapis}/client-api.go | 29 ++-- .../fabric-api-wrapper.go} | 14 +- .../fabric-management-api.go} | 49 +++---- controller/webapis/fabric-request-wrapper.go | 127 ++++++++++++++++++ .../{server => webapis}/management-api.go | 14 +- .../{api_impl => webapis}/metrics-api.go | 11 +- controller/{server => webapis}/oidc-api.go | 7 +- controller/{ => webapis}/versions.go | 2 +- tests/fabric_context.go | 4 +- tests/identity_api_session_activity_test.go | 4 +- tests/link_test.go | 4 + 20 files changed, 312 insertions(+), 233 deletions(-) create mode 100644 common/concurrency/initialized_notifier.go rename controller/{server => webapis}/client-api.go (84%) rename controller/{server/fabric-api.go => webapis/fabric-api-wrapper.go} (95%) rename controller/{api_impl/management-api.go => webapis/fabric-management-api.go} (69%) create mode 100644 controller/webapis/fabric-request-wrapper.go rename controller/{server => webapis}/management-api.go (92%) rename controller/{api_impl => webapis}/metrics-api.go (94%) rename controller/{server => webapis}/oidc-api.go (97%) rename controller/{ => webapis}/versions.go (98%) diff --git a/common/concurrency/initialized_notifier.go b/common/concurrency/initialized_notifier.go new file mode 100644 index 000000000..b65155bbf --- /dev/null +++ b/common/concurrency/initialized_notifier.go @@ -0,0 +1,50 @@ +/* + Copyright NetFoundry Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package concurrency + +import ( + "github.com/michaelquigley/pfxlog" + "sync/atomic" +) + +type InitState interface { + WaitTillInitialized() + MarkInitialized() +} + +func NewInitState() InitState { + return &channelInitState{ + c: make(chan struct{}), + } +} + +type channelInitState struct { + c chan struct{} + initialized atomic.Bool +} + +func (self *channelInitState) WaitTillInitialized() { + <-self.c +} + +func (self *channelInitState) MarkInitialized() { + if self.initialized.CompareAndSwap(false, true) { + close(self.c) + } else { + pfxlog.Logger().Panic("initialized marked complete more than once") + } +} diff --git a/controller/api_impl/context.go b/controller/api_impl/context.go index c6c552311..75ba0646a 100644 --- a/controller/api_impl/context.go +++ b/controller/api_impl/context.go @@ -37,7 +37,7 @@ func NewRequestContext(rw http.ResponseWriter, r *http.Request) api.RequestConte Request: r, } - requestContext.Responder = api.NewResponder(requestContext, fabricResponseMapper{}) + requestContext.Responder = api.NewResponder(requestContext, FabricResponseMapper{}) return requestContext } diff --git a/controller/api_impl/responder.go b/controller/api_impl/responder.go index 448fdadc3..10aa7be0e 100644 --- a/controller/api_impl/responder.go +++ b/controller/api_impl/responder.go @@ -1,9 +1,9 @@ package api_impl import ( + "github.com/openziti/foundation/v2/errorz" "github.com/openziti/ziti/controller/api" "github.com/openziti/ziti/controller/rest_model" - "github.com/openziti/foundation/v2/errorz" "net/http" ) @@ -28,16 +28,16 @@ func RespondWithOk(responder api.Responder, data interface{}, meta *rest_model.M }, http.StatusOK) } -type fabricResponseMapper struct{} +type FabricResponseMapper struct{} -func (self fabricResponseMapper) EmptyOkData() interface{} { +func (self FabricResponseMapper) EmptyOkData() interface{} { return &rest_model.Empty{ Data: map[string]interface{}{}, Meta: &rest_model.Meta{}, } } -func (self fabricResponseMapper) MapApiError(requestId string, apiError *errorz.ApiError) interface{} { +func (self FabricResponseMapper) MapApiError(requestId string, apiError *errorz.ApiError) interface{} { return &rest_model.APIErrorEnvelope{ Error: ToRestModel(apiError, requestId), Meta: &rest_model.Meta{ diff --git a/controller/api_impl/routers.go b/controller/api_impl/routers.go index d7a122309..a2c81a3a2 100644 --- a/controller/api_impl/routers.go +++ b/controller/api_impl/routers.go @@ -1,11 +1,13 @@ package api_impl -import "github.com/openziti/ziti/controller/rest_server/operations" +import ( + "github.com/openziti/ziti/controller/rest_server/operations" +) -var routers []Router +var Routers []Router func AddRouter(router Router) { - routers = append(routers, router) + Routers = append(Routers, router) } type Router interface { diff --git a/controller/api_impl/wrapper.go b/controller/api_impl/wrapper.go index 43ea53cd6..d15bd2ca0 100644 --- a/controller/api_impl/wrapper.go +++ b/controller/api_impl/wrapper.go @@ -1,134 +1,16 @@ package api_impl import ( - "crypto/x509" - "github.com/go-openapi/runtime" - openApiMiddleware "github.com/go-openapi/runtime/middleware" - "github.com/michaelquigley/pfxlog" - "github.com/openziti/ziti/common/build" + "github.com/go-openapi/runtime/middleware" "github.com/openziti/ziti/controller/api" - "github.com/openziti/ziti/controller/apierror" "github.com/openziti/ziti/controller/network" - "github.com/openziti/ziti/controller/rest_server" - "github.com/openziti/foundation/v2/errorz" - "github.com/openziti/identity" - "github.com/pkg/errors" "net/http" - "time" ) -var requestWrapper RequestWrapper - -func OverrideRequestWrapper(rw RequestWrapper) { - if requestWrapper != nil { - pfxlog.Logger().Warn("requestWrapper overridden more than once") - } - requestWrapper = rw -} - type RequestHandler func(network *network.Network, rc api.RequestContext) type RequestWrapper interface { - WrapRequest(handler RequestHandler, request *http.Request, entityId, entitySubId string) openApiMiddleware.Responder + WrapRequest(handler RequestHandler, request *http.Request, entityId, entitySubId string) middleware.Responder WrapHttpHandler(handler http.Handler) http.Handler WrapWsHandler(handler http.Handler) http.Handler } - -type FabricRequestWrapper struct { - nodeId identity.Identity - network *network.Network -} - -func (self *FabricRequestWrapper) WrapRequest(handler RequestHandler, request *http.Request, entityId, entitySubId string) openApiMiddleware.Responder { - return openApiMiddleware.ResponderFunc(func(writer http.ResponseWriter, producer runtime.Producer) { - rc, err := api.GetRequestContextFromHttpContext(request) - - if rc == nil { - rc = NewRequestContext(writer, request) - } - - rc.SetProducer(producer) - rc.SetEntityId(entityId) - rc.SetEntitySubId(entitySubId) - - if err != nil { - pfxlog.Logger().WithError(err).Error("could not retrieve request context") - rc.RespondWithError(err) - return - } - - handler(self.network, rc) - }) -} - -func (self *FabricRequestWrapper) WrapHttpHandler(handler http.Handler) http.Handler { - wrapper := http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { - if r.URL.Path == FabricRestApiSpecUrl { - rw.Header().Set("content-type", "application/json") - rw.WriteHeader(http.StatusOK) - _, _ = rw.Write(rest_server.SwaggerJSON) - return - } - - rc := NewRequestContext(rw, r) - - if err := self.verifyCert(r); err != nil { - rc.RespondWithError(apierror.NewInvalidAuth()) - return - } - - api.AddRequestContextToHttpContext(r, rc) - - //after request context is filled so that api session is present for session expiration headers - buildInfo := build.GetBuildInfo() - if buildInfo != nil { - rc.GetResponseWriter().Header().Set(ServerHeader, "ziti-controller/"+buildInfo.Version()) - } - - handler.ServeHTTP(rw, r) - }) - - return api.TimeoutHandler(api.WrapCorsHandler(wrapper), 10*time.Second, apierror.NewTimeoutError(), fabricResponseMapper{}) -} - -func (self *FabricRequestWrapper) WrapWsHandler(handler http.Handler) http.Handler { - wrapper := http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { - if err := self.verifyCert(r); err != nil { - rc := NewRequestContext(rw, r) - rc.RespondWithError(apierror.NewInvalidAuth()) - return - } - - handler.ServeHTTP(rw, r) - }) - - return wrapper -} - -func (self *FabricRequestWrapper) verifyCert(r *http.Request) error { - certificates := r.TLS.PeerCertificates - if len(certificates) == 0 { - return errors.New("no certificates provided, unable to verify dialer") - } - - config := self.nodeId.ServerTLSConfig() - - opts := x509.VerifyOptions{ - Roots: config.RootCAs, - Intermediates: x509.NewCertPool(), - KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageAny}, - } - - var errorList errorz.MultipleErrors - - for _, cert := range certificates { - if _, err := cert.Verify(opts); err == nil { - return nil - } else { - errorList = append(errorList, err) - } - } - - //goland:noinspection GoNilness - return errorList.ToError() -} diff --git a/controller/controller.go b/controller/controller.go index 92bc4b51f..af2bc8a95 100644 --- a/controller/controller.go +++ b/controller/controller.go @@ -26,11 +26,13 @@ import ( "github.com/openziti/transport/v2" "github.com/openziti/transport/v2/tls" "github.com/openziti/ziti/common/capabilities" + "github.com/openziti/ziti/common/concurrency" "github.com/openziti/ziti/controller/config" "github.com/openziti/ziti/controller/env" "github.com/openziti/ziti/controller/event" "github.com/openziti/ziti/controller/events" "github.com/openziti/ziti/controller/handler_peer_ctrl" + "github.com/openziti/ziti/controller/webapis" "github.com/openziti/ziti/controller/xt_sticky" "github.com/openziti/ziti/controller/zac" "math/big" @@ -54,7 +56,6 @@ import ( fabricMetrics "github.com/openziti/ziti/common/metrics" "github.com/openziti/ziti/common/pb/ctrl_pb" "github.com/openziti/ziti/common/profiler" - "github.com/openziti/ziti/controller/api_impl" "github.com/openziti/ziti/controller/command" "github.com/openziti/ziti/controller/handler_ctrl" "github.com/openziti/ziti/controller/network" @@ -95,6 +96,8 @@ type Controller struct { apiData map[string][]event.ApiAddress apiDataBytes []byte apiDataOnce sync.Once + + xwebInitialized concurrency.InitState } func (c *Controller) GetPeerSigners() []*x509.Certificate { @@ -222,8 +225,11 @@ func NewController(cfg *config.Config, versionProvider versions.VersionProvider) metricsRegistry: metricRegistry, versionProvider: versionProvider, eventDispatcher: events.NewDispatcher(shutdownC), + xwebInitialized: concurrency.NewInitState(), } + c.xweb = xweb.NewDefaultInstance(c.xwebFactoryRegistry, c.config.Id) + if cfg.Raft != nil { c.raftController = raft.NewController(c, c) if err := c.raftController.Init(); err != nil { @@ -287,24 +293,46 @@ func (c *Controller) initWeb() { logrus.WithError(err).Fatalf("failed to create health checker") } - c.xweb = xweb.NewDefaultInstance(c.xwebFactoryRegistry, c.config.Id) - - if err := c.xweb.GetRegistry().Add(health.NewHealthCheckApiFactory(healthChecker)); err != nil { + if err = c.xweb.GetRegistry().Add(health.NewHealthCheckApiFactory(healthChecker)); err != nil { logrus.WithError(err).Fatalf("failed to create health checks api factory") } - if err := c.xweb.GetRegistry().Add(api_impl.NewManagementApiFactory(c.config.Id, c.network, &c.xmgmts)); err != nil { + if err = c.xweb.GetRegistry().Add(webapis.NewFabricManagementApiFactory(c.config.Id, c.network, &c.xmgmts)); err != nil { logrus.WithError(err).Fatalf("failed to create management api factory") } - if err := c.xweb.GetRegistry().Add(api_impl.NewMetricsApiFactory(c.config.Id, c.network)); err != nil { + if err = c.xweb.GetRegistry().Add(webapis.NewMetricsApiFactory(c.config.Id, c.network)); err != nil { logrus.WithError(err).Fatalf("failed to create metrics api factory") } - if err := c.xweb.GetRegistry().Add(zac.NewZitiAdminConsoleFactory()); err != nil { + if err = c.xweb.GetRegistry().Add(zac.NewZitiAdminConsoleFactory()); err != nil { logrus.WithError(err).Fatalf("failed to create single page application factory") } + if c.IsEdgeEnabled() { + managementApiFactory := webapis.NewManagementApiFactory(c.env) + clientApiFactory := webapis.NewClientApiFactory(c.env) + oidcApiFactory := webapis.NewOidcApiFactory(c.env) + + if err = c.xweb.GetRegistry().Add(managementApiFactory); err != nil { + pfxlog.Logger().Fatalf("failed to create Edge Management API factory: %v", err) + } + + if err = c.xweb.GetRegistry().Add(clientApiFactory); err != nil { + pfxlog.Logger().Fatalf("failed to create Edge Client API factory: %v", err) + } + + if err = c.xweb.GetRegistry().Add(oidcApiFactory); err != nil { + pfxlog.Logger().Fatalf("failed to create OIDC API factory: %v", err) + } + + webapis.OverrideRequestWrapper(webapis.NewFabricApiWrapper(c.env)) + } + c.xwebInitialized.MarkInitialized() +} + +func (c *Controller) IsEdgeEnabled() bool { + return c.config.Edge.Enabled } func (c *Controller) Run() error { @@ -366,7 +394,7 @@ func (c *Controller) Run() error { go underlayDispatcher.Run() - if err := c.config.Configure(c.xweb); err != nil { + if err = c.config.Configure(c.xweb); err != nil { panic(err) } @@ -634,6 +662,7 @@ func getApiPath(binding string) string { func (c *Controller) GetApiAddresses() (map[string][]event.ApiAddress, []byte) { c.apiDataOnce.Do(func() { + c.xwebInitialized.WaitTillInitialized() xwebConfig := c.xweb.GetConfig() apiData := map[string][]event.ApiAddress{} diff --git a/controller/internal/routes/controller_api_model.go b/controller/internal/routes/controller_api_model.go index e4aa93b12..6c3e3f510 100644 --- a/controller/internal/routes/controller_api_model.go +++ b/controller/internal/routes/controller_api_model.go @@ -18,10 +18,10 @@ package routes import ( "github.com/openziti/edge-api/rest_model" - edgeController "github.com/openziti/ziti/controller" "github.com/openziti/ziti/controller/env" "github.com/openziti/ziti/controller/model" "github.com/openziti/ziti/controller/response" + edgeController "github.com/openziti/ziti/controller/webapis" ) const EntityNameController = "controllers" diff --git a/controller/internal/routes/version_router.go b/controller/internal/routes/version_router.go index 9b2663d3c..8fadb605e 100644 --- a/controller/internal/routes/version_router.go +++ b/controller/internal/routes/version_router.go @@ -24,10 +24,10 @@ import ( "github.com/openziti/edge-api/rest_model" "github.com/openziti/xweb/v2" "github.com/openziti/ziti/common/build" - "github.com/openziti/ziti/controller" "github.com/openziti/ziti/controller/env" "github.com/openziti/ziti/controller/internal/permissions" "github.com/openziti/ziti/controller/response" + "github.com/openziti/ziti/controller/webapis" "runtime" "sync" ) @@ -85,13 +85,13 @@ func (ir *VersionRouter) List(ae *env.AppEnv, rc *response.RequestContext) { RuntimeVersion: runtime.Version(), Version: buildInfo.Version(), APIVersions: map[string]map[string]rest_model.APIVersion{ - controller.ClientApiBinding: {controller.VersionV1: mapApiVersionToRestModel(controller.ClientRestApiBaseUrlV1)}, - controller.ManagementApiBinding: {controller.VersionV1: mapApiVersionToRestModel(controller.ManagementRestApiBaseUrlV1)}, + webapis.ClientApiBinding: {webapis.VersionV1: mapApiVersionToRestModel(webapis.ClientRestApiBaseUrlV1)}, + webapis.ManagementApiBinding: {webapis.VersionV1: mapApiVersionToRestModel(webapis.ManagementRestApiBaseUrlV1)}, }, Capabilities: []string{}, } - for apiBinding, apiVersionToPathMap := range controller.AllApiBindingVersions { + for apiBinding, apiVersionToPathMap := range webapis.AllApiBindingVersions { ir.cachedVersions.APIVersions[apiBinding] = map[string]rest_model.APIVersion{} for apiVersion, apiPath := range apiVersionToPathMap { @@ -120,7 +120,7 @@ func (ir *VersionRouter) List(ae *env.AppEnv, rc *response.RequestContext) { for _, serverConfig := range ae.HostController.GetXWebInstance().GetConfig().ServerConfigs { for _, api := range serverConfig.APIs { - if api.Binding() == controller.OidcApiBinding { + if api.Binding() == webapis.OidcApiBinding { oidcEnabled = true break } @@ -139,7 +139,7 @@ func (ir *VersionRouter) List(ae *env.AppEnv, rc *response.RequestContext) { } } - ir.cachedVersions.APIVersions[controller.LegacyClientApiBinding] = ir.cachedVersions.APIVersions[controller.ClientApiBinding] + ir.cachedVersions.APIVersions[webapis.LegacyClientApiBinding] = ir.cachedVersions.APIVersions[webapis.ClientApiBinding] if oidcEnabled { ir.cachedVersions.Capabilities = append(ir.cachedVersions.Capabilities, string(rest_model.CapabilitiesOIDCAUTH)) @@ -166,11 +166,11 @@ func (ir *VersionRouter) ListCapabilities(_ *env.AppEnv, rc *response.RequestCon func apiBindingToPath(binding string) string { switch binding { case "edge": - return controller.ClientRestApiBaseUrlV1 - case controller.ClientApiBinding: - return controller.ClientRestApiBaseUrlV1 - case controller.ManagementApiBinding: - return controller.ManagementRestApiBaseUrlV1 + return webapis.ClientRestApiBaseUrlV1 + case webapis.ClientApiBinding: + return webapis.ClientRestApiBaseUrlV1 + case webapis.ManagementApiBinding: + return webapis.ManagementRestApiBaseUrlV1 } return "" } diff --git a/controller/server/controller.go b/controller/server/controller.go index abdb5af2c..d88d5e965 100644 --- a/controller/server/controller.go +++ b/controller/server/controller.go @@ -23,7 +23,6 @@ import ( "github.com/openziti/storage/boltz" "github.com/openziti/ziti/common/pb/edge_ctrl_pb" runner2 "github.com/openziti/ziti/common/runner" - "github.com/openziti/ziti/controller/api_impl" edgeconfig "github.com/openziti/ziti/controller/config" "github.com/openziti/ziti/controller/env" "github.com/openziti/ziti/controller/handler_edge_ctrl" @@ -52,8 +51,6 @@ const ( policyMaxFreq = 1 * time.Hour policyAppWanFreq = 1 * time.Second policySessionFreq = 5 * time.Second - - ZitiInstanceId = "ziti-instance-id" ) func NewController(host env.HostController) (*Controller, error) { @@ -112,8 +109,6 @@ func NewController(host env.HostController) (*Controller, error) { return nil })) - api_impl.OverrideRequestWrapper(&fabricWrapper{ae: c.AppEnv}) - return c, nil } @@ -247,22 +242,6 @@ func (c *Controller) Run() { go c.checkEdgeInitialized() - managementApiFactory := NewManagementApiFactory(c.AppEnv) - clientApiFactory := NewClientApiFactory(c.AppEnv) - oidcApiFactory := NewOidcApiFactory(c.AppEnv) - - if err := c.AppEnv.HostController.GetXWebInstance().GetRegistry().Add(managementApiFactory); err != nil { - pfxlog.Logger().Fatalf("failed to create Edge Management API factory: %v", err) - } - - if err := c.AppEnv.HostController.GetXWebInstance().GetRegistry().Add(clientApiFactory); err != nil { - pfxlog.Logger().Fatalf("failed to create Edge Client API factory: %v", err) - } - - if err := c.AppEnv.HostController.GetXWebInstance().GetRegistry().Add(oidcApiFactory); err != nil { - pfxlog.Logger().Fatalf("failed to create OIDC API factory: %v", err) - } - if err := c.policyEngine.Start(c.AppEnv.HostController.GetCloseNotifyChannel()); err != nil { log.WithError(err).Fatalf("error starting policy engine") } diff --git a/controller/server/client-api.go b/controller/webapis/client-api.go similarity index 84% rename from controller/server/client-api.go rename to controller/webapis/client-api.go index 4288b7ad7..b8333fd45 100644 --- a/controller/server/client-api.go +++ b/controller/webapis/client-api.go @@ -14,7 +14,7 @@ limitations under the License. */ -package server +package webapis import ( "fmt" @@ -22,7 +22,6 @@ import ( "github.com/openziti/edge-api/rest_client_api_server" "github.com/openziti/edge-api/rest_management_api_server" "github.com/openziti/xweb/v2" - "github.com/openziti/ziti/controller" "github.com/openziti/ziti/controller/api" "github.com/openziti/ziti/controller/apierror" "github.com/openziti/ziti/controller/env" @@ -34,6 +33,8 @@ import ( "time" ) +const ZitiInstanceId = "ziti-instance-id" + var _ xweb.ApiHandlerFactory = &ClientApiFactory{} type ClientApiFactory struct { @@ -47,7 +48,7 @@ func (factory ClientApiFactory) Validate(config *xweb.InstanceConfig) error { for _, webListener := range config.ServerConfigs { for _, api := range webListener.APIs { - if webListener.Identity != nil && (api.Binding() == controller.ClientApiBinding || api.Binding() == controller.ManagementApiBinding) { + if webListener.Identity != nil && (api.Binding() == ClientApiBinding || api.Binding() == ManagementApiBinding) { caBytes, err := os.ReadFile(webListener.Identity.GetConfig().CA) if err != nil { @@ -57,7 +58,7 @@ func (factory ClientApiFactory) Validate(config *xweb.InstanceConfig) error { edgeConfig.AddCaPems(caBytes) } - if !clientApiFound && api.Binding() == controller.ClientApiBinding { + if !clientApiFound && api.Binding() == ClientApiBinding { for _, bindPoint := range webListener.BindPoints { if bindPoint.Address == edgeConfig.Api.Address { factory.appEnv.SetServerCert(webListener.Identity.ServerCert()[0]) @@ -72,7 +73,7 @@ func (factory ClientApiFactory) Validate(config *xweb.InstanceConfig) error { edgeConfig.RefreshCas() if !clientApiFound { - return errors.Errorf("could not find [edge.api.address] value [%s] as a bind point any instance of ApiConfig [%s]", edgeConfig.Api.Address, controller.ClientApiBinding) + return errors.Errorf("could not find [edge.api.address] value [%s] as a bind point any instance of ApiConfig [%s]", edgeConfig.Api.Address, ClientApiBinding) } return nil @@ -85,7 +86,7 @@ func NewClientApiFactory(appEnv *env.AppEnv) *ClientApiFactory { } func (factory ClientApiFactory) Binding() string { - return controller.ClientApiBinding + return ClientApiBinding } func (factory ClientApiFactory) New(_ *xweb.ServerConfig, options map[interface{}]interface{}) (xweb.ApiHandler, error) { @@ -111,7 +112,7 @@ type ClientApiHandler struct { } func (clientApi ClientApiHandler) Binding() string { - return controller.ClientApiBinding + return ClientApiBinding } func (clientApi ClientApiHandler) Options() map[interface{}]interface{} { @@ -154,28 +155,28 @@ func (clientApi ClientApiHandler) newHandler(ae *env.AppEnv) http.Handler { //if not /edge prefix and not /fabric, translate to "/edge/client/v", this is a hack //that should be removed once non-prefixed URLs are no longer used. //This will affect older go-lang enrolled SDKs and the C-SDK. - if !strings.HasPrefix(r.URL.Path, controller.RestApiRootPath) && !strings.HasPrefix(r.URL.Path, "/fabric") && !strings.HasPrefix(r.URL.Path, "/.well-known") { - r.URL.Path = controller.ClientRestApiBaseUrlLatest + r.URL.Path + if !strings.HasPrefix(r.URL.Path, RestApiRootPath) && !strings.HasPrefix(r.URL.Path, "/fabric") && !strings.HasPrefix(r.URL.Path, "/.well-known") { + r.URL.Path = ClientRestApiBaseUrlLatest + r.URL.Path } //translate /edge/v1 to /edge/client/v1 - r.URL.Path = strings.Replace(r.URL.Path, controller.LegacyClientRestApiBaseUrlV1, controller.ClientRestApiBaseUrlLatest, 1) + r.URL.Path = strings.Replace(r.URL.Path, LegacyClientRestApiBaseUrlV1, ClientRestApiBaseUrlLatest, 1) // .well-known/est/cacerts can be handled by the client API but the generated server requires // the prefixed path for route resolution. if r.URL.Path == WellKnownEstCaCerts { - r.URL.Path = controller.ClientRestApiBaseUrlLatest + WellKnownEstCaCerts + r.URL.Path = ClientRestApiBaseUrlLatest + WellKnownEstCaCerts } if r.URL.Path == VersionPath || r.URL.Path == RootPath { - r.URL.Path = controller.ClientRestApiBaseUrlLatest + VersionPath + r.URL.Path = ClientRestApiBaseUrlLatest + VersionPath } - if r.URL.Path == controller.ClientRestApiSpecUrl { + if r.URL.Path == ClientRestApiSpecUrl { //work around for: https://github.com/go-openapi/runtime/issues/226 if referer := r.Header.Get("Referer"); referer != "" { - if strings.Contains(referer, controller.ManagementRestApiBaseUrlLatest) { + if strings.Contains(referer, ManagementRestApiBaseUrlLatest) { rw.Header().Set("content-type", "application/json") rw.WriteHeader(http.StatusOK) _, _ = rw.Write(rest_management_api_server.SwaggerJSON) diff --git a/controller/server/fabric-api.go b/controller/webapis/fabric-api-wrapper.go similarity index 95% rename from controller/server/fabric-api.go rename to controller/webapis/fabric-api-wrapper.go index 340d84f24..6812d5239 100644 --- a/controller/server/fabric-api.go +++ b/controller/webapis/fabric-api-wrapper.go @@ -1,21 +1,25 @@ -package server +package webapis import ( "github.com/go-openapi/runtime" "github.com/go-openapi/runtime/middleware" "github.com/michaelquigley/pfxlog" "github.com/openziti/edge-api/rest_management_api_server" - "github.com/openziti/ziti/controller/env" - "github.com/openziti/ziti/controller/internal/permissions" - "github.com/openziti/ziti/controller/response" + "github.com/openziti/foundation/v2/errorz" "github.com/openziti/ziti/controller/api" "github.com/openziti/ziti/controller/api_impl" "github.com/openziti/ziti/controller/apierror" - "github.com/openziti/foundation/v2/errorz" + "github.com/openziti/ziti/controller/env" + "github.com/openziti/ziti/controller/internal/permissions" + "github.com/openziti/ziti/controller/response" "net/http" "time" ) +func NewFabricApiWrapper(ae *env.AppEnv) api_impl.RequestWrapper { + return &fabricWrapper{ae: ae} +} + type fabricWrapper struct { ae *env.AppEnv } diff --git a/controller/api_impl/management-api.go b/controller/webapis/fabric-management-api.go similarity index 69% rename from controller/api_impl/management-api.go rename to controller/webapis/fabric-management-api.go index 4e003f4a8..071929186 100644 --- a/controller/api_impl/management-api.go +++ b/controller/webapis/fabric-management-api.go @@ -14,7 +14,7 @@ limitations under the License. */ -package api_impl +package webapis import ( "crypto/x509" @@ -27,6 +27,7 @@ import ( "github.com/openziti/foundation/v2/concurrenz" "github.com/openziti/identity" "github.com/openziti/xweb/v2" + "github.com/openziti/ziti/controller/api_impl" "github.com/openziti/ziti/controller/handler_mgmt" "github.com/openziti/ziti/controller/network" "github.com/openziti/ziti/controller/rest_client" @@ -41,40 +42,40 @@ const ( ServerHeader = "server" ) -var _ xweb.ApiHandlerFactory = &ManagementApiFactory{} +var _ xweb.ApiHandlerFactory = &FabricManagementApiFactory{} -type ManagementApiFactory struct { - InitFunc func(managementApi *ManagementApiHandler) error +type FabricManagementApiFactory struct { + InitFunc func(managementApi *FabricManagementApiHandler) error network *network.Network nodeId identity.Identity xmgmts *concurrenz.CopyOnWriteSlice[xmgmt.Xmgmt] } -func (factory *ManagementApiFactory) Validate(_ *xweb.InstanceConfig) error { +func (factory *FabricManagementApiFactory) Validate(_ *xweb.InstanceConfig) error { return nil } -func NewManagementApiFactory(nodeId identity.Identity, network *network.Network, xmgmts *concurrenz.CopyOnWriteSlice[xmgmt.Xmgmt]) *ManagementApiFactory { +func NewFabricManagementApiFactory(nodeId identity.Identity, network *network.Network, xmgmts *concurrenz.CopyOnWriteSlice[xmgmt.Xmgmt]) *FabricManagementApiFactory { pfxlog.Logger().Infof("initializing management api factory with %d xmgmt instances", len(xmgmts.Value())) - return &ManagementApiFactory{ + return &FabricManagementApiFactory{ network: network, nodeId: nodeId, xmgmts: xmgmts, } } -func (factory *ManagementApiFactory) Binding() string { - return FabricApiBinding +func (factory *FabricManagementApiFactory) Binding() string { + return api_impl.FabricApiBinding } -func (factory *ManagementApiFactory) New(_ *xweb.ServerConfig, options map[interface{}]interface{}) (xweb.ApiHandler, error) { +func (factory *FabricManagementApiFactory) New(_ *xweb.ServerConfig, options map[interface{}]interface{}) (xweb.ApiHandler, error) { managementSpec, err := loads.Embedded(rest_server.SwaggerJSON, rest_server.FlatSwaggerJSON) if err != nil { pfxlog.Logger().Fatalln(err) } fabricAPI := operations.NewZitiFabricAPI(managementSpec) - fabricAPI.ServeError = ServeError + fabricAPI.ServeError = api_impl.ServeError if requestWrapper == nil { requestWrapper = &FabricRequestWrapper{ @@ -83,11 +84,11 @@ func (factory *ManagementApiFactory) New(_ *xweb.ServerConfig, options map[inter } } - for _, router := range routers { + for _, router := range api_impl.Routers { router.Register(fabricAPI, requestWrapper) } - managementApiHandler, err := NewManagementApiHandler(fabricAPI, options) + managementApiHandler, err := NewFabricManagementApiHandler(fabricAPI, options) if err != nil { return nil, err @@ -104,8 +105,8 @@ func (factory *ManagementApiFactory) New(_ *xweb.ServerConfig, options map[inter return managementApiHandler, nil } -func NewManagementApiHandler(fabricApi *operations.ZitiFabricAPI, options map[interface{}]interface{}) (*ManagementApiHandler, error) { - managementApi := &ManagementApiHandler{ +func NewFabricManagementApiHandler(fabricApi *operations.ZitiFabricAPI, options map[interface{}]interface{}) (*FabricManagementApiHandler, error) { + managementApi := &FabricManagementApiHandler{ fabricApi: fabricApi, options: options, } @@ -117,7 +118,7 @@ func NewManagementApiHandler(fabricApi *operations.ZitiFabricAPI, options map[in return managementApi, nil } -type ManagementApiHandler struct { +type FabricManagementApiHandler struct { fabricApi *operations.ZitiFabricAPI handler http.Handler wsHandler http.Handler @@ -126,23 +127,23 @@ type ManagementApiHandler struct { bindHandler channel.BindHandler } -func (managementApi *ManagementApiHandler) Binding() string { - return FabricApiBinding +func (managementApi *FabricManagementApiHandler) Binding() string { + return api_impl.FabricApiBinding } -func (managementApi *ManagementApiHandler) Options() map[interface{}]interface{} { +func (managementApi *FabricManagementApiHandler) Options() map[interface{}]interface{} { return managementApi.options } -func (managementApi *ManagementApiHandler) RootPath() string { +func (managementApi *FabricManagementApiHandler) RootPath() string { return rest_client.DefaultBasePath } -func (managementApi *ManagementApiHandler) IsHandler(r *http.Request) bool { +func (managementApi *FabricManagementApiHandler) IsHandler(r *http.Request) bool { return strings.HasPrefix(r.URL.Path, managementApi.RootPath()) } -func (managementApi *ManagementApiHandler) ServeHTTP(writer http.ResponseWriter, request *http.Request) { +func (managementApi *FabricManagementApiHandler) ServeHTTP(writer http.ResponseWriter, request *http.Request) { if request.URL.Path == managementApi.wsUrl { managementApi.wsHandler.ServeHTTP(writer, request) } else { @@ -150,12 +151,12 @@ func (managementApi *ManagementApiHandler) ServeHTTP(writer http.ResponseWriter, } } -func (managementApi *ManagementApiHandler) newHandler() http.Handler { +func (managementApi *FabricManagementApiHandler) newHandler() http.Handler { innerManagementHandler := managementApi.fabricApi.Serve(nil) return requestWrapper.WrapHttpHandler(innerManagementHandler) } -func (managementApi *ManagementApiHandler) handleWebSocket(writer http.ResponseWriter, request *http.Request) { +func (managementApi *FabricManagementApiHandler) handleWebSocket(writer http.ResponseWriter, request *http.Request) { log := pfxlog.Logger() log.Debug("handling mgmt channel websocket upgrade") upgrader := websocket.Upgrader{} diff --git a/controller/webapis/fabric-request-wrapper.go b/controller/webapis/fabric-request-wrapper.go new file mode 100644 index 000000000..dad530356 --- /dev/null +++ b/controller/webapis/fabric-request-wrapper.go @@ -0,0 +1,127 @@ +package webapis + +import ( + "crypto/x509" + "github.com/go-openapi/runtime" + openApiMiddleware "github.com/go-openapi/runtime/middleware" + "github.com/michaelquigley/pfxlog" + "github.com/openziti/foundation/v2/errorz" + "github.com/openziti/identity" + "github.com/openziti/ziti/common/build" + "github.com/openziti/ziti/controller/api" + "github.com/openziti/ziti/controller/api_impl" + "github.com/openziti/ziti/controller/apierror" + "github.com/openziti/ziti/controller/network" + "github.com/openziti/ziti/controller/rest_server" + "github.com/pkg/errors" + "net/http" + "time" +) + +var requestWrapper api_impl.RequestWrapper + +func OverrideRequestWrapper(rw api_impl.RequestWrapper) { + if requestWrapper != nil { + pfxlog.Logger().Warn("requestWrapper overridden more than once") + } + requestWrapper = rw +} + +type FabricRequestWrapper struct { + nodeId identity.Identity + network *network.Network +} + +func (self *FabricRequestWrapper) WrapRequest(handler api_impl.RequestHandler, request *http.Request, entityId, entitySubId string) openApiMiddleware.Responder { + return openApiMiddleware.ResponderFunc(func(writer http.ResponseWriter, producer runtime.Producer) { + rc, err := api.GetRequestContextFromHttpContext(request) + + if rc == nil { + rc = api_impl.NewRequestContext(writer, request) + } + + rc.SetProducer(producer) + rc.SetEntityId(entityId) + rc.SetEntitySubId(entitySubId) + + if err != nil { + pfxlog.Logger().WithError(err).Error("could not retrieve request context") + rc.RespondWithError(err) + return + } + + handler(self.network, rc) + }) +} + +func (self *FabricRequestWrapper) WrapHttpHandler(handler http.Handler) http.Handler { + wrapper := http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + if r.URL.Path == api_impl.FabricRestApiSpecUrl { + rw.Header().Set("content-type", "application/json") + rw.WriteHeader(http.StatusOK) + _, _ = rw.Write(rest_server.SwaggerJSON) + return + } + + rc := api_impl.NewRequestContext(rw, r) + + if err := self.verifyCert(r); err != nil { + rc.RespondWithError(apierror.NewInvalidAuth()) + return + } + + api.AddRequestContextToHttpContext(r, rc) + + //after request context is filled so that api session is present for session expiration headers + buildInfo := build.GetBuildInfo() + if buildInfo != nil { + rc.GetResponseWriter().Header().Set(ServerHeader, "ziti-controller/"+buildInfo.Version()) + } + + handler.ServeHTTP(rw, r) + }) + + return api.TimeoutHandler(api.WrapCorsHandler(wrapper), 10*time.Second, apierror.NewTimeoutError(), api_impl.FabricResponseMapper{}) +} + +func (self *FabricRequestWrapper) WrapWsHandler(handler http.Handler) http.Handler { + wrapper := http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + if err := self.verifyCert(r); err != nil { + rc := api_impl.NewRequestContext(rw, r) + rc.RespondWithError(apierror.NewInvalidAuth()) + return + } + + handler.ServeHTTP(rw, r) + }) + + return wrapper +} + +func (self *FabricRequestWrapper) verifyCert(r *http.Request) error { + certificates := r.TLS.PeerCertificates + if len(certificates) == 0 { + return errors.New("no certificates provided, unable to verify dialer") + } + + config := self.nodeId.ServerTLSConfig() + + opts := x509.VerifyOptions{ + Roots: config.RootCAs, + Intermediates: x509.NewCertPool(), + KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageAny}, + } + + var errorList errorz.MultipleErrors + + for _, cert := range certificates { + if _, err := cert.Verify(opts); err == nil { + return nil + } else { + errorList = append(errorList, err) + } + } + + //goland:noinspection GoNilness + return errorList.ToError() +} diff --git a/controller/server/management-api.go b/controller/webapis/management-api.go similarity index 92% rename from controller/server/management-api.go rename to controller/webapis/management-api.go index 21dffe023..2ef2b69a3 100644 --- a/controller/server/management-api.go +++ b/controller/webapis/management-api.go @@ -14,14 +14,13 @@ limitations under the License. */ -package server +package webapis import ( "fmt" "github.com/openziti/edge-api/rest_management_api_client" "github.com/openziti/edge-api/rest_management_api_server" "github.com/openziti/xweb/v2" - "github.com/openziti/ziti/controller" "github.com/openziti/ziti/controller/api" "github.com/openziti/ziti/controller/apierror" "github.com/openziti/ziti/controller/env" @@ -55,7 +54,7 @@ func NewManagementApiFactory(appEnv *env.AppEnv) *ManagementApiFactory { } func (factory ManagementApiFactory) Binding() string { - return controller.ManagementApiBinding + return ManagementApiBinding } func (factory ManagementApiFactory) New(_ *xweb.ServerConfig, options map[interface{}]interface{}) (xweb.ApiHandler, error) { @@ -81,7 +80,7 @@ type ManagementApiHandler struct { } func (managementApi ManagementApiHandler) Binding() string { - return controller.ManagementApiBinding + return ManagementApiBinding } func (managementApi ManagementApiHandler) Options() map[interface{}]interface{} { @@ -110,13 +109,14 @@ func NewManagementApiHandler(ae *env.AppEnv, options map[interface{}]interface{} return managementApi, nil } + func (managementApi ManagementApiHandler) newHandler(ae *env.AppEnv) http.Handler { innerManagementHandler := ae.ManagementApi.Serve(nil) handler := http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { rw.Header().Set(ZitiInstanceId, ae.InstanceId) - if r.URL.Path == controller.ManagementRestApiSpecUrl { + if r.URL.Path == ManagementRestApiSpecUrl { rw.Header().Set("content-type", "application/json") rw.WriteHeader(http.StatusOK) _, _ = rw.Write(rest_management_api_server.SwaggerJSON) @@ -126,11 +126,11 @@ func (managementApi ManagementApiHandler) newHandler(ae *env.AppEnv) http.Handle // .well-known/est/cacerts can be handled by the management API but the generated server requires // the prefixed path for route resolution if r.URL.Path == WellKnownEstCaCerts { - r.URL.Path = controller.ManagementRestApiBaseUrlLatest + WellKnownEstCaCerts + r.URL.Path = ManagementRestApiBaseUrlLatest + WellKnownEstCaCerts } if r.URL.Path == VersionPath || r.URL.Path == RootPath { - r.URL.Path = controller.ManagementRestApiBaseUrlLatest + VersionPath + r.URL.Path = ManagementRestApiBaseUrlLatest + VersionPath } rc := ae.CreateRequestContext(rw, r) diff --git a/controller/api_impl/metrics-api.go b/controller/webapis/metrics-api.go similarity index 94% rename from controller/api_impl/metrics-api.go rename to controller/webapis/metrics-api.go index 4bd9d2d70..887be15df 100644 --- a/controller/api_impl/metrics-api.go +++ b/controller/webapis/metrics-api.go @@ -14,7 +14,7 @@ limitations under the License. */ -package api_impl +package webapis import ( "bytes" @@ -25,6 +25,7 @@ import ( "github.com/michaelquigley/pfxlog" "github.com/openziti/identity" "github.com/openziti/xweb/v2" + "github.com/openziti/ziti/controller/api_impl" "github.com/openziti/ziti/controller/network" "net/http" "os" @@ -50,7 +51,7 @@ func NewMetricsApiFactory(nodeId identity.Identity, network *network.Network) *M } func (factory *MetricsApiFactory) Binding() string { - return MetricApiBinding + return api_impl.MetricApiBinding } func (factory *MetricsApiFactory) New(_ *xweb.ServerConfig, options map[interface{}]interface{}) (xweb.ApiHandler, error) { @@ -104,7 +105,7 @@ func NewMetricsApiHandler(n *network.Network, options map[interface{}]interface{ } } - metricsApi.modelMapper = NewMetricsModelMapper(n, "prometheus", includeTimestamps) + metricsApi.modelMapper = api_impl.NewMetricsModelMapper(n, "prometheus", includeTimestamps) metricsApi.handler = metricsApi.newHandler() return metricsApi, nil @@ -115,12 +116,12 @@ type MetricsApiHandler struct { handler http.Handler network *network.Network scrapeCert *x509.Certificate - modelMapper MetricsModelMapper + modelMapper api_impl.MetricsModelMapper options map[interface{}]interface{} } func (metricsApi *MetricsApiHandler) Binding() string { - return MetricApiBinding + return api_impl.MetricApiBinding } func (metricsApi *MetricsApiHandler) Options() map[interface{}]interface{} { diff --git a/controller/server/oidc-api.go b/controller/webapis/oidc-api.go similarity index 97% rename from controller/server/oidc-api.go rename to controller/webapis/oidc-api.go index af4ea91f5..bab0cc1ed 100644 --- a/controller/server/oidc-api.go +++ b/controller/webapis/oidc-api.go @@ -14,7 +14,7 @@ limitations under the License. */ -package server +package webapis import ( "context" @@ -25,7 +25,6 @@ import ( "strings" "github.com/openziti/xweb/v2" - "github.com/openziti/ziti/controller" "github.com/openziti/ziti/controller/api" "github.com/openziti/ziti/controller/env" "github.com/openziti/ziti/controller/oidc_auth" @@ -49,7 +48,7 @@ func NewOidcApiFactory(appEnv *env.AppEnv) *OidcApiFactory { } func (factory OidcApiFactory) Binding() string { - return controller.OidcApiBinding + return OidcApiBinding } func (factory OidcApiFactory) New(serverConfig *xweb.ServerConfig, options map[interface{}]interface{}) (xweb.ApiHandler, error) { @@ -75,7 +74,7 @@ type OidcApiHandler struct { } func (h OidcApiHandler) Binding() string { - return controller.OidcApiBinding + return OidcApiBinding } func (h OidcApiHandler) Options() map[interface{}]interface{} { diff --git a/controller/versions.go b/controller/webapis/versions.go similarity index 98% rename from controller/versions.go rename to controller/webapis/versions.go index 34bde93f1..f3a50e0aa 100644 --- a/controller/versions.go +++ b/controller/webapis/versions.go @@ -14,7 +14,7 @@ limitations under the License. */ -package controller +package webapis const ( VersionV1 = "v1" diff --git a/tests/fabric_context.go b/tests/fabric_context.go index c45810456..e4aed0ae8 100644 --- a/tests/fabric_context.go +++ b/tests/fabric_context.go @@ -27,12 +27,12 @@ import ( "github.com/openziti/foundation/v2/versions" id "github.com/openziti/identity" "github.com/openziti/identity/certtools" - "github.com/openziti/ziti/controller/api_impl" "github.com/openziti/ziti/controller/config" "github.com/openziti/ziti/controller/rest_client" restClientRouter "github.com/openziti/ziti/controller/rest_client/router" "github.com/openziti/ziti/controller/rest_model" "github.com/openziti/ziti/controller/rest_util" + "github.com/openziti/ziti/controller/webapis" "github.com/openziti/ziti/router" "net" "net/http" @@ -154,7 +154,7 @@ func (ctx *FabricTestContext) StartServer() { } func (ctx *FabricTestContext) StartServerFor(test string, clean bool) { - api_impl.OverrideRequestWrapper(nil) // clear possible wrapper from another test + webapis.OverrideRequestWrapper(nil) // clear possible wrapper from another test if ctx.LogLevel != "" { if level, err := logrus.ParseLevel(ctx.LogLevel); err == nil { logrus.StandardLogger().SetLevel(level) diff --git a/tests/identity_api_session_activity_test.go b/tests/identity_api_session_activity_test.go index 21c25df59..c7dd2338b 100644 --- a/tests/identity_api_session_activity_test.go +++ b/tests/identity_api_session_activity_test.go @@ -5,7 +5,7 @@ import ( "github.com/openziti/edge-api/rest_model" edge_apis "github.com/openziti/sdk-golang/edge-apis" "github.com/openziti/sdk-golang/ziti" - "github.com/openziti/ziti/controller" + "github.com/openziti/ziti/controller/webapis" "net/url" "testing" "time" @@ -33,7 +33,7 @@ func Test_Identity_HasErConnection(t *testing.T) { isRunning = false ctx.Req.NoError(listener.Close()) }() - managementStr := "https://" + ctx.ApiHost + controller.ManagementRestApiBaseUrlV1 + managementStr := "https://" + ctx.ApiHost + webapis.ManagementRestApiBaseUrlV1 managementUrl, err := url.Parse(managementStr) ctx.Req.NoError(err) diff --git a/tests/link_test.go b/tests/link_test.go index fecbc96a4..cdee86e85 100644 --- a/tests/link_test.go +++ b/tests/link_test.go @@ -79,6 +79,10 @@ func (self *testRegistryEnv) GetRateLimiterPool() goroutines.Pool { panic("implement me") } +func (self *testRegistryEnv) GetMetricsRegistry() metrics.UsageRegistry { + panic("implement me") +} + type testDial struct { Key string LinkId string