Merge pull request #603 from openziti/host.v1-support

Host.v1 and Host.v2 support

controller/handler_edge_ctrl/common.go

@@ -71,6 +71,22 @@ func (self *baseRequestHandler) returnError(ctx requestContext, err error) {
 	}
 }
 
+func (self *baseRequestHandler) logResult(ctx requestContext, err error) {
+	logger := logrus.
+		WithField("router", ctx.GetHandler().getChannel().Id().Token).
+		WithField("operation", ctx.GetHandler().Label())
+
+	if sessionCtx, ok := ctx.(sessionRequestContext); ok {
+		logger = logger.WithField("token", sessionCtx.GetSessionToken())
+	}
+
+	if err != nil {
+		logger.WithError(err).Error("operation failed")
+	} else {
+		logger.Debug("operation success")
+	}
+}
+
 type requestContext interface {
 	GetHandler() requestHandler
 	GetMessage() *channel2.Message

controller/handler_edge_ctrl/common_tunnel.go

@@ -191,8 +191,8 @@ func (self *baseTunnelRequestContext) getCreateApiSessionResponse() *edge_ctrl_p
 
 func (self *baseTunnelRequestContext) getCreateSessionResponse() *edge_ctrl_pb.CreateSessionResponse {
 	return &edge_ctrl_pb.CreateSessionResponse{
-		SessionId: self.apiSession.Id,
-		Token:     self.apiSession.Token,
+		SessionId: self.session.Id,
+		Token:     self.session.Token,
 	}
 }
 

controller/handler_edge_ctrl/create_tunnel_terminator.go

@@ -121,7 +121,7 @@ func (self *createTunnelTerminatorHandler) CreateTerminator(ctx *CreateTunnelTer
 
 	response := &edge_ctrl_pb.CreateTunnelTerminatorResponse{
 		Session:      ctx.getCreateSessionResponse(),
-		TerminatorId: "",
+		TerminatorId: id,
 	}
 
 	if newApiSession {

controller/handler_edge_ctrl/health_event.go

@@ -0,0 +1,98 @@
+/*
+	Copyright NetFoundry, Inc.
+
+	Licensed under the Apache License, Version 2.0 (the "License");
+	you may not use this file except in compliance with the License.
+	You may obtain a copy of the License at
+
+	https://www.apache.org/licenses/LICENSE-2.0
+
+	Unless required by applicable law or agreed to in writing, software
+	distributed under the License is distributed on an "AS IS" BASIS,
+	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+	See the License for the specific language governing permissions and
+	limitations under the License.
+*/
+
+package handler_edge_ctrl
+
+import (
+	"github.com/golang/protobuf/proto"
+	"github.com/michaelquigley/pfxlog"
+	"github.com/openziti/edge/controller/env"
+	"github.com/openziti/edge/controller/persistence"
+	"github.com/openziti/edge/pb/edge_ctrl_pb"
+	"github.com/openziti/foundation/channel2"
+	"github.com/openziti/foundation/metrics"
+	"time"
+)
+
+type healthEventHandler struct {
+	baseRequestHandler
+	serviceHealthCheckPassedCounter metrics.IntervalCounter
+	serviceHealthCheckFailedCounter metrics.IntervalCounter
+}
+
+func NewHealthEventHandler(appEnv *env.AppEnv, ch channel2.Channel) channel2.ReceiveHandler {
+	serviceEventMetrics := appEnv.GetHostController().GetNetwork().GetServiceEventsMetricsRegistry()
+	return &healthEventHandler{
+		baseRequestHandler: baseRequestHandler{
+			ch:     ch,
+			appEnv: appEnv,
+		},
+		serviceHealthCheckPassedCounter: serviceEventMetrics.IntervalCounter("service.health_check.passed", time.Minute),
+		serviceHealthCheckFailedCounter: serviceEventMetrics.IntervalCounter("service.health_check.failed", time.Minute),
+	}
+}
+
+func (self *healthEventHandler) ContentType() int32 {
+	return int32(edge_ctrl_pb.ContentType_HealthEventType)
+}
+
+func (self *healthEventHandler) Label() string {
+	return "health.event"
+}
+
+func (self *healthEventHandler) HandleReceive(msg *channel2.Message, ch channel2.Channel) {
+	req := &edge_ctrl_pb.HealthEventRequest{}
+	if err := proto.Unmarshal(msg.Body, req); err != nil {
+		pfxlog.ContextLogger(ch.Label()).WithError(err).Error("could not unmarshal health event")
+		return
+	}
+
+	ctx := &HealthEventRequestContext{
+		baseSessionRequestContext: baseSessionRequestContext{handler: self, msg: msg},
+		req:                       req,
+	}
+
+	go self.handleHealthEvent(ctx)
+}
+
+func (self *healthEventHandler) handleHealthEvent(ctx *HealthEventRequestContext) {
+	if !ctx.loadRouter() {
+		return
+	}
+
+	ctx.loadSession(ctx.req.SessionToken)
+	ctx.checkSessionType(persistence.SessionTypeBind)
+	ctx.checkSessionFingerprints(ctx.req.Fingerprints)
+
+	if ctx.err == nil {
+		if ctx.req.CheckPassed {
+			self.serviceHealthCheckPassedCounter.Update(ctx.session.ServiceId, time.Now(), 1)
+		} else {
+			self.serviceHealthCheckFailedCounter.Update(ctx.session.ServiceId, time.Now(), 1)
+		}
+	}
+
+	self.logResult(ctx, ctx.err)
+}
+
+type HealthEventRequestContext struct {
+	baseSessionRequestContext
+	req *edge_ctrl_pb.HealthEventRequest
+}
+
+func (self *HealthEventRequestContext) GetSessionToken() string {
+	return self.req.SessionToken
+}

controller/handler_edge_ctrl/tunnel_health_event.go

@@ -0,0 +1,88 @@
+/*
+	Copyright NetFoundry, Inc.
+
+	Licensed under the Apache License, Version 2.0 (the "License");
+	you may not use this file except in compliance with the License.
+	You may obtain a copy of the License at
+
+	https://www.apache.org/licenses/LICENSE-2.0
+
+	Unless required by applicable law or agreed to in writing, software
+	distributed under the License is distributed on an "AS IS" BASIS,
+	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+	See the License for the specific language governing permissions and
+	limitations under the License.
+*/
+
+package handler_edge_ctrl
+
+import (
+	"github.com/openziti/edge/controller/env"
+	"github.com/openziti/edge/edge_common"
+	"github.com/openziti/edge/pb/edge_ctrl_pb"
+	"github.com/openziti/foundation/channel2"
+	"github.com/openziti/foundation/metrics"
+	"time"
+)
+
+type tunnelHealthEventHandler struct {
+	baseRequestHandler
+	serviceHealthCheckPassedCounter metrics.IntervalCounter
+	serviceHealthCheckFailedCounter metrics.IntervalCounter
+}
+
+func NewTunnelHealthEventHandler(appEnv *env.AppEnv, ch channel2.Channel) channel2.ReceiveHandler {
+	serviceEventMetrics := appEnv.GetHostController().GetNetwork().GetServiceEventsMetricsRegistry()
+	return &tunnelHealthEventHandler{
+		baseRequestHandler: baseRequestHandler{
+			ch:     ch,
+			appEnv: appEnv,
+		},
+		serviceHealthCheckPassedCounter: serviceEventMetrics.IntervalCounter("service.health_check.passed", time.Minute),
+		serviceHealthCheckFailedCounter: serviceEventMetrics.IntervalCounter("service.health_check.failed", time.Minute),
+	}
+}
+
+func (self *tunnelHealthEventHandler) ContentType() int32 {
+	return int32(edge_ctrl_pb.ContentType_TunnelHealthEventType)
+}
+
+func (self *tunnelHealthEventHandler) Label() string {
+	return "tunnel.health.event"
+}
+
+func (self *tunnelHealthEventHandler) HandleReceive(msg *channel2.Message, _ channel2.Channel) {
+	terminatorId, _ := msg.GetStringHeader(int32(edge_ctrl_pb.Header_TerminatorId))
+	checkPassed, _ := msg.GetBoolHeader(int32(edge_ctrl_pb.Header_CheckPassed))
+
+	ctx := &TunnelHealthEventRequestContext{
+		baseSessionRequestContext: baseSessionRequestContext{handler: self, msg: msg},
+		terminatorId:              terminatorId,
+		checkPassed:               checkPassed,
+	}
+
+	go self.handleHealthEvent(ctx)
+}
+
+func (self *tunnelHealthEventHandler) handleHealthEvent(ctx *TunnelHealthEventRequestContext) {
+	if !ctx.loadRouter() {
+		return
+	}
+
+	terminator := ctx.verifyTerminator(ctx.terminatorId, edge_common.TunnelBinding)
+	if terminator != nil && ctx.err == nil {
+		if ctx.checkPassed {
+			self.serviceHealthCheckPassedCounter.Update(terminator.Service, time.Now(), 1)
+		} else {
+			self.serviceHealthCheckFailedCounter.Update(terminator.Service, time.Now(), 1)
+		}
+	}
+
+	self.logResult(ctx, ctx.err)
+}
+
+type TunnelHealthEventRequestContext struct {
+	baseSessionRequestContext
+	terminatorId string
+	checkPassed  bool
+}

controller/model/api_session_certificate_handlers.go

@@ -119,17 +119,17 @@ func (handler *ApiSessionCertificateHandler) Delete(id string) error {
 	return handler.deleteEntity(id)
 }
 
-func (handler *ApiSessionCertificateHandler) Query(query string) (*ApiSessionCertificateListResult, error) {
+func (handler *ApiSessionCertificateHandler) Query(tx *bbolt.Tx, query string) (*ApiSessionCertificateListResult, error) {
 	result := &ApiSessionCertificateListResult{handler: handler}
-	err := handler.list(query, result.collect)
+	err := handler.ListWithTx(tx, query, result.collect)
 	if err != nil {
 		return nil, err
 	}
 	return result, nil
 }
 
-func (handler *ApiSessionCertificateHandler) ReadByApiSessionId(apiSessionId string) ([]*ApiSessionCertificate, error) {
-	result, err := handler.Query(fmt.Sprintf(`apiSession = "%s"`, apiSessionId))
+func (handler *ApiSessionCertificateHandler) ReadByApiSessionId(tx *bbolt.Tx, apiSessionId string) ([]*ApiSessionCertificate, error) {
+	result, err := handler.Query(tx, fmt.Sprintf(`apiSession = "%s"`, apiSessionId))
 
 	if err != nil {
 		return nil, err

controller/model/api_session_handlers.go

@@ -138,7 +138,7 @@ func (handler *ApiSessionHandler) heartbeatFlush(beats []*Heartbeat) {
 		for _, beat := range beats {
 			err := store.Update(mutCtx, &persistence.ApiSession{
 				BaseExtEntity: boltz.BaseExtEntity{
-					Id:        beat.ApiSessionId,
+					Id: beat.ApiSessionId,
 				},
 				LastActivityAt: beat.LastActivityAt,
 			}, persistence.UpdateLastActivityAtChecker{})
@@ -204,21 +204,22 @@ func (handler *ApiSessionHandler) Query(query string) (*ApiSessionListResult, er
 }
 
 func (handler *ApiSessionHandler) VisitFingerprintsForApiSessionId(apiSessionId string, visitor func(fingerprint string) bool) error {
-	apiSession, err := handler.Read(apiSessionId)
+	return handler.GetDb().View(func(tx *bbolt.Tx) error {
+		apiSession, err := handler.readInTx(tx, apiSessionId)
+		if err != nil {
+			return errors.Wrapf(err, "could not query fingerprints by api session id [%s]", apiSessionId)
+		}
 
-	if err != nil {
-		return errors.Wrapf(err, "could not query fingerprints by api session id [%s]", apiSessionId)
-	}
-
-	return handler.VisitFingerprintsForApiSession(apiSession.IdentityId, apiSessionId, visitor)
+		return handler.VisitFingerprintsForApiSession(tx, apiSession.IdentityId, apiSessionId, visitor)
+	})
 }
 
-func (handler *ApiSessionHandler) VisitFingerprintsForApiSession(identityId, apiSessionId string, visitor func(fingerprint string) bool) error {
-	if stopVisiting, err := handler.env.GetHandlers().Identity.VisitIdentityAuthenticatorFingerprints(identityId, visitor); stopVisiting || err != nil {
+func (handler *ApiSessionHandler) VisitFingerprintsForApiSession(tx *bbolt.Tx, identityId, apiSessionId string, visitor func(fingerprint string) bool) error {
+	if stopVisiting, err := handler.env.GetHandlers().Identity.VisitIdentityAuthenticatorFingerprints(tx, identityId, visitor); stopVisiting || err != nil {
 		return err
 	}
 
-	apiSessionCerts, err := handler.env.GetHandlers().ApiSessionCertificate.ReadByApiSessionId(apiSessionId)
+	apiSessionCerts, err := handler.env.GetHandlers().ApiSessionCertificate.ReadByApiSessionId(tx, apiSessionId)
 	if err != nil {
 		return err
 	}

controller/model/identity_handlers.go

@@ -318,24 +318,23 @@ func (handler *IdentityHandler) CollectAuthenticators(id string, collector func(
 	})
 }
 
-func (handler *IdentityHandler) visitAuthenticators(id string, visitor func(entity *Authenticator) bool) error {
-	return handler.GetDb().View(func(tx *bbolt.Tx) error {
-		_, err := handler.readInTx(tx, id)
-		if err != nil {
+func (handler *IdentityHandler) visitAuthenticators(tx *bbolt.Tx, id string, visitor func(entity *Authenticator) bool) error {
+	_, err := handler.readInTx(tx, id)
+	if err != nil {
+		return err
+	}
+	authenticatorIds := handler.GetStore().GetRelatedEntitiesIdList(tx, id, persistence.FieldIdentityAuthenticators)
+	for _, authenticatorId := range authenticatorIds {
+		authenticator := &Authenticator{}
+		if err := handler.env.GetHandlers().Authenticator.readEntityInTx(tx, authenticatorId, authenticator); err != nil {
 			return err
 		}
-		authenticatorIds := handler.GetStore().GetRelatedEntitiesIdList(tx, id, persistence.FieldIdentityAuthenticators)
-		for _, authenticatorId := range authenticatorIds {
-			authenticator := &Authenticator{}
-			if err := handler.env.GetHandlers().Authenticator.readEntityInTx(tx, authenticatorId, authenticator); err != nil {
-				return err
-			}
-			if visitor(authenticator) {
-				return nil
-			}
+		if visitor(authenticator) {
+			return nil
 		}
-		return nil
-	})
+	}
+	return nil
+
 }
 
 func (handler *IdentityHandler) CollectEnrollments(id string, collector func(entity *Enrollment) error) error {
@@ -496,9 +495,9 @@ func (handler *IdentityHandler) IsActive(id string) bool {
 	return handler.identityStatusMap.IsActive(id)
 }
 
-func (handler *IdentityHandler) VisitIdentityAuthenticatorFingerprints(identityId string, visitor func(string) bool) (bool, error) {
+func (handler *IdentityHandler) VisitIdentityAuthenticatorFingerprints(tx *bbolt.Tx, identityId string, visitor func(string) bool) (bool, error) {
 	stopVisit := false
-	err := handler.visitAuthenticators(identityId, func(authenticator *Authenticator) bool {
+	err := handler.visitAuthenticators(tx, identityId, func(authenticator *Authenticator) bool {
 		for _, authPrint := range authenticator.Fingerprints() {
 			if visitor(authPrint) {
 				stopVisit = true

controller/persistence/migration_initialize.go

@@ -39,6 +39,7 @@ func (m *Migrations) initialize(step *boltz.MigrationStep) int {
 	m.addPostureCheckTypes(step)
 	m.createInterceptV1ConfigType(step)
 	m.createHostV1ConfigType(step)
+	step.SetError(m.stores.ConfigType.Create(step.Ctx, hostV2ConfigType))
 
 	return CurrentDbVersion
 }
@@ -125,6 +126,7 @@ var serverConfigTypeV1 = &ConfigType{
 						"enum": []interface{}{
 							"fail",
 							"pass",
+							"change",
 						},
 					},
 					"consecutiveEvents": map[string]interface{}{
@@ -137,7 +139,7 @@ var serverConfigTypeV1 = &ConfigType{
 					},
 					"action": map[string]interface{}{
 						"type":    "string",
-						"pattern": "(mark (un)?healthy|increase cost [0-9]+|decrease cost [0-9]+)",
+						"pattern": "(mark (un)?healthy|increase cost [0-9]+|decrease cost [0-9]+|send event)",
 					},
 				},
 			},
@@ -233,6 +235,474 @@ var serverConfigTypeV1 = &ConfigType{
 	},
 }
 
+var hostV1ConfigTypeId = "NH5p4FpGR"
+var hostV1ConfigType = &ConfigType{
+	BaseExtEntity: boltz.BaseExtEntity{Id: hostV1ConfigTypeId},
+	Name:          "host.v1",
+	Schema: map[string]interface{}{
+		"$id": "http://ziti-edge.netfoundry.io/schemas/host-v1.schema.json",
+		"definitions": map[string]interface{}{
+			"duration": map[string]interface{}{
+				"type":    "string",
+				"pattern": "[0-9]+(h|m|s|ms)",
+			},
+			"method": map[string]interface{}{
+				"type": "string",
+				"enum": []interface{}{
+					"GET",
+					"POST",
+					"PUT",
+					"PATCH",
+				},
+			},
+			"action": map[string]interface{}{
+				"type":                 "object",
+				"additionalProperties": false,
+				"required": []interface{}{
+					"trigger",
+					"action",
+				},
+				"properties": map[string]interface{}{
+					"trigger": map[string]interface{}{
+						"type": "string",
+						"enum": []interface{}{
+							"fail",
+							"pass",
+							"change",
+						},
+					},
+					"consecutiveEvents": map[string]interface{}{
+						"type":    "integer",
+						"minimum": float64(0),
+						"maximum": float64(math.MaxUint16),
+					},
+					"duration": map[string]interface{}{
+						"$ref": "#/definitions/duration",
+					},
+					"action": map[string]interface{}{
+						"type":    "string",
+						"pattern": "(mark (un)?healthy|increase cost [0-9]+|decrease cost [0-9]+|send event)",
+					},
+				},
+			},
+			"actionList": map[string]interface{}{
+				"type": "array",
+				"items": map[string]interface{}{
+					"$ref": "#/definitions/action",
+				},
+				"minItems": 1,
+				"maxItems": 20,
+			},
+			"portCheck": map[string]interface{}{
+				"type":                 "object",
+				"additionalProperties": false,
+				"required": []interface{}{
+					"interval",
+					"timeout",
+					"address",
+				},
+				"properties": map[string]interface{}{
+					"interval": map[string]interface{}{"$ref": "#/definitions/duration"},
+					"timeout":  map[string]interface{}{"$ref": "#/definitions/duration"},
+					"address":  map[string]interface{}{"type": "string"},
+					"actions":  map[string]interface{}{"$ref": "#/definitions/actionList"},
+				},
+			},
+			"httpCheck": map[string]interface{}{
+				"type":                 "object",
+				"additionalProperties": false,
+				"required": []interface{}{
+					"interval",
+					"timeout",
+					"url",
+				},
+				"properties": map[string]interface{}{
+					"url":      map[string]interface{}{"type": "string"},
+					"method":   map[string]interface{}{"$ref": "#/definitions/method"},
+					"body":     map[string]interface{}{"type": "string"},
+					"interval": map[string]interface{}{"$ref": "#/definitions/duration"},
+					"timeout":  map[string]interface{}{"$ref": "#/definitions/duration"},
+					"actions":  map[string]interface{}{"$ref": "#/definitions/actionList"},
+					"expectStatus": map[string]interface{}{
+						"type":    "integer",
+						"minimum": float64(100),
+						"maximum": float64(599),
+					},
+					"expectInBody": map[string]interface{}{"type": "string"},
+				},
+			},
+			"portCheckList": map[string]interface{}{
+				"type": "array",
+				"items": map[string]interface{}{
+					"$ref": "#/definitions/portCheck",
+				},
+			},
+			"httpCheckList": map[string]interface{}{
+				"type": "array",
+				"items": map[string]interface{}{
+					"$ref": "#/definitions/httpCheck",
+				},
+			},
+			"ipAddressFormat": map[string]interface{}{
+				"oneOf": []interface{}{
+					map[string]interface{}{"format": "ipv4"},
+					map[string]interface{}{"format": "ipv6"},
+				},
+			},
+			"ipAddress": map[string]interface{}{
+				"type": "string",
+				"$ref": "#/definitions/ipAddressFormat",
+			},
+			"hostname": map[string]interface{}{
+				"type":   "string",
+				"format": "hostname",
+				"not":    map[string]interface{}{"$ref": "#/definitions/ipAddressFormat"},
+			},
+			"address": map[string]interface{}{
+				"oneOf": []interface{}{
+					map[string]interface{}{"$ref": "#/definitions/ipAddress"},
+					map[string]interface{}{"$ref": "#/definitions/hostname"},
+				},
+			},
+		},
+		"type": "object",
+		"properties": map[string]interface{}{
+			"protocol": map[string]interface{}{
+				"type":        "string",
+				"enum":        []interface{}{"tcp", "udp", "sctp"},
+				"description": "Dial the specified protocol when a ziti client connects to the service.",
+			},
+			"dialInterceptedProtocol": map[string]interface{}{
+				"type":        "boolean",
+				"enum":        []interface{}{true},
+				"description": "Dial the same protocol that was intercepted at the client tunneler. 'protocol' and 'dialInterceptedProtocol' are mutually exclusive.",
+			},
+			"address": map[string]interface{}{
+				"$ref":        "#/definitions/address",
+				"description": "Dial the specified ip address or hostname when a ziti client connects to the service.",
+			},
+			"dialInterceptedAddress": map[string]interface{}{
+				"type":        "boolean",
+				"enum":        []interface{}{true},
+				"description": "Dial the same ip address that was intercepted at the client tunneler. 'address' and 'dialInterceptedAddress' are mutually exclusive.",
+			},
+			"port": map[string]interface{}{
+				"type":        "integer",
+				"minimum":     0,
+				"maximum":     65535,
+				"description": "Dial the specified port when a ziti client connects to the service.",
+			},
+			"dialInterceptedPort": map[string]interface{}{
+				"type":        "boolean",
+				"enum":        []interface{}{true},
+				"description": "Dial the same port that was intercepted at the client tunneler. 'port' and 'dialInterceptedPort' are mutually exclusive.",
+			},
+			"portChecks": map[string]interface{}{
+				"$ref": "#/definitions/portCheckList",
+			},
+			"httpChecks": map[string]interface{}{
+				"$ref": "#/definitions/httpCheckList",
+			},
+			"listenOptions": map[string]interface{}{
+				"type":                 "object",
+				"additionalProperties": false,
+				"properties": map[string]interface{}{
+					"cost": map[string]interface{}{
+						"type":        "integer",
+						"minimum":     0,
+						"maximum":     65535,
+						"description": "defaults to 0",
+					},
+					"precedence": map[string]interface{}{
+						"type":        "string",
+						"enum":        []interface{}{"default", "required", "failed"},
+						"description": "defaults to 'default'",
+					},
+					"connectTimeoutSeconds": map[string]interface{}{
+						"type":        "integer",
+						"minimum":     0,
+						"maximum":     2147483647,
+						"description": "defaults to 5",
+					},
+					"maxConnections": map[string]interface{}{
+						"type":        "integer",
+						"minimum":     1,
+						"description": "defaults to 3",
+					},
+					"identity": map[string]interface{}{
+						"type":        "string",
+						"description": "Associate the hosting terminator with the specified identity. '$tunneler_id.name' resolves to the name of the hosting tunneler's identity. '$tunneler_id.tag[tagName]' resolves to the value of the 'tagName' tag on the hosting tunneler's identity.",
+					},
+					"bindUsingEdgeIdentity": map[string]interface{}{
+						"type":        "boolean",
+						"description": "Associate the hosting terminator with the name of the hosting tunneler's identity. Setting this to 'true' is equivalent to setting 'identiy=$tunneler_id.name'",
+					},
+				},
+			},
+		},
+		"additionalProperties": false,
+		"allOf": []interface{}{
+			map[string]interface{}{
+				"oneOf": []interface{}{
+					map[string]interface{}{"required": []interface{}{"protocol"}},
+					map[string]interface{}{"required": []interface{}{"dialInterceptedProtocol"}},
+				},
+			},
+			map[string]interface{}{
+				"oneOf": []interface{}{
+					map[string]interface{}{"required": []interface{}{"address"}},
+					map[string]interface{}{"required": []interface{}{"dialInterceptedAddress"}},
+				},
+			},
+			map[string]interface{}{
+				"oneOf": []interface{}{
+					map[string]interface{}{"required": []interface{}{"port"}},
+					map[string]interface{}{"required": []interface{}{"dialInterceptedPort"}},
+				},
+			},
+		},
+	},
+}
+
+var hostV2ConfigTypeId = "host.v2"
+var hostV2ConfigType = &ConfigType{
+	BaseExtEntity: boltz.BaseExtEntity{Id: hostV2ConfigTypeId},
+	Name:          "host.v2",
+	Schema: map[string]interface{}{
+		"$id": "http://ziti-edge.netfoundry.io/schemas/host-v2.schema.json",
+		"definitions": map[string]interface{}{
+			"duration": map[string]interface{}{
+				"type":    "string",
+				"pattern": "[0-9]+(h|m|s|ms)",
+			},
+			"method": map[string]interface{}{
+				"type": "string",
+				"enum": []interface{}{
+					"GET",
+					"POST",
+					"PUT",
+					"PATCH",
+				},
+			},
+			"action": map[string]interface{}{
+				"type":                 "object",
+				"additionalProperties": false,
+				"required": []interface{}{
+					"trigger",
+					"action",
+				},
+				"properties": map[string]interface{}{
+					"trigger": map[string]interface{}{
+						"type": "string",
+						"enum": []interface{}{
+							"fail",
+							"pass",
+							"change",
+						},
+					},
+					"consecutiveEvents": map[string]interface{}{
+						"type":    "integer",
+						"minimum": float64(0),
+						"maximum": float64(math.MaxUint16),
+					},
+					"duration": map[string]interface{}{
+						"$ref": "#/definitions/duration",
+					},
+					"action": map[string]interface{}{
+						"type":    "string",
+						"pattern": "(mark (un)?healthy|increase cost [0-9]+|decrease cost [0-9]+|send event)",
+					},
+				},
+			},
+			"actionList": map[string]interface{}{
+				"type": "array",
+				"items": map[string]interface{}{
+					"$ref": "#/definitions/action",
+				},
+				"minItems": 1,
+				"maxItems": 20,
+			},
+			"portCheck": map[string]interface{}{
+				"type":                 "object",
+				"additionalProperties": false,
+				"required": []interface{}{
+					"interval",
+					"timeout",
+					"address",
+				},
+				"properties": map[string]interface{}{
+					"interval": map[string]interface{}{"$ref": "#/definitions/duration"},
+					"timeout":  map[string]interface{}{"$ref": "#/definitions/duration"},
+					"address":  map[string]interface{}{"type": "string"},
+					"actions":  map[string]interface{}{"$ref": "#/definitions/actionList"},
+				},
+			},
+			"httpCheck": map[string]interface{}{
+				"type":                 "object",
+				"additionalProperties": false,
+				"required": []interface{}{
+					"interval",
+					"timeout",
+					"url",
+				},
+				"properties": map[string]interface{}{
+					"url":      map[string]interface{}{"type": "string"},
+					"method":   map[string]interface{}{"$ref": "#/definitions/method"},
+					"body":     map[string]interface{}{"type": "string"},
+					"interval": map[string]interface{}{"$ref": "#/definitions/duration"},
+					"timeout":  map[string]interface{}{"$ref": "#/definitions/duration"},
+					"actions":  map[string]interface{}{"$ref": "#/definitions/actionList"},
+					"expectStatus": map[string]interface{}{
+						"type":    "integer",
+						"minimum": float64(100),
+						"maximum": float64(599),
+					},
+					"expectInBody": map[string]interface{}{"type": "string"},
+				},
+			},
+			"portCheckList": map[string]interface{}{
+				"type": "array",
+				"items": map[string]interface{}{
+					"$ref": "#/definitions/portCheck",
+				},
+			},
+			"httpCheckList": map[string]interface{}{
+				"type": "array",
+				"items": map[string]interface{}{
+					"$ref": "#/definitions/httpCheck",
+				},
+			},
+			"ipAddressFormat": map[string]interface{}{
+				"oneOf": []interface{}{
+					map[string]interface{}{"format": "ipv4"},
+					map[string]interface{}{"format": "ipv6"},
+				},
+			},
+			"ipAddress": map[string]interface{}{
+				"type": "string",
+				"$ref": "#/definitions/ipAddressFormat",
+			},
+			"hostname": map[string]interface{}{
+				"type":   "string",
+				"format": "hostname",
+				"not":    map[string]interface{}{"$ref": "#/definitions/ipAddressFormat"},
+			},
+			"address": map[string]interface{}{
+				"oneOf": []interface{}{
+					map[string]interface{}{"$ref": "#/definitions/ipAddress"},
+					map[string]interface{}{"$ref": "#/definitions/hostname"},
+				},
+			},
+			"terminator": map[string]interface{}{
+				"type": "object",
+				"properties": map[string]interface{}{
+					"protocol": map[string]interface{}{
+						"type":        "string",
+						"enum":        []interface{}{"tcp", "udp", "sctp"},
+						"description": "Dial the specified protocol when a ziti client connects to the service.",
+					},
+					"dialInterceptedProtocol": map[string]interface{}{
+						"type":        "boolean",
+						"enum":        []interface{}{true},
+						"description": "Dial the same protocol that was intercepted at the client tunneler. 'protocol' and 'dialInterceptedProtocol' are mutually exclusive.",
+					},
+					"address": map[string]interface{}{
+						"$ref":        "#/definitions/address",
+						"description": "Dial the specified ip address or hostname when a ziti client connects to the service.",
+					},
+					"dialInterceptedAddress": map[string]interface{}{
+						"type":        "boolean",
+						"enum":        []interface{}{true},
+						"description": "Dial the same ip address that was intercepted at the client tunneler. 'address' and 'dialInterceptedAddress' are mutually exclusive.",
+					},
+					"port": map[string]interface{}{
+						"type":        "integer",
+						"minimum":     0,
+						"maximum":     65535,
+						"description": "Dial the specified port when a ziti client connects to the service.",
+					},
+					"dialInterceptedPort": map[string]interface{}{
+						"type":        "boolean",
+						"enum":        []interface{}{true},
+						"description": "Dial the same port that was intercepted at the client tunneler. 'port' and 'dialInterceptedPort' are mutually exclusive.",
+					},
+					"portChecks": map[string]interface{}{
+						"$ref": "#/definitions/portCheckList",
+					},
+					"httpChecks": map[string]interface{}{
+						"$ref": "#/definitions/httpCheckList",
+					},
+					"listenOptions": map[string]interface{}{
+						"type":                 "object",
+						"additionalProperties": false,
+						"properties": map[string]interface{}{
+							"cost": map[string]interface{}{
+								"type":        "integer",
+								"minimum":     0,
+								"maximum":     65535,
+								"description": "defaults to 0",
+							},
+							"precedence": map[string]interface{}{
+								"type":        "string",
+								"enum":        []interface{}{"default", "required", "failed"},
+								"description": "defaults to 'default'",
+							},
+							"connectTimeout": map[string]interface{}{"$ref": "#/definitions/duration"},
+							"maxConnections": map[string]interface{}{
+								"type":        "integer",
+								"minimum":     1,
+								"description": "defaults to 3",
+							},
+							"identity": map[string]interface{}{
+								"type":        "string",
+								"description": "Associate the hosting terminator with the specified identity. '$tunneler_id.name' resolves to the name of the hosting tunneler's identity. '$tunneler_id.tag[tagName]' resolves to the value of the 'tagName' tag on the hosting tunneler's identity.",
+							},
+							"bindUsingEdgeIdentity": map[string]interface{}{
+								"type":        "boolean",
+								"description": "Associate the hosting terminator with the name of the hosting tunneler's identity. Setting this to 'true' is equivalent to setting 'identiy=$tunneler_id.name'",
+							},
+						},
+					},
+				},
+				"additionalProperties": false,
+				"allOf": []interface{}{
+					map[string]interface{}{
+						"oneOf": []interface{}{
+							map[string]interface{}{"required": []interface{}{"protocol"}},
+							map[string]interface{}{"required": []interface{}{"dialInterceptedProtocol"}},
+						},
+					},
+					map[string]interface{}{
+						"oneOf": []interface{}{
+							map[string]interface{}{"required": []interface{}{"address"}},
+							map[string]interface{}{"required": []interface{}{"dialInterceptedAddress"}},
+						},
+					},
+					map[string]interface{}{
+						"oneOf": []interface{}{
+							map[string]interface{}{"required": []interface{}{"port"}},
+							map[string]interface{}{"required": []interface{}{"dialInterceptedPort"}},
+						},
+					},
+				},
+			},
+			"terminatorList": map[string]interface{}{
+				"type":     "array",
+				"minItems": 1,
+				"items": map[string]interface{}{
+					"$ref": "#/definitions/terminator",
+				},
+			},
+		},
+		"type": "object",
+		"properties": map[string]interface{}{
+			"terminators": map[string]interface{}{
+				"$ref": "#/definitions/terminatorList",
+			},
+		},
+	},
+}
+
 func (m *Migrations) createInitialTunnelerConfigTypes(step *boltz.MigrationStep) {
 	clientConfigTypeV1 := &ConfigType{
 		BaseExtEntity: boltz.BaseExtEntity{Id: clientConfigV1TypeId},

controller/persistence/migration_v14.go

@@ -133,133 +133,10 @@ func (m *Migrations) createInterceptV1ConfigType(step *boltz.MigrationStep) {
 }
 
 func (m *Migrations) createHostV1ConfigType(step *boltz.MigrationStep) {
-	configTypeName := "host.v1"
-	configTypeId := "NH5p4FpGR"
-	configType := &ConfigType{
-		BaseExtEntity: boltz.BaseExtEntity{Id: configTypeId},
-		Name:          configTypeName,
-		Schema: map[string]interface{}{
-			"$id": "http://ziti-edge.netfoundry.io/schemas/host-v1.schema.json",
-			"$defs": map[string]interface{}{
-				"ipAddressFormat": map[string]interface{}{
-					"oneOf": []interface{}{
-						map[string]interface{}{"format": "ipv4"},
-						map[string]interface{}{"format": "ipv6"},
-					},
-				},
-				"ipAddress": map[string]interface{}{
-					"type": "string",
-					"$ref": "#/$defs/ipAddressFormat",
-				},
-				"hostname": map[string]interface{}{
-					"type":   "string",
-					"format": "hostname",
-					"not":    map[string]interface{}{"$ref": "#/$defs/ipAddressFormat"},
-				},
-				"address": map[string]interface{}{
-					"oneOf": []interface{}{
-						map[string]interface{}{"$ref": "#/$defs/ipAddress"},
-						map[string]interface{}{"$ref": "#/$defs/hostname"},
-					},
-				},
-			},
-			"type": "object",
-			"properties": map[string]interface{}{
-				"protocol": map[string]interface{}{
-					"type":        "string",
-					"enum":        []interface{}{"tcp", "udp", "sctp"},
-					"description": "Dial the specified protocol when a ziti client connects to the service.",
-				},
-				"dialInterceptedProtocol": map[string]interface{}{
-					"type":        "boolean",
-					"enum":        []interface{}{true},
-					"description": "Dial the same protocol that was intercepted at the client tunneler. 'protocol' and 'dialInterceptedProtocol' are mutually exclusive.",
-				},
-				"address": map[string]interface{}{
-					"$ref":        "#/$defs/address",
-					"description": "Dial the specified ip address or hostname when a ziti client connects to the service.",
-				},
-				"dialInterceptedAddress": map[string]interface{}{
-					"type":        "boolean",
-					"enum":        []interface{}{true},
-					"description": "Dial the same ip address that was intercepted at the client tunneler. 'address' and 'dialInterceptedAddress' are mutually exclusive.",
-				},
-				"port": map[string]interface{}{
-					"type":        "integer",
-					"minimum":     0,
-					"maximum":     65535,
-					"description": "Dial the specified port when a ziti client connects to the service.",
-				},
-				"dialInterceptedPort": map[string]interface{}{
-					"type":        "boolean",
-					"enum":        []interface{}{true},
-					"description": "Dial the same port that was intercepted at the client tunneler. 'port' and 'dialInterceptedPort' are mutually exclusive.",
-				},
-				"listenOptions": map[string]interface{}{
-					"type":                 "object",
-					"additionalProperties": false,
-					"properties": map[string]interface{}{
-						"cost": map[string]interface{}{
-							"type":        "integer",
-							"minimum":     0,
-							"maximum":     65535,
-							"description": "defaults to 0",
-						},
-						"precedence": map[string]interface{}{
-							"type":        "string",
-							"enum":        []interface{}{"default", "required", "failed"},
-							"description": "defaults to 'default'",
-						},
-						"connectTimeoutSeconds": map[string]interface{}{
-							"type":        "integer",
-							"minimum":     0,
-							"maximum":     2147483647,
-							"description": "defaults to 5",
-						},
-						"maxConnections": map[string]interface{}{
-							"type":        "integer",
-							"minimum":     1,
-							"description": "defaults to 3",
-						},
-						"identity": map[string]interface{}{
-							"type":        "string",
-							"description": "Associate the hosting terminator with the specified identity. '$tunneler_id.name' resolves to the name of the hosting tunneler's identity. '$tunneler_id.tag[tagName]' resolves to the value of the 'tagName' tag on the hosting tunneler's identity.",
-						},
-						"bindUsingEdgeIdentity": map[string]interface{}{
-							"type":        "boolean",
-							"description": "Associate the hosting terminator with the name of the hosting tunneler's identity. Setting this to 'true' is equivalent to setting 'identiy=$tunneler_id.name'",
-						},
-					},
-				},
-			},
-			"additionalProperties": false,
-			"allOf": []interface{}{
-				map[string]interface{}{
-					"oneOf": []interface{}{
-						map[string]interface{}{"required": []interface{}{"protocol"}},
-						map[string]interface{}{"required": []interface{}{"dialInterceptedProtocol"}},
-					},
-				},
-				map[string]interface{}{
-					"oneOf": []interface{}{
-						map[string]interface{}{"required": []interface{}{"address"}},
-						map[string]interface{}{"required": []interface{}{"dialInterceptedAddress"}},
-					},
-				},
-				map[string]interface{}{
-					"oneOf": []interface{}{
-						map[string]interface{}{"required": []interface{}{"port"}},
-						map[string]interface{}{"required": []interface{}{"dialInterceptedPort"}},
-					},
-				},
-			},
-		},
-	}
-
-	cfg, _ := m.stores.ConfigType.LoadOneByName(step.Ctx.Tx(), configTypeName)
+	cfg, _ := m.stores.ConfigType.LoadOneByName(step.Ctx.Tx(), hostV1ConfigType.Name)
 	if cfg == nil {
-		step.SetError(m.stores.ConfigType.Create(step.Ctx, configType))
+		step.SetError(m.stores.ConfigType.Create(step.Ctx, hostV1ConfigType))
 	} else {
-		log.Debugf("'%s' config type already exists. not creating.", configTypeName)
+		log.Debugf("'%s' config type already exists. not creating.", hostV1ConfigType.Name)
 	}
 }

controller/persistence/migrations.go

@@ -22,7 +22,7 @@ import (
 )
 
 const (
-	CurrentDbVersion = 19
+	CurrentDbVersion = 20
 	FieldVersion     = "version"
 )
 
@@ -79,6 +79,12 @@ func (m *Migrations) migrate(step *boltz.MigrationStep) int {
 		m.updateIdentityTypes(step)
 	}
 
+	if step.CurrentVersion < 20 {
+		step.SetError(m.stores.ConfigType.Update(step.Ctx, serverConfigTypeV1, nil))
+		step.SetError(m.stores.ConfigType.Update(step.Ctx, hostV1ConfigType, nil))
+		step.SetError(m.stores.ConfigType.Create(step.Ctx, hostV2ConfigType))
+	}
+
 	// current version
 	if step.CurrentVersion <= CurrentDbVersion {
 		return CurrentDbVersion

controller/server/controller.go

@@ -140,12 +140,15 @@ func (c *Controller) GetCtrlHandlers(ch channel2.Channel) []channel2.ReceiveHand
 		handler_edge_ctrl.NewUpdateTerminatorHandler(c.AppEnv, ch),
 		handler_edge_ctrl.NewRemoveTerminatorHandler(c.AppEnv, ch),
 		handler_edge_ctrl.NewValidateSessionsHandler(c.AppEnv, ch),
+		handler_edge_ctrl.NewHealthEventHandler(c.AppEnv, ch),
+
 		handler_edge_ctrl.NewCreateApiSessionHandler(c.AppEnv, ch, tunnelState),
 		handler_edge_ctrl.NewCreateCircuitForTunnelHandler(c.AppEnv, ch, tunnelState),
 		handler_edge_ctrl.NewCreateTunnelTerminatorHandler(c.AppEnv, ch, tunnelState),
 		handler_edge_ctrl.NewUpdateTunnelTerminatorHandler(c.AppEnv, ch),
 		handler_edge_ctrl.NewRemoveTunnelTerminatorHandler(c.AppEnv, ch),
 		handler_edge_ctrl.NewListTunnelServicesHandler(c.AppEnv, ch, tunnelState),
+		handler_edge_ctrl.NewTunnelHealthEventHandler(c.AppEnv, ch),
 	}
 }
 

controller/sync_strats/marshal.go

@@ -17,13 +17,23 @@
 package sync_strats
 
 import (
-	"fmt"
 	"github.com/openziti/edge/controller/env"
 	"github.com/openziti/edge/pb/edge_ctrl_pb"
+	"go.etcd.io/bbolt"
 )
 
 func apiSessionToProto(ae *env.AppEnv, token, identityId, apiSessionId string) (*edge_ctrl_pb.ApiSession, error) {
-	fingerprints, err := getFingerprints(ae, identityId, apiSessionId)
+	var result *edge_ctrl_pb.ApiSession
+	err := ae.GetDbProvider().GetDb().View(func(tx *bbolt.Tx) error {
+		var err error
+		result, err = apiSessionToProtoWithTx(tx, ae, token, identityId, apiSessionId)
+		return err
+	})
+	return result, err
+}
+
+func apiSessionToProtoWithTx(tx *bbolt.Tx, ae *env.AppEnv, token, identityId, apiSessionId string) (*edge_ctrl_pb.ApiSession, error) {
+	fingerprints, err := getFingerprints(tx, ae, identityId, apiSessionId)
 	if err != nil {
 		return nil, err
 	}
@@ -35,19 +45,9 @@ func apiSessionToProto(ae *env.AppEnv, token, identityId, apiSessionId string) (
 	}, nil
 }
 
-func GetFingerprintsByApiSessionId(ae *env.AppEnv, apiSessionId string) ([]string, error) {
-	apiSession, err := ae.GetHandlers().ApiSession.Read(apiSessionId)
-
-	if err != nil {
-		return nil, fmt.Errorf("could not query fingerprints by api session id [%s]: %s", apiSessionId, err)
-	}
-
-	return getFingerprints(ae, apiSession.IdentityId, apiSessionId)
-}
-
-func getFingerprints(ae *env.AppEnv, identityId, apiSessionId string) ([]string, error) {
+func getFingerprints(tx *bbolt.Tx, ae *env.AppEnv, identityId, apiSessionId string) ([]string, error) {
 	prints := map[string]struct{}{}
-	err := ae.Handlers.ApiSession.VisitFingerprintsForApiSession(identityId, apiSessionId, func(fingerprint string) bool {
+	err := ae.Handlers.ApiSession.VisitFingerprintsForApiSession(tx, identityId, apiSessionId, func(fingerprint string) bool {
 		prints[fingerprint] = struct{}{}
 		return false
 	})

controller/sync_strats/sync_instant.go

@@ -436,7 +436,7 @@ func (strategy *InstantStrategy) synchronize(rtx *RouterSender) {
 				continue
 			}
 
-			apiSessionProto, err := apiSessionToProto(strategy.ae, apiSession.Token, apiSession.IdentityId, apiSession.Id)
+			apiSessionProto, err := apiSessionToProtoWithTx(tx, strategy.ae, apiSession.Token, apiSession.IdentityId, apiSession.Id)
 
 			if err != nil {
 				logger.WithError(err).WithField("apiSessionId", string(current)).Errorf("error turning apiSession [%s] into proto: %v", string(current), err)

go.mod

@@ -13,7 +13,7 @@ go 1.16
 replace go.etcd.io/bbolt => github.com/openziti/bbolt v1.3.6-0.20210317142109-547da822475e
 
 require (
-	github.com/AppsFlyer/go-sundheit v0.2.0
+	github.com/AppsFlyer/go-sundheit v0.3.1
 	github.com/Jeffail/gabs v1.4.0
 	github.com/achanda/go-sysctl v0.0.0-20160222034550-6be7678c45d2
 	github.com/blang/semver v3.5.1+incompatible
@@ -23,7 +23,7 @@ require (
 	github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa
 	github.com/go-openapi/errors v0.20.0
 	github.com/go-openapi/loads v0.20.2
-	github.com/go-openapi/runtime v0.19.26
+	github.com/go-openapi/runtime v0.19.27
 	github.com/go-openapi/spec v0.20.3
 	github.com/go-openapi/strfmt v0.20.0
 	github.com/go-openapi/swag v0.19.14
@@ -38,12 +38,12 @@ require (
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mdlayher/netlink v1.4.0
 	github.com/michaelquigley/pfxlog v0.3.7
-	github.com/miekg/dns v1.1.40
+	github.com/miekg/dns v1.1.41
 	github.com/mitchellh/mapstructure v1.4.1
 	github.com/netfoundry/secretstream v0.1.2
 	github.com/openziti/fabric v0.16.44
 	github.com/openziti/foundation v0.15.44
-	github.com/openziti/sdk-golang v0.15.27
+	github.com/openziti/sdk-golang v0.15.31
 	github.com/orcaman/concurrent-map v0.0.0-20210106121528-16402b402231
 	github.com/pkg/errors v0.9.1
 	github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0
@@ -55,9 +55,9 @@ require (
 	github.com/xeipuuv/gojsonschema v1.2.0
 	go.etcd.io/bbolt v1.3.5-0.20200615073812-232d8fc87f50
 	golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a
-	golang.org/x/net v0.0.0-20210226101413-39120d07d75e
-	golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208
-	golang.org/x/sys v0.0.0-20210216163648-f7da38b97c65
+	golang.org/x/net v0.0.0-20210226172049-e18ecbb05110
+	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
+	golang.org/x/sys v0.0.0-20210303074136-134d130e1a04
 	google.golang.org/protobuf v1.25.0
 	gopkg.in/resty.v1 v1.12.0
 	gopkg.in/yaml.v2 v2.4.0

go.sum

@@ -20,6 +20,8 @@ dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D
 git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg=
 github.com/AppsFlyer/go-sundheit v0.2.0 h1:FArqX+HbqZ6U32RC3giEAWRUpkggqxHj91KIvxNgwjU=
 github.com/AppsFlyer/go-sundheit v0.2.0/go.mod h1:rCRkVTMQo7/krF7xQ9X0XEF1an68viFR6/Gy02q+4ds=
+github.com/AppsFlyer/go-sundheit v0.3.1 h1:Zqnr3wV3WQmXonc234k9XZAoV2KHUHw3osR5k2iHQZE=
+github.com/AppsFlyer/go-sundheit v0.3.1/go.mod h1:iZ8zWMS7idcvmqewf5mEymWWgoOiG/0WD4+aeh+heX4=
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
@@ -174,8 +176,8 @@ github.com/go-openapi/runtime v0.19.4/go.mod h1:X277bwSUBxVlCYR3r7xgZZGKVvBd/29g
 github.com/go-openapi/runtime v0.19.15/go.mod h1:dhGWCTKRXlAfGnQG0ONViOZpjfg0m2gUt9nTQPQZuoo=
 github.com/go-openapi/runtime v0.19.16/go.mod h1:5P9104EJgYcizotuXhEuUrzVc+j1RiSjahULvYmlv98=
 github.com/go-openapi/runtime v0.19.24/go.mod h1:Lm9YGCeecBnUUkFTxPC4s1+lwrkJ0pthx8YvyjCfkgk=
-github.com/go-openapi/runtime v0.19.26 h1:K/6PoVNj5WJXUnMk+VEbELeXjtBkCS1UxTDa04tdXE0=
-github.com/go-openapi/runtime v0.19.26/go.mod h1:BvrQtn6iVb2QmiVXRsFAm6ZCAZBpbVKFfN6QWCp582M=
+github.com/go-openapi/runtime v0.19.27 h1:4zrQCJoP7rqNCUaApDv1MdPkaa5TuPfO05Lq5WLhX9I=
+github.com/go-openapi/runtime v0.19.27/go.mod h1:BvrQtn6iVb2QmiVXRsFAm6ZCAZBpbVKFfN6QWCp582M=
 github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI=
 github.com/go-openapi/spec v0.18.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI=
 github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY=
@@ -438,8 +440,8 @@ github.com/michaelquigley/pfxlog v0.3.7 h1:itV+da+Kek0GwApI2vOcGhLdMk6eCNfjXoR6/
 github.com/michaelquigley/pfxlog v0.3.7/go.mod h1:s4uqqB54PDLSU6JvpizMBJ2VU4cpG3bXsXJvFHqHDLU=
 github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
-github.com/miekg/dns v1.1.40 h1:pyyPFfGMnciYUk/mXpKkVmeMQjfXqt3FAJ2hy7tPiLA=
-github.com/miekg/dns v1.1.40/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
+github.com/miekg/dns v1.1.41 h1:WMszZWJG0XmzbK9FEmzH2TVcqYzFesusSIB41b8KHxY=
+github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI=
 github.com/miekg/pkcs11 v1.0.3 h1:iMwmD7I5225wv84WxIG/bmxz9AXjWvTWIbM/TYHvWtw=
 github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs=
 github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
@@ -487,12 +489,10 @@ github.com/openziti/bbolt v1.3.6-0.20210317142109-547da822475e/go.mod h1:G5EMThw
 github.com/openziti/dilithium v0.3.3/go.mod h1:vsCjI2AU/hon9e+dLhUFbCNGesJDj2ASgkySOcpmvjo=
 github.com/openziti/fabric v0.16.44 h1:pMEXl8uSdgd/Dj7d4F0qK5TnlLuse5uCpZs3Ah6fPbM=
 github.com/openziti/fabric v0.16.44/go.mod h1:soU2ImwCdNOnLDM5JUqpk5wAvrN9LaslZL02oThJQUM=
-github.com/openziti/foundation v0.15.40 h1:3xbBIARO16x5JgRHhzMMQQrkigWv7DpnuH5yP3railM=
-github.com/openziti/foundation v0.15.40/go.mod h1:kYCUttsGO/GjoD7BSNhVOeyYDXHYMAfz59YeLoEyhHY=
 github.com/openziti/foundation v0.15.44 h1:+XMOMaog+ABBvxgmDhk70H0buhw2RtpeDClxk+TJ3qQ=
 github.com/openziti/foundation v0.15.44/go.mod h1:kYCUttsGO/GjoD7BSNhVOeyYDXHYMAfz59YeLoEyhHY=
-github.com/openziti/sdk-golang v0.15.27 h1:mMyQqbkUlvq6NY7+bXt3LQRMTwTCtDOZZIwxgFWNdLY=
-github.com/openziti/sdk-golang v0.15.27/go.mod h1:pGaUSyl0WRgeJ70evLKPlSwOt1rJoEPDORUFO/WSE8M=
+github.com/openziti/sdk-golang v0.15.31 h1:vo+/wRbNtYBvkxHrAo9Is32SAJQjlhIS+9gHoajB1g0=
+github.com/openziti/sdk-golang v0.15.31/go.mod h1:jrvkpxP7U1ejSKEj4J9/UTNWlt6nJaofUGKkTx7afvM=
 github.com/orcaman/concurrent-map v0.0.0-20190826125027-8c72a8bb44f6/go.mod h1:Lu3tH6HLW3feq74c2GC+jIMS/K2CFcDWnWD9XkenwhI=
 github.com/orcaman/concurrent-map v0.0.0-20210106121528-16402b402231 h1:fa50YL1pzKW+1SsBnJDOHppJN9stOEwS+CRWyUtyYGU=
 github.com/orcaman/concurrent-map v0.0.0-20210106121528-16402b402231/go.mod h1:Lu3tH6HLW3feq74c2GC+jIMS/K2CFcDWnWD9XkenwhI=
@@ -654,7 +654,6 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
@@ -680,7 +679,6 @@ golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU
 golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
 golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
 golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
-golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -703,7 +701,6 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR
 golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191007182048-72f939374954/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191112182307-2180aed22343/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -718,8 +715,8 @@ golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwY
 golang.org/x/net v0.0.0-20201216054612-986b41b23924/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20210226101413-39120d07d75e h1:jIQURUJ9mlLvYwTBtRHm9h58rYhSonLvRvgAnP8Nr7I=
-golang.org/x/net v0.0.0-20210226101413-39120d07d75e/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -733,8 +730,8 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 h1:qwRHBd0NqMbJxfbotnDhm2ByMI1Shq4Y6oRJo21SGJA=
-golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -762,7 +759,6 @@ golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -782,8 +778,9 @@ golang.org/x/sys v0.0.0-20201218084310-7d0127a74742/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210110051926-789bb1bd4061/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210123111255-9b0068b26619/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210216163648-f7da38b97c65 h1:pTMjDVnP5eVRRlWO76rEWJ8JoC6Lf1CmyjPZXRiy2Sw=
 golang.org/x/sys v0.0.0-20210216163648-f7da38b97c65/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210303074136-134d130e1a04 h1:cEhElsAv9LUt9ZUUocxzWe05oFLVd+AA2nstydTeI8g=
+golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -825,9 +822,7 @@ golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

health/checks.go

@@ -48,6 +48,10 @@ func (self *ActionDefinition) CreateAction() (Action, error) {
 		result.actionImpl = func(state *ServiceState) {
 			state.nextPrecedence = edge.PrecedenceFailed
 		}
+	} else if self.Action == "send event" {
+		result.actionImpl = func(state *ServiceState) {
+			state.sendEvent = true
+		}
 	} else {
 		increase := true
 		var costStr string
@@ -120,10 +124,12 @@ func (self *actionImpl) Matches(result *health.Result) bool {
 	if self.Trigger == "fail" && result.IsHealthy() {
 		return false
 	}
-
 	if self.Trigger == "pass" && !result.IsHealthy() {
 		return false
 	}
+	if self.Trigger == "change" && self.consectivePasses != 1 && result.ContiguousFailures != 1 {
+		return false
+	}
 
 	if self.ConsecutiveEvents != nil {
 		if result.IsHealthy() && self.consectivePasses < int64(*self.ConsecutiveEvents) {

health/health.go

@@ -7,6 +7,7 @@ import (
 	"github.com/openziti/foundation/util/concurrenz"
 	"github.com/openziti/sdk-golang/ziti"
 	"github.com/openziti/sdk-golang/ziti/edge"
+	"github.com/sirupsen/logrus"
 	"reflect"
 	"sync"
 	"time"
@@ -18,6 +19,7 @@ const (
 
 type ServiceUpdater interface {
 	UpdateCostAndPrecedence(cost uint16, precedence edge.Precedence) error
+	SendHealthEvent(pass bool) error
 }
 
 type Check interface {
@@ -32,8 +34,13 @@ type Action interface {
 }
 
 func NewServiceState(service string, precedence ziti.Precedence, cost uint16, updater ServiceUpdater) *ServiceState {
+	return NewServiceStateWithContext("0", service, precedence, cost, updater)
+}
+
+func NewServiceStateWithContext(service, hostContext string, precedence ziti.Precedence, cost uint16, updater ServiceUpdater) *ServiceState {
 	return &ServiceState{
 		Service:            service,
+		HostContext:        hostContext,
 		BaselinePrecedence: edge.Precedence(precedence),
 		currentPrecedence:  edge.Precedence(precedence),
 		nextPrecedence:     edge.Precedence(precedence),
@@ -45,11 +52,13 @@ func NewServiceState(service string, precedence ziti.Precedence, cost uint16, up
 }
 
 type ServiceState struct {
-	Service string
+	Service     string
+	HostContext string
 
 	BaselinePrecedence edge.Precedence
 	currentPrecedence  edge.Precedence
 	nextPrecedence     edge.Precedence
+	sendEvent          bool
 
 	BaselineCost uint16
 	Updater      ServiceUpdater
@@ -61,16 +70,26 @@ func (self *ServiceState) IsChanged() bool {
 	return self.nextPrecedence != self.currentPrecedence || self.nextCost != self.currentCost
 }
 
-func (self *ServiceState) Update() {
+func (self *ServiceState) HandleActionResults(healthy bool) {
+	if self.sendEvent {
+		if err := self.Updater.SendHealthEvent(healthy); err != nil {
+			logrus.WithError(err).
+				WithField("service", self.Service).
+				WithField("hostContext", self.HostContext).
+				WithField("isHealth", healthy).
+				Error("error sending health event")
+		}
+		self.sendEvent = false
+	}
+
 	if !self.IsChanged() {
 		return
 	}
 
-	log := pfxlog.Logger()
-
 	if err := self.Updater.UpdateCostAndPrecedence(self.nextCost, self.nextPrecedence); err != nil {
-		log.WithError(err).
+		logrus.WithError(err).
 			WithField("service", self.Service).
+			WithField("hostContext", self.HostContext).
 			WithField("nextCost", self.nextCost).
 			WithField("nextPrecedence", self.nextPrecedence).
 			Error("error updating cost/precedence on service")
@@ -89,11 +108,10 @@ type checkContext struct {
 
 func NewManager() Manager {
 	result := &manager{
-		health:  health.New(),
 		results: make(chan *result, 16),
 	}
+	result.health = health.New(health.WithCheckListeners(result))
 
-	result.health.WithCheckListener(result)
 	go result.handleResults()
 
 	return result
@@ -102,6 +120,7 @@ func NewManager() Manager {
 type Manager interface {
 	RegisterServiceChecks(service *ServiceState, checkDefinitions []CheckDefinition) error
 	UnregisterServiceChecks(service string)
+	UnregisterServiceContextChecks(service, context string)
 	Shutdown()
 }
 
@@ -137,12 +156,12 @@ func (self *manager) handleResult(result *result) {
 					action.Invoke(check.serviceState)
 				}
 			}
-			check.serviceState.Update()
+			check.serviceState.HandleActionResults(result.IsHealthy())
 		} else {
-			pfxlog.Logger().Errorf("coding error, check was of incorrect type: %v", reflect.TypeOf(val))
+			logrus.Errorf("coding error, check was of incorrect type: %v", reflect.TypeOf(val))
 		}
 	} else {
-		pfxlog.Logger().Warnf("no health check context found for %v", result.name)
+		logrus.Warnf("no health check context found for %v", result.name)
 	}
 }
 
@@ -157,15 +176,30 @@ func (self *manager) UnregisterServiceChecks(service string) {
 	})
 }
 
+func (self *manager) UnregisterServiceContextChecks(service, hostContext string) {
+	log := logrus.WithField("service", service).WithField("hostContext", hostContext)
+	log.Debug("removing health checks")
+
+	self.checks.Range(func(key, val interface{}) bool {
+		check := val.(*checkContext)
+		if check.serviceState.Service == service && check.serviceState.HostContext == hostContext {
+			log.WithField("check", check.id).Debug("removing check")
+			self.checks.Delete(check.id)
+			self.health.Deregister(check.id)
+		}
+		return true
+	})
+}
+
 func (self *manager) RegisterServiceChecks(service *ServiceState, checkDefinitions []CheckDefinition) error {
 	logger := pfxlog.Logger()
 	for idx, checkDefinition := range checkDefinitions {
-		id := fmt.Sprintf("%v_%v", service.Service, idx)
+		id := fmt.Sprintf("%v_%v_%v", service.Service, service.HostContext, idx)
 		_, found := self.checks.Load(id)
 		counter := 0
 		for found {
 			counter++
-			id := fmt.Sprintf("%v_%v_%v", service.Service, idx, counter)
+			id := fmt.Sprintf("%v_%v_%v_%v", service.Service, service.HostContext, idx, counter)
 			_, found = self.checks.Load(id)
 		}
 
@@ -195,7 +229,9 @@ func (self *manager) RegisterServiceChecks(service *ServiceState, checkDefinitio
 			InitiallyPassing: true,
 		}
 
-		logger.WithField("service", service.Service).Debugf("adding check: %v", checkDefinition.String())
+		logger.WithField("service", service.Service).
+			WithField("hostContext", service.HostContext).
+			Debugf("adding check: %v", checkDefinition.String())
 		if err = self.health.RegisterCheck(checkConfig); err != nil {
 			return err
 		}
@@ -207,6 +243,10 @@ func (self *manager) OnCheckStarted(string) {
 	// does nothing
 }
 
+func (self *manager) OnCheckRegistered(string, health.Result) {
+	// does nothing
+}
+
 func (self *manager) OnCheckCompleted(name string, r health.Result) {
 	if self.closed.Get() {
 		return

health/health_test.go

@@ -21,9 +21,23 @@ type svcChangeEvent struct {
 	precedence edge.Precedence
 }
 
+func newEventTracker(t *testing.T) *eventTracker {
+	return &eventTracker{
+		Assertions:  require.New(t),
+		events:      make(chan *svcChangeEvent, 16),
+		healthSends: make(chan bool, 16),
+	}
+}
+
 type eventTracker struct {
 	*require.Assertions
-	events chan *svcChangeEvent
+	events      chan *svcChangeEvent
+	healthSends chan bool
+}
+
+func (self *eventTracker) SendHealthEvent(pass bool) error {
+	self.healthSends <- pass
+	return nil
 }
 
 func (self *eventTracker) UpdateCostAndPrecedence(cost uint16, precedence edge.Precedence) error {
@@ -32,6 +46,7 @@ func (self *eventTracker) UpdateCostAndPrecedence(cost uint16, precedence edge.P
 		cost:       cost,
 		precedence: precedence,
 	}
+
 	return nil
 }
 
@@ -52,6 +67,20 @@ func (self *eventTracker) assertEvent(t time.Duration, cost uint16, p edge.Prece
 	self.Equal(int(p), int(evt.precedence))
 }
 
+func (self *eventTracker) assertHealthSend(t time.Duration, checkPassed bool) {
+	time.Sleep(t)
+
+	var send bool
+	select {
+	case send = <-self.healthSends:
+	case <-time.After(100 * time.Millisecond):
+		self.Fail("no event found")
+		return
+	}
+
+	self.Equal(checkPassed, send)
+}
+
 func (self *eventTracker) assertNoEvent(t time.Duration) {
 	select {
 	case <-self.events:
@@ -60,16 +89,21 @@ func (self *eventTracker) assertNoEvent(t time.Duration) {
 	}
 }
 
+func (self *eventTracker) assertNoSend(t time.Duration) {
+	select {
+	case <-self.healthSends:
+		self.Fail("no health send should be found")
+	case <-time.After(t):
+	}
+}
+
 func Test_ManagerWithEventCounts(t *testing.T) {
 	mgr := NewManager()
 	defer mgr.Shutdown()
 
 	req := require.New(t)
 
-	et := &eventTracker{
-		Assertions: req,
-		events:     make(chan *svcChangeEvent, 16),
-	}
+	et := newEventTracker(t)
 
 	state := NewServiceState("my-service", ziti.PrecedenceRequired, 100, et)
 
@@ -149,10 +183,7 @@ func Test_ManagerWithDurations(t *testing.T) {
 
 	req := require.New(t)
 
-	et := &eventTracker{
-		Assertions: req,
-		events:     make(chan *svcChangeEvent, 16),
-	}
+	et := newEventTracker(t)
 
 	state := NewServiceState("my-service", ziti.PrecedenceDefault, 0, et)
 
@@ -280,7 +311,7 @@ func Test_ManagerWithSimpleHttp(t *testing.T) {
 	defer mgr.Shutdown()
 
 	req := require.New(t)
-	et := &eventTracker{Assertions: req, events: make(chan *svcChangeEvent, 16)}
+	et := newEventTracker(t)
 	state := NewServiceState("my-service", ziti.PrecedenceDefault, 0, et)
 
 	healthCheckDef := newHttpCheck()
@@ -289,7 +320,7 @@ func Test_ManagerWithSimpleHttp(t *testing.T) {
 	req.NoError(err)
 	et.assertEvent(100*time.Millisecond, 0, edge.PrecedenceFailed)
 
-	mgr.UnregisterServiceChecks("my-server")
+	mgr.UnregisterServiceChecks("my-service")
 
 	state = NewServiceState("my-service", ziti.PrecedenceDefault, 0, et)
 	closeF := runHttpCheck(req, func(resp http.ResponseWriter, req *http.Request) {
@@ -307,7 +338,7 @@ func Test_ManagerWithHttpStatusCode(t *testing.T) {
 	defer mgr.Shutdown()
 
 	req := require.New(t)
-	et := &eventTracker{Assertions: req, events: make(chan *svcChangeEvent, 16)}
+	et := newEventTracker(t)
 	state := NewServiceState("my-service", ziti.PrecedenceDefault, 0, et)
 
 	healthCheckDef := newHttpCheck()
@@ -324,7 +355,7 @@ func Test_ManagerWithHttpStatusCode(t *testing.T) {
 	et.assertEvent(100*time.Millisecond, 0, edge.PrecedenceFailed)
 
 	closeF()
-	mgr.UnregisterServiceChecks("my-server")
+	mgr.UnregisterServiceChecks("my-service")
 
 	state = NewServiceState("my-service", ziti.PrecedenceDefault, 0, et)
 	closeF = runHttpCheck(req, func(resp http.ResponseWriter, req *http.Request) {
@@ -343,7 +374,7 @@ func Test_ManagerWithHttpExpectBody(t *testing.T) {
 	defer mgr.Shutdown()
 
 	req := require.New(t)
-	et := &eventTracker{Assertions: req, events: make(chan *svcChangeEvent, 16)}
+	et := newEventTracker(t)
 	state := NewServiceState("my-service", ziti.PrecedenceDefault, 0, et)
 
 	healthCheckDef := newHttpCheck()
@@ -361,7 +392,7 @@ func Test_ManagerWithHttpExpectBody(t *testing.T) {
 	et.assertEvent(100*time.Millisecond, 0, edge.PrecedenceFailed)
 
 	closeF()
-	mgr.UnregisterServiceChecks("my-server")
+	mgr.UnregisterServiceChecks("my-service")
 
 	state = NewServiceState("my-service", ziti.PrecedenceDefault, 0, et)
 	closeF = runHttpCheck(req, func(resp http.ResponseWriter, req *http.Request) {
@@ -381,7 +412,7 @@ func Test_ManagerWithHttpMethod(t *testing.T) {
 	defer mgr.Shutdown()
 
 	req := require.New(t)
-	et := &eventTracker{Assertions: req, events: make(chan *svcChangeEvent, 16)}
+	et := newEventTracker(t)
 	state := NewServiceState("my-service", ziti.PrecedenceDefault, 0, et)
 
 	healthCheckDef := newHttpCheck()
@@ -405,7 +436,7 @@ func Test_ManagerWithHttpMethod(t *testing.T) {
 	et.assertEvent(100*time.Millisecond, 0, edge.PrecedenceFailed)
 
 	closeF()
-	mgr.UnregisterServiceChecks("my-server")
+	mgr.UnregisterServiceChecks("my-service")
 
 	state = NewServiceState("my-service", ziti.PrecedenceDefault, 0, et)
 	closeF = runHttpCheck(req, func(resp http.ResponseWriter, req *http.Request) {
@@ -429,7 +460,7 @@ func Test_ManagerWithHttpBody(t *testing.T) {
 	defer mgr.Shutdown()
 
 	req := require.New(t)
-	et := &eventTracker{Assertions: req, events: make(chan *svcChangeEvent, 16)}
+	et := newEventTracker(t)
 	state := NewServiceState("my-service", ziti.PrecedenceDefault, 0, et)
 
 	healthCheckDef := newHttpCheck()
@@ -459,7 +490,7 @@ func Test_ManagerWithHttpBody(t *testing.T) {
 	et.assertEvent(100*time.Millisecond, 0, edge.PrecedenceFailed)
 
 	closeF()
-	mgr.UnregisterServiceChecks("my-server")
+	mgr.UnregisterServiceChecks("my-service")
 
 	state = NewServiceState("my-service", ziti.PrecedenceDefault, 0, et)
 	closeF = runHttpCheck(req, func(resp http.ResponseWriter, req *http.Request) {
@@ -487,3 +518,38 @@ func Test_ManagerWithHttpBody(t *testing.T) {
 	req.NoError(err)
 	et.assertEvent(100*time.Millisecond, 10, edge.PrecedenceDefault)
 }
+
+func Test_ChangeAndHealthSend(t *testing.T) {
+	mgr := NewManager()
+	defer mgr.Shutdown()
+
+	req := require.New(t)
+	et := newEventTracker(t)
+	state := NewServiceState("my-service", ziti.PrecedenceDefault, 0, et)
+
+	healthCheckDef := &HttpCheckDefinition{
+		BaseCheckDefinition: BaseCheckDefinition{
+			Interval: 100 * time.Millisecond,
+			Actions: []*ActionDefinition{
+				{Trigger: "change", Action: "send event"},
+			},
+		},
+		Url:     "http://localhost:9876",
+		Timeout: 100 * time.Millisecond,
+	}
+
+	err := mgr.RegisterServiceChecks(state, []CheckDefinition{healthCheckDef})
+	req.NoError(err)
+	et.assertHealthSend(100*time.Millisecond, false)
+	et.assertNoSend(200 * time.Millisecond)
+
+	state = NewServiceState("my-service", ziti.PrecedenceDefault, 0, et)
+	closeF := runHttpCheck(req, func(resp http.ResponseWriter, req *http.Request) {
+		resp.WriteHeader(200)
+	})
+	defer closeF()
+
+	et.assertHealthSend(500*time.Millisecond, true)
+	et.assertNoSend(200 * time.Millisecond)
+
+}

pb/edge_ctrl_pb/edge_ctrl.proto

@@ -26,6 +26,7 @@ enum ContentType {
   RemoveTerminatorRequestType = 20211;
   RemoveTerminatorResponseType = 20212;
   ValidateSessionsRequestType = 20213;
+  HealthEventType = 20214;
   EnrollType = 20300;
   EnrollCertsType = 20301;
 
@@ -43,6 +44,8 @@ enum ContentType {
 
   RemoveTunnelTerminatorRequestType = 20410;
   RemoveTunnelTerminatorResponseType = 20411;
+
+  TunnelHealthEventType = 20412;
 }
 
 enum SessionType {
@@ -50,6 +53,12 @@ enum SessionType {
   Bind = 1;
 }
 
+enum Header {
+  HeaderZero = 0;
+  TerminatorId = 1022;
+  CheckPassed = 1023;
+}
+
 message ServerHello {
   string version = 1;
   map<string, string> data = 2;
@@ -163,6 +172,13 @@ message UpdateTerminatorRequest {
   bool updateCost = 7;
 }
 
+message HealthEventRequest {
+  string sessionToken = 1;
+  repeated string fingerprints = 2;
+  string terminatorId = 3;
+  bool checkPassed = 4;
+}
+
 message ValidateSessionsRequest {
   repeated string sessionTokens = 1;
 }

pb/edge_ctrl_pb/messages.go

@@ -36,6 +36,10 @@ func (request *ValidateSessionsRequest) GetContentType() int32 {
 	return int32(ContentType_ValidateSessionsRequestType)
 }
 
+func (request *HealthEventRequest) GetContentType() int32 {
+	return int32(ContentType_HealthEventType)
+}
+
 func (request *CreateApiSessionRequest) GetContentType() int32 {
 	return int32(ContentType_CreateApiSessionRequestType)
 }

router/handler_edge_ctrl/sessionRemoved.go

@@ -53,7 +53,7 @@ func (h *sessionRemovedHandler) HandleReceive(msg *channel2.Message, ch channel2
 				pfxlog.Logger().
 					WithField("sessionToken", token).
 					WithField("sessionId", id).
-					Debugf("removing api session [token: %s] [id: %s]", token, id)
+					Debugf("removing session [token: %s] [id: %s]", token, id)
 				h.sm.RemoveSession(token)
 			}
 		} else {

router/xgress_edge/accept.go

@@ -70,6 +70,11 @@ func (handler edgeBindHandler) BindChannel(ch channel2.Channel) error {
 		Handler: proxy.processUpdateBind,
 	})
 
+	ch.AddReceiveHandler(&edge.AsyncFunctionReceiveAdapter{
+		Type:    edge.ContentTypeHealthEvent,
+		Handler: proxy.processHealthEvent,
+	})
+
 	ch.AddReceiveHandler(&edge.FunctionReceiveAdapter{
 		Type:    edge.ContentTypeStateClosed,
 		Handler: proxy.msgMux.HandleReceive,

router/xgress_edge/listener.go

@@ -28,6 +28,7 @@ import (
 	"github.com/openziti/foundation/channel2"
 	"github.com/openziti/foundation/identity/identity"
 	"github.com/openziti/foundation/transport"
+	"github.com/openziti/foundation/util/protobufs"
 	"github.com/openziti/sdk-golang/ziti/edge"
 	"github.com/pkg/errors"
 	"time"
@@ -334,6 +335,35 @@ func (self *edgeClientConn) processUpdateBind(req *channel2.Message, ch channel2
 	}
 }
 
+func (self *edgeClientConn) processHealthEvent(req *channel2.Message, ch channel2.Channel) {
+	token := string(req.Body)
+	log := pfxlog.ContextLogger(ch.Label()).WithField("sessionId", token).WithFields(edge.GetLoggerFields(req))
+
+	terminator, ok := self.listener.factory.hostedServices.Get(token)
+
+	if !ok {
+		log.Error("failed to update bind, no listener found")
+		return
+	}
+
+	checkPassed, _ := req.GetBoolHeader(edge.HealthStatusHeader)
+
+	request := &edge_ctrl_pb.HealthEventRequest{
+		SessionToken: token,
+		Fingerprints: self.fingerprints.Prints(),
+		TerminatorId: terminator.terminatorId,
+		CheckPassed:  checkPassed,
+	}
+
+	log = log.WithField("terminator", terminator.terminatorId).
+		WithField("checkPassed", checkPassed)
+	log.Debug("sending health event")
+
+	if err := protobufs.Send(self.listener.factory.Channel(), request); err != nil {
+		log.WithError(err).Error("send failed")
+	}
+}
+
 func (self *edgeClientConn) sendStateConnectedReply(req *channel2.Message, hostData map[uint32][]byte) {
 	connId, _ := req.GetUint32Header(edge.ConnIdHeader)
 	msg := edge.NewStateConnectedMsg(connId)

router/xgress_edge_tunnel/fabric.go

@@ -267,7 +267,7 @@ func (self *fabricProvider) HostService(hostCtx tunnel.HostingContext) (tunnel.H
 		}
 	})
 
-	terminator.terminatorId = string(responseMsg.Body)
+	terminator.terminatorId = response.TerminatorId
 	return terminator, nil
 }
 
@@ -314,6 +314,24 @@ func (self *fabricProvider) updateTerminator(terminatorId string, cost *uint16,
 	return nil
 }
 
+func (self *fabricProvider) sendHealthEvent(terminatorId string, checkPassed bool) error {
+	msg := channel2.NewMessage(int32(edge_ctrl_pb.ContentType_TunnelHealthEventType), nil)
+	msg.Headers[int32(edge_ctrl_pb.Header_TerminatorId)] = []byte(terminatorId)
+	msg.PutBoolHeader(int32(edge_ctrl_pb.Header_CheckPassed), checkPassed)
+
+	logger := logrus.WithField("terminator", terminatorId).
+		WithField("checkPassed", checkPassed)
+	logger.Debug("sending health event")
+
+	if err := self.ctrlCh.Channel().Send(msg); err != nil {
+		logger.WithError(err).Error("health event send failed")
+	} else {
+		logger.Debug("health event sent")
+	}
+
+	return nil
+}
+
 func (self *fabricProvider) requestServiceList(lastUpdateToken []byte) {
 	msg := channel2.NewMessage(int32(edge_ctrl_pb.ContentType_ListServicesRequestType), lastUpdateToken)
 	if err := self.ctrlCh.Channel().Send(msg); err != nil {
@@ -329,9 +347,18 @@ type tunnelTerminator struct {
 	closed        concurrenz.AtomicBoolean
 }
 
+func (self *tunnelTerminator) SendHealthEvent(pass bool) error {
+	return self.provider.sendHealthEvent(self.terminatorId, pass)
+}
+
 func (self *tunnelTerminator) Close() error {
 	if self.closed.CompareAndSwap(false, true) {
+		log := logrus.WithField("service", self.context.ServiceName()).WithField("terminator", self.terminatorId)
+		log.Debug("closing tunnel terminator context")
+		self.context.OnClose()
+		log.Debug("unregistering session listener for tunnel terminator")
 		self.closeCallback()
+		log.Debug("removing tunnel terminator")
 		return self.provider.removeTerminator(self.terminatorId)
 	}
 	return nil

router/xgress_edge_tunnel/servicepoll.go

@@ -63,15 +63,21 @@ func (self *servicePoller) handleServiceListUpdate(lastUpdateToken []byte, servi
 		idMap[s.Id] = s
 	}
 
+	var toRemove []string
+
 	// process Deletes
 	self.services.IterCb(func(k string, value interface{}) {
 		svc := value.(*edge.Service)
 		if _, found := idMap[svc.Id]; !found {
-			self.services.Remove(k)
+			toRemove = append(toRemove, k)
 			self.serviceListener.HandleServicesChange(ziti.ServiceRemoved, svc)
 		}
 	})
 
+	for _, key := range toRemove {
+		self.services.Remove(key)
+	}
+
 	// Adds and Updates
 	for _, s := range services {
 		self.services.Upsert(s.Name, s, func(exist bool, valueInMap interface{}, newValue interface{}) interface{} {
@@ -82,6 +88,8 @@ func (self *servicePoller) handleServiceListUpdate(lastUpdateToken []byte, servi
 			if !reflect.DeepEqual(valueInMap, s) {
 				self.serviceListener.HandleServicesChange(ziti.ServiceChanged, s)
 				return s
+			} else {
+				logrus.WithField("service", s.Name).Debug("no change detected in service definition")
 			}
 			return valueInMap
 		})
@@ -91,7 +99,8 @@ func (self *servicePoller) handleServiceListUpdate(lastUpdateToken []byte, servi
 // TODO: just push updates down the control channel when necessary
 func (self *servicePoller) pollServices(pollInterval time.Duration) {
 	if err := self.fabricProvider.authenticate(); err != nil {
-		logrus.Panic(err)
+		logrus.WithError(err).Fatal("xgress_edge_tunnel unable to authenticate to controller. " +
+			"ensure tunneler mode is enabled for this router or disable tunnel listener. exiting ")
 	}
 
 	ticker := time.NewTicker(pollInterval)

router/xgress_edge_tunnel/tunneler.go

@@ -65,6 +65,7 @@ func (self *tunneler) Start() error {
 			return errors.Wrap(err, "failed to initialize tproxy interceptor")
 		}
 	} else if self.listenOptions.mode == "host" {
+		self.listenOptions.resolver = ""
 		self.interceptor = host.New()
 	} else if self.listenOptions.mode == "proxy" {
 		self.listenOptions.resolver = ""

tunnel/const.go

@@ -1,7 +1,8 @@
-package intercept
+package tunnel
 
 const (
 	InterceptedProtocolKey = "intercepted_protocol"
 	InterceptedIpKey       = "intercepted_ip"
 	InterceptedPortKey     = "intercepted_port"
+	SourceIpKey            = "source_ip"
 )

tunnel/entities/host.v1.json

@@ -0,0 +1,166 @@
+{
+  "schema": {
+    "$defs": {
+      "address": {
+        "oneOf": [
+          {
+            "$ref": "#/$defs/ipAddress"
+          },
+          {
+            "$ref": "#/$defs/hostname"
+          }
+        ]
+      },
+      "hostname": {
+        "format": "hostname",
+        "not": {
+          "$ref": "#/$defs/ipAddressFormat"
+        },
+        "type": "string"
+      },
+      "ipAddress": {
+        "$ref": "#/$defs/ipAddressFormat",
+        "type": "string"
+      },
+      "ipAddressFormat": {
+        "oneOf": [
+          {
+            "format": "ipv4"
+          },
+          {
+            "format": "ipv6"
+          }
+        ]
+      }
+    },
+    "$id": "http://ziti-edge.netfoundry.io/schemas/host-v1.schema.json",
+    "additionalProperties": false,
+    "allOf": [
+      {
+        "oneOf": [
+          {
+            "required": [
+              "protocol"
+            ]
+          },
+          {
+            "required": [
+              "dialInterceptedProtocol"
+            ]
+          }
+        ]
+      },
+      {
+        "oneOf": [
+          {
+            "required": [
+              "address"
+            ]
+          },
+          {
+            "required": [
+              "dialInterceptedAddress"
+            ]
+          }
+        ]
+      },
+      {
+        "oneOf": [
+          {
+            "required": [
+              "port"
+            ]
+          },
+          {
+            "required": [
+              "dialInterceptedPort"
+            ]
+          }
+        ]
+      }
+    ],
+    "properties": {
+      "address": {
+        "$ref": "#/$defs/address",
+        "description": "Dial the specified ip address or hostname when a ziti client connects to the service."
+      },
+      "dialInterceptedAddress": {
+        "description": "Dial the same ip address that was intercepted at the client tunneler. 'address' and 'dialInterceptedAddress' are mutually exclusive.",
+        "enum": [
+          true
+        ],
+        "type": "boolean"
+      },
+      "dialInterceptedPort": {
+        "description": "Dial the same port that was intercepted at the client tunneler. 'port' and 'dialInterceptedPort' are mutually exclusive.",
+        "enum": [
+          true
+        ],
+        "type": "boolean"
+      },
+      "dialInterceptedProtocol": {
+        "description": "Dial the same protocol that was intercepted at the client tunneler. 'protocol' and 'dialInterceptedProtocol' are mutually exclusive.",
+        "enum": [
+          true
+        ],
+        "type": "boolean"
+      },
+      "listenOptions": {
+        "additionalProperties": false,
+        "properties": {
+          "bindUsingEdgeIdentity": {
+            "description": "Associate the hosting terminator with the name of the hosting tunneler's identity. Setting this to 'true' is equivalent to setting 'identiy=$tunneler_id.name'",
+            "type": "boolean"
+          },
+          "connectTimeoutSeconds": {
+            "description": "defaults to 5",
+            "maximum": 2147483647,
+            "minimum": 0,
+            "type": "integer"
+          },
+          "cost": {
+            "description": "defaults to 0",
+            "maximum": 65535,
+            "minimum": 0,
+            "type": "integer"
+          },
+          "identity": {
+            "description": "Associate the hosting terminator with the specified identity. '$tunneler_id.name' resolves to the name of the hosting tunneler's identity. '$tunneler_id.tag[tagName]' resolves to the value of the 'tagName' tag on the hosting tunneler's identity.",
+            "type": "string"
+          },
+          "maxConnections": {
+            "description": "defaults to 3",
+            "minimum": 1,
+            "type": "integer"
+          },
+          "precedence": {
+            "description": "defaults to 'default'",
+            "enum": [
+              "default",
+              "required",
+              "failed"
+            ],
+            "type": "string"
+          }
+        },
+        "type": "object"
+      },
+      "port": {
+        "description": "Dial the specified port when a ziti client connects to the service.",
+        "maximum": 65535,
+        "minimum": 0,
+        "type": "integer"
+      },
+      "protocol": {
+        "description": "Dial the specified protocol when a ziti client connects to the service.",
+        "enum": [
+          "tcp",
+          "udp",
+          "sctp"
+        ],
+        "type": "string"
+      }
+    },
+    "type": "object"
+  }
+}

tunnel/entities/host.v2.json

@@ -0,0 +1,305 @@
+{
+  "$id": "http://ziti-edge.netfoundry.io/schemas/host-v2.schema.json",
+  "definitions": {
+    "action": {
+      "additionalProperties": false,
+      "properties": {
+        "action": {
+          "pattern": "(mark (un)?healthy|increase cost [0-9]+|decrease cost [0-9]+|send event)",
+          "type": "string"
+        },
+        "consecutiveEvents": {
+          "maximum": 65535,
+          "minimum": 0,
+          "type": "integer"
+        },
+        "duration": {
+          "$ref": "#/definitions/duration"
+        },
+        "trigger": {
+          "enum": [
+            "fail",
+            "pass",
+            "change"
+          ],
+          "type": "string"
+        }
+      },
+      "required": [
+        "trigger",
+        "action"
+      ],
+      "type": "object"
+    },
+    "actionList": {
+      "items": {
+        "$ref": "#/definitions/action"
+      },
+      "maxItems": 20,
+      "minItems": 1,
+      "type": "array"
+    },
+    "address": {
+      "oneOf": [
+        {
+          "$ref": "#/definitions/ipAddress"
+        },
+        {
+          "$ref": "#/definitions/hostname"
+        }
+      ]
+    },
+    "duration": {
+      "pattern": "[0-9]+(h|m|s|ms)",
+      "type": "string"
+    },
+    "hostname": {
+      "format": "hostname",
+      "not": {
+        "$ref": "#/definitions/ipAddressFormat"
+      },
+      "type": "string"
+    },
+    "httpCheck": {
+      "additionalProperties": false,
+      "properties": {
+        "actions": {
+          "$ref": "#/definitions/actionList"
+        },
+        "body": {
+          "type": "string"
+        },
+        "expectInBody": {
+          "type": "string"
+        },
+        "expectStatus": {
+          "maximum": 599,
+          "minimum": 100,
+          "type": "integer"
+        },
+        "interval": {
+          "$ref": "#/definitions/duration"
+        },
+        "method": {
+          "$ref": "#/definitions/method"
+        },
+        "timeout": {
+          "$ref": "#/definitions/duration"
+        },
+        "url": {
+          "type": "string"
+        }
+      },
+      "required": [
+        "interval",
+        "timeout",
+        "url"
+      ],
+      "type": "object"
+    },
+    "httpCheckList": {
+      "items": {
+        "$ref": "#/definitions/httpCheck"
+      },
+      "type": "array"
+    },
+    "ipAddress": {
+      "$ref": "#/definitions/ipAddressFormat",
+      "type": "string"
+    },
+    "ipAddressFormat": {
+      "oneOf": [
+        {
+          "format": "ipv4"
+        },
+        {
+          "format": "ipv6"
+        }
+      ]
+    },
+    "method": {
+      "enum": [
+        "GET",
+        "POST",
+        "PUT",
+        "PATCH"
+      ],
+      "type": "string"
+    },
+    "portCheck": {
+      "additionalProperties": false,
+      "properties": {
+        "actions": {
+          "$ref": "#/definitions/actionList"
+        },
+        "address": {
+          "type": "string"
+        },
+        "interval": {
+          "$ref": "#/definitions/duration"
+        },
+        "timeout": {
+          "$ref": "#/definitions/duration"
+        }
+      },
+      "required": [
+        "interval",
+        "timeout",
+        "address"
+      ],
+      "type": "object"
+    },
+    "portCheckList": {
+      "items": {
+        "$ref": "#/definitions/portCheck"
+      },
+      "type": "array"
+    },
+    "terminator": {
+      "additionalProperties": false,
+      "allOf": [
+        {
+          "oneOf": [
+            {
+              "required": [
+                "protocol"
+              ]
+            },
+            {
+              "required": [
+                "dialInterceptedProtocol"
+              ]
+            }
+          ]
+        },
+        {
+          "oneOf": [
+            {
+              "required": [
+                "address"
+              ]
+            },
+            {
+              "required": [
+                "dialInterceptedAddress"
+              ]
+            }
+          ]
+        },
+        {
+          "oneOf": [
+            {
+              "required": [
+                "port"
+              ]
+            },
+            {
+              "required": [
+                "dialInterceptedPort"
+              ]
+            }
+          ]
+        }
+      ],
+      "properties": {
+        "address": {
+          "$ref": "#/definitions/address",
+          "description": "Dial the specified ip address or hostname when a ziti client connects to the service."
+        },
+        "dialInterceptedAddress": {
+          "description": "Dial the same ip address that was intercepted at the client tunneler. 'address' and 'dialInterceptedAddress' are mutually exclusive.",
+          "enum": [
+            true
+          ],
+          "type": "boolean"
+        },
+        "dialInterceptedPort": {
+          "description": "Dial the same port that was intercepted at the client tunneler. 'port' and 'dialInterceptedPort' are mutually exclusive.",
+          "enum": [
+            true
+          ],
+          "type": "boolean"
+        },
+        "dialInterceptedProtocol": {
+          "description": "Dial the same protocol that was intercepted at the client tunneler. 'protocol' and 'dialInterceptedProtocol' are mutually exclusive.",
+          "enum": [
+            true
+          ],
+          "type": "boolean"
+        },
+        "httpChecks": {
+          "$ref": "#/definitions/httpCheckList"
+        },
+        "listenOptions": {
+          "additionalProperties": false,
+          "properties": {
+            "bindUsingEdgeIdentity": {
+              "description": "Associate the hosting terminator with the name of the hosting tunneler's identity. Setting this to 'true' is equivalent to setting 'identiy=$tunneler_id.name'",
+              "type": "boolean"
+            },
+            "connectTimeout": {
+              "$ref": "#/definitions/duration"
+            },
+            "cost": {
+              "description": "defaults to 0",
+              "maximum": 65535,
+              "minimum": 0,
+              "type": "integer"
+            },
+            "identity": {
+              "description": "Associate the hosting terminator with the specified identity. '$tunneler_id.name' resolves to the name of the hosting tunneler's identity. '$tunneler_id.tag[tagName]' resolves to the value of the 'tagName' tag on the hosting tunneler's identity.",
+              "type": "string"
+            },
+            "maxConnections": {
+              "description": "defaults to 3",
+              "minimum": 1,
+              "type": "integer"
+            },
+            "precedence": {
+              "description": "defaults to 'default'",
+              "enum": [
+                "default",
+                "required",
+                "failed"
+              ],
+              "type": "string"
+            }
+          },
+          "type": "object"
+        },
+        "port": {
+          "description": "Dial the specified port when a ziti client connects to the service.",
+          "maximum": 65535,
+          "minimum": 0,
+          "type": "integer"
+        },
+        "portChecks": {
+          "$ref": "#/definitions/portCheckList"
+        },
+        "protocol": {
+          "description": "Dial the specified protocol when a ziti client connects to the service.",
+          "enum": [
+            "tcp",
+            "udp",
+            "sctp"
+          ],
+          "type": "string"
+        }
+      },
+      "type": "object"
+    },
+    "terminatorList": {
+      "items": {
+        "$ref": "#/definitions/terminator"
+      },
+      "minItems": 1,
+      "type": "array"
+    }
+  },
+  "properties": {
+    "terminators": {
+      "$ref": "#/definitions/terminatorList"
+    }
+  },
+  "type": "object"
+}

tunnel/entities/service.go

@@ -3,12 +3,20 @@ package entities
 import (
 	"fmt"
 	"github.com/openziti/edge/health"
+	"github.com/openziti/edge/tunnel"
+	"github.com/openziti/sdk-golang/ziti"
 	"github.com/openziti/sdk-golang/ziti/edge"
+	"github.com/pkg/errors"
+	"reflect"
+	"strconv"
+	"time"
 )
 
 const (
 	ClientConfigV1 = "ziti-tunneler-client.v1"
 	ServerConfigV1 = "ziti-tunneler-server.v1"
+	HostConfigV1   = "host.v1"
+	HostConfigV2   = "host.v2"
 )
 
 type ServiceConfig struct {
@@ -19,13 +27,168 @@ type ServiceConfig struct {
 	HttpChecks []*health.HttpCheckDefinition
 }
 
+func (self *ServiceConfig) GetPortChecks() []*health.PortCheckDefinition {
+	return self.PortChecks
+}
+
+func (self *ServiceConfig) GetHttpChecks() []*health.HttpCheckDefinition {
+	return self.HttpChecks
+}
+
 func (s *ServiceConfig) String() string {
 	return fmt.Sprintf("%v:%v:%v", s.Protocol, s.Hostname, s.Port)
 }
 
+type HostV1ListenOptions struct {
+	BindUsingEdgeIdentity bool
+	ConnectTimeoutSeconds *int
+	Cost                  *int
+	Identity              string
+	MaxConnections        int
+	Precedence            *string
+}
+
+type HostV1Config struct {
+	Protocol                string
+	DialInterceptedProtocol bool
+	Address                 string
+	DialInterceptedAddress  bool
+	Port                    int
+	DialInterceptedPort     bool
+
+	PortChecks []*health.PortCheckDefinition
+	HttpChecks []*health.HttpCheckDefinition
+
+	ListenOptions *HostV1ListenOptions
+}
+
+func (self *HostV1Config) ToHostV2Config() *HostV2Config {
+	terminator := &HostV2Terminator{
+		Protocol:                self.Protocol,
+		DialInterceptedProtocol: self.DialInterceptedProtocol,
+		Address:                 self.Address,
+		DialInterceptedAddress:  self.DialInterceptedAddress,
+		Port:                    self.Port,
+		DialInterceptedPort:     self.DialInterceptedPort,
+		PortChecks:              self.PortChecks,
+		HttpChecks:              self.HttpChecks,
+	}
+
+	if self.ListenOptions != nil {
+		var timeout *time.Duration
+		if self.ListenOptions.ConnectTimeoutSeconds != nil {
+			val := time.Duration(*self.ListenOptions.ConnectTimeoutSeconds) * time.Second
+			timeout = &val
+		}
+		terminator.ListenOptions = &HostV2ListenOptions{
+			BindUsingEdgeIdentity: self.ListenOptions.BindUsingEdgeIdentity,
+			ConnectTimeout:        timeout,
+			Cost:                  self.ListenOptions.Cost,
+			Identity:              self.ListenOptions.Identity,
+			MaxConnections:        self.ListenOptions.MaxConnections,
+			Precedence:            self.ListenOptions.Precedence,
+		}
+	}
+
+	return &HostV2Config{
+		Terminators: []*HostV2Terminator{
+			terminator,
+		},
+	}
+}
+
+type HostV2ListenOptions struct {
+	BindUsingEdgeIdentity bool
+	ConnectTimeout        *time.Duration
+	Cost                  *int
+	Identity              string
+	MaxConnections        int
+	Precedence            *string
+}
+
+type HostV2Terminator struct {
+	Protocol                string
+	DialInterceptedProtocol bool
+	Address                 string
+	DialInterceptedAddress  bool
+	Port                    int
+	DialInterceptedPort     bool
+
+	PortChecks []*health.PortCheckDefinition
+	HttpChecks []*health.HttpCheckDefinition
+
+	ListenOptions *HostV2ListenOptions
+}
+
+func (self *HostV2Terminator) SetListenOptions(options *ziti.ListenOptions) {
+	if self.ListenOptions != nil {
+		if self.ListenOptions.Cost != nil {
+			options.Cost = uint16(*self.ListenOptions.Cost)
+		}
+		if self.ListenOptions.Precedence != nil {
+			options.Precedence = ziti.GetPrecedenceForLabel(*self.ListenOptions.Precedence)
+		}
+	}
+}
+
+func (self *HostV2Terminator) GetDialTimeout(defaultTimeout time.Duration) time.Duration {
+	if self.ListenOptions != nil && self.ListenOptions.ConnectTimeout != nil {
+		return *self.ListenOptions.ConnectTimeout
+	}
+	return defaultTimeout
+}
+
+func (self *HostV2Terminator) GetPortChecks() []*health.PortCheckDefinition {
+	return self.PortChecks
+}
+
+func (self *HostV2Terminator) GetHttpChecks() []*health.HttpCheckDefinition {
+	return self.HttpChecks
+}
+
+func (self *HostV2Terminator) getValue(options map[string]interface{}, key string) (string, error) {
+	val, ok := options[key]
+	if !ok {
+		return "", errors.Errorf("%v required but not provided", key)
+	}
+	result, ok := val.(string)
+	if !ok {
+		return "", errors.Errorf("%v required and present but not a string. val: %v, type: %v", key, val, reflect.TypeOf(val))
+	}
+	return result, nil
+}
+
+func (self *HostV2Terminator) GetProtocol(options map[string]interface{}) (string, error) {
+	if self.DialInterceptedProtocol {
+		return self.getValue(options, tunnel.InterceptedProtocolKey)
+	}
+	return self.Protocol, nil
+}
+
+func (self *HostV2Terminator) GetAddress(options map[string]interface{}) (string, error) {
+	if self.DialInterceptedAddress {
+		return self.getValue(options, tunnel.InterceptedIpKey)
+	}
+	return self.Address, nil
+}
+
+func (self *HostV2Terminator) GetPort(options map[string]interface{}) (string, error) {
+	if self.DialInterceptedPort {
+		return self.getValue(options, tunnel.InterceptedPortKey)
+	}
+	return strconv.Itoa(self.Port), nil
+}
+
+type HostV2Config struct {
+	Terminators []*HostV2Terminator
+}
+
 type Service struct {
 	edge.Service
 	ClientConfig *ServiceConfig
+
 	ServerConfig *ServiceConfig
+	HostV2Config *HostV2Config
+
 	StopHostHook func()
 }

tunnel/entities/ziti-tunneler-server-v1.json

@@ -0,0 +1,161 @@
+{
+  "schema": {
+    "$id": "http://edge.openziti.org/schemas/ziti-tunneler-server.v1.config.json",
+    "additionalProperties": false,
+    "definitions": {
+      "action": {
+        "additionalProperties": false,
+        "properties": {
+          "action": {
+            "pattern": "(mark (un)?healthy|increase cost [0-9]+|decrease cost [0-9]+)",
+            "type": "string"
+          },
+          "consecutiveEvents": {
+            "maximum": 65535,
+            "minimum": 0,
+            "type": "integer"
+          },
+          "duration": {
+            "$ref": "#/definitions/duration"
+          },
+          "trigger": {
+            "enum": [
+              "fail",
+              "pass"
+            ],
+            "type": "string"
+          }
+        },
+        "required": [
+          "trigger",
+          "action"
+        ],
+        "type": "object"
+      },
+      "actionList": {
+        "items": {
+          "$ref": "#/definitions/action"
+        },
+        "maxItems": 20,
+        "minItems": 1,
+        "type": "array"
+      },
+      "duration": {
+        "pattern": "[0-9]+(h|m|s|ms)",
+        "type": "string"
+      },
+      "httpCheck": {
+        "additionalProperties": false,
+        "properties": {
+          "actions": {
+            "$ref": "#/definitions/actionList"
+          },
+          "body": {
+            "type": "string"
+          },
+          "expectInBody": {
+            "type": "string"
+          },
+          "expectStatus": {
+            "maximum": 599,
+            "minimum": 100,
+            "type": "integer"
+          },
+          "interval": {
+            "$ref": "#/definitions/duration"
+          },
+          "method": {
+            "$ref": "#/definitions/method"
+          },
+          "timeout": {
+            "$ref": "#/definitions/duration"
+          },
+          "url": {
+            "type": "string"
+          }
+        },
+        "required": [
+          "interval",
+          "timeout",
+          "url"
+        ],
+        "type": "object"
+      },
+      "httpCheckList": {
+        "items": {
+          "$ref": "#/definitions/httpCheck"
+        },
+        "type": "array"
+      },
+      "method": {
+        "enum": [
+          "GET",
+          "POST",
+          "PUT",
+          "PATCH"
+        ],
+        "type": "string"
+      },
+      "portCheck": {
+        "additionalProperties": false,
+        "properties": {
+          "actions": {
+            "$ref": "#/definitions/actionList"
+          },
+          "address": {
+            "type": "string"
+          },
+          "interval": {
+            "$ref": "#/definitions/duration"
+          },
+          "timeout": {
+            "$ref": "#/definitions/duration"
+          }
+        },
+        "required": [
+          "interval",
+          "timeout",
+          "address"
+        ],
+        "type": "object"
+      },
+      "portCheckList": {
+        "items": {
+          "$ref": "#/definitions/portCheck"
+        },
+        "type": "array"
+      }
+    },
+    "properties": {
+      "hostname": {
+        "type": "string"
+      },
+      "httpChecks": {
+        "$ref": "#/definitions/httpCheckList"
+      },
+      "port": {
+        "maximum": 65535,
+        "minimum": 0,
+        "type": "integer"
+      },
+      "portChecks": {
+        "$ref": "#/definitions/portCheckList"
+      },
+      "protocol": {
+        "enum": [
+          "tcp",
+          "udp"
+        ],
+        "type": [
+          "string",
+          "null"
+        ]
+      }
+    },
+    "required": [
+      "hostname",
+      "port"
+    ],
+    "type": "object"
+  }
+}

tunnel/intercept/hosting.go

@@ -0,0 +1,178 @@
+package intercept
+
+import (
+	"github.com/openziti/edge/health"
+	"github.com/openziti/edge/tunnel"
+	"github.com/openziti/edge/tunnel/entities"
+	"github.com/openziti/sdk-golang/ziti"
+	"github.com/openziti/sdk-golang/ziti/edge"
+	"net"
+	"strconv"
+	"strings"
+	"time"
+)
+
+const (
+	defaultDialTimeout = 5 * time.Second
+)
+
+type healthChecksProvider interface {
+	GetPortChecks() []*health.PortCheckDefinition
+	GetHttpChecks() []*health.HttpCheckDefinition
+}
+
+type serviceConfiguration interface {
+	healthChecksProvider
+	SetListenOptions(options *ziti.ListenOptions)
+	GetDialTimeout(defaultTimeout time.Duration) time.Duration
+	GetProtocol(options map[string]interface{}) (string, error)
+	GetAddress(options map[string]interface{}) (string, error)
+	GetPort(options map[string]interface{}) (string, error)
+}
+
+func createHostingContexts(service *entities.Service, identity *edge.CurrentIdentity) []tunnel.HostingContext {
+	if service.HostV2Config != nil {
+		var result []tunnel.HostingContext
+		for _, t := range service.HostV2Config.Terminators {
+			context := newDefaultHostingContext(identity, service, t)
+			result = append(result, context)
+		}
+		return result
+	}
+
+	context := &legacyHostingContext{
+		baseHostingContext{
+			service:     service,
+			options:     getDefaultOptions(identity),
+			dialTimeout: defaultDialTimeout,
+		},
+	}
+	return []tunnel.HostingContext{context}
+}
+
+type baseHostingContext struct {
+	service     *entities.Service
+	options     *ziti.ListenOptions
+	dialTimeout time.Duration
+	onClose     func()
+}
+
+func (self *baseHostingContext) ServiceName() string {
+	return self.service.Name
+}
+
+func (self *baseHostingContext) ListenOptions() *ziti.ListenOptions {
+	return self.options
+}
+
+func (self *baseHostingContext) dialAddress(options map[string]interface{}, protocol string, address string) (net.Conn, error) {
+	var sourceIp string
+	if val, ok := options[tunnel.SourceIpKey]; ok {
+		sourceIp = val.(string)
+	}
+
+	if sourceIp != "" {
+		dialer := net.Dialer{
+			LocalAddr: &net.TCPAddr{IP: net.ParseIP(sourceIp)},
+			Timeout:   self.dialTimeout,
+		}
+		return dialer.Dial(protocol, address)
+	}
+
+	return net.DialTimeout(protocol, address, self.dialTimeout)
+}
+
+func (self *baseHostingContext) SupportHalfClose() bool {
+	return !strings.Contains(self.service.ServerConfig.Protocol, "udp")
+}
+
+func (self *baseHostingContext) SetCloseCallback(f func()) {
+	self.onClose = f
+}
+
+func (self *baseHostingContext) OnClose() {
+	if self.onClose != nil {
+		self.onClose()
+	}
+}
+
+func (self *baseHostingContext) getHealthChecks(provider healthChecksProvider) []health.CheckDefinition {
+	var checkDefinitions []health.CheckDefinition
+
+	for _, checkDef := range provider.GetPortChecks() {
+		checkDefinitions = append(checkDefinitions, checkDef)
+	}
+
+	for _, checkDef := range provider.GetHttpChecks() {
+		checkDefinitions = append(checkDefinitions, checkDef)
+	}
+
+	return checkDefinitions
+}
+
+func (self *baseHostingContext) GetInitialHealthState() (ziti.Precedence, uint16) {
+	return self.options.Precedence, self.options.Cost
+}
+
+type legacyHostingContext struct {
+	baseHostingContext
+}
+
+func (self *legacyHostingContext) GetHealthChecks() []health.CheckDefinition {
+	return self.getHealthChecks(self.service.ServerConfig)
+}
+
+func (self *legacyHostingContext) Dial(options map[string]interface{}) (net.Conn, error) {
+	config := self.service.ServerConfig
+	return self.dialAddress(options, config.Protocol, config.Hostname+":"+strconv.Itoa(config.Port))
+}
+
+func newDefaultHostingContext(identity *edge.CurrentIdentity, service *entities.Service, config serviceConfiguration) *hostV2HostingContext {
+	options := getDefaultOptions(identity)
+	config.SetListenOptions(options)
+
+	return &hostV2HostingContext{
+		baseHostingContext: baseHostingContext{
+			service:     service,
+			options:     options,
+			dialTimeout: config.GetDialTimeout(5 * time.Second),
+		},
+		config: config,
+	}
+}
+
+type hostV2HostingContext struct {
+	baseHostingContext
+	config serviceConfiguration
+}
+
+func (self *hostV2HostingContext) GetHealthChecks() []health.CheckDefinition {
+	return self.getHealthChecks(self.config)
+}
+
+func (self *hostV2HostingContext) Dial(options map[string]interface{}) (net.Conn, error) {
+	protocol, err := self.config.GetProtocol(options)
+	if err != nil {
+		return nil, err
+	}
+
+	address, err := self.config.GetAddress(options)
+	if err != nil {
+		return nil, err
+	}
+
+	port, err := self.config.GetPort(options)
+	if err != nil {
+		return nil, err
+	}
+
+	return self.dialAddress(options, protocol, address+":"+port)
+}
+
+func getDefaultOptions(identity *edge.CurrentIdentity) *ziti.ListenOptions {
+	options := ziti.DefaultListenOptions()
+	options.ManualStart = true
+	options.Precedence = ziti.GetPrecedenceForLabel(identity.DefaultHostingPrecedence)
+	options.Cost = identity.DefaultHostingCost
+	return options
+}

tunnel/intercept/svcpoll.go

@@ -25,12 +25,10 @@ import (
 	"github.com/openziti/foundation/util/stringz"
 	"github.com/openziti/sdk-golang/ziti"
 	"github.com/openziti/sdk-golang/ziti/edge"
-	log "github.com/sirupsen/logrus"
-	"net"
+	logrus "github.com/sirupsen/logrus"
 	"os"
 	"os/signal"
 	"strconv"
-	"strings"
 	"sync"
 	"syscall"
 )
@@ -60,7 +58,7 @@ func (self *ServiceListener) WaitForShutdown() {
 	signal.Notify(sig, syscall.SIGINT, syscall.SIGKILL, syscall.SIGTERM)
 
 	for s := range sig {
-		log.Debugf("caught signal %v", s)
+		logrus.Debugf("caught signal %v", s)
 		break
 	}
 
@@ -80,13 +78,20 @@ func (self *ServiceListener) HandleProviderReady(provider tunnel.FabricProvider)
 func (self *ServiceListener) HandleServicesChange(eventType ziti.ServiceEventType, service *edge.Service) {
 	tunnelerService := &entities.Service{Service: *service}
 
+	log := logrus.WithField("service", service.Name)
+
 	switch eventType {
 	case ziti.ServiceAdded:
+		log.Info("adding service")
 		self.addService(tunnelerService)
 	case ziti.ServiceRemoved:
+		log.Info("removing service")
 		self.removeService(tunnelerService)
 	case ziti.ServiceChanged:
+		log.Info("updating service: removing old service")
 		self.removeService(tunnelerService)
+
+		log.Info("updating service: adding new service")
 		self.addService(tunnelerService)
 	default:
 		pfxlog.Logger().Errorf("unhandled service change event type: %v", eventType)
@@ -116,11 +121,27 @@ func (self *ServiceListener) addService(svc *entities.Service) {
 	}
 
 	if stringz.Contains(svc.Permissions, "Bind") {
-		serverConfig := &entities.ServiceConfig{}
-		found, err := svc.GetConfigOfType(entities.ServerConfigV1, serverConfig)
+		hostV2config := &entities.HostV2Config{}
+		found, err := svc.GetConfigOfType(entities.HostConfigV2, hostV2config)
+		if found {
+			svc.HostV2Config = hostV2config
+		}
+
+		if !found {
+			hostV1config := &entities.HostV1Config{}
+			if found, err = svc.GetConfigOfType(entities.HostConfigV1, hostV1config); found {
+				svc.HostV2Config = hostV1config.ToHostV2Config()
+			}
+		}
+
+		if !found {
+			serverConfig := &entities.ServiceConfig{}
+			if found, err = svc.GetConfigOfType(entities.ServerConfigV1, serverConfig); found {
+				svc.ServerConfig = serverConfig
+			}
+		}
 
 		if found && err == nil {
-			svc.ServerConfig = serverConfig
 			log.Infof("Hosting newly available service %s", svc.Name)
 			go self.host(svc)
 		} else if !found {
@@ -176,83 +197,40 @@ func (self *ServiceListener) host(svc *entities.Service) {
 		return
 	}
 
-	options := ziti.DefaultListenOptions()
-	options.ManualStart = true
-	options.Precedence = ziti.GetPrecedenceForLabel(currentIdentity.DefaultHostingPrecedence)
-	options.Cost = currentIdentity.DefaultHostingCost
+	hostContexts := createHostingContexts(svc, currentIdentity)
 
-	context := &hostingContext{
-		service: svc,
-		options: options,
-		onClose: func() {
-			self.healthCheckMgr.UnregisterServiceChecks(svc.Id)
-		},
-	}
-
-	hostControl, err := self.provider.HostService(context)
-	if err != nil {
-		logger.WithError(err).WithField("service", svc.Name).Errorf("error listening for service")
-		return
-	}
+	var hostControls []tunnel.HostControl
 
 	stopHook := func() {
-		_ = hostControl.Close()
+		for _, hostControl := range hostControls {
+			_ = hostControl.Close()
+		}
 		self.healthCheckMgr.UnregisterServiceChecks(svc.Id)
 	}
-
 	svc.StopHostHook = stopHook
 
-	if err := self.setupHealthChecks(hostControl, svc, currentIdentity); err != nil {
-		logger.WithError(err).Error("error setting up health checks")
-		stopHook()
-		return
+	for idx, hostContext := range hostContexts {
+		hostControl, err := self.provider.HostService(hostContext)
+		if err != nil {
+			logger.WithError(err).WithField("service", svc.Name).Errorf("error listening for service")
+			return
+		}
+
+		context := strconv.Itoa(idx)
+
+		hostContext.SetCloseCallback(func() {
+			self.healthCheckMgr.UnregisterServiceContextChecks(svc.Name, context)
+		})
+
+		hostControls = append(hostControls, hostControl)
+
+		precedence, cost := hostContext.GetInitialHealthState()
+		serviceState := health.NewServiceStateWithContext(svc.Name, context, precedence, cost, hostControl)
+
+		if err := self.healthCheckMgr.RegisterServiceChecks(serviceState, hostContext.GetHealthChecks()); err != nil {
+			logger.WithError(err).Error("error setting up health checks")
+			hostContext.OnClose()
+			return
+		}
 	}
 }
-
-func (self *ServiceListener) setupHealthChecks(serviceUpdater health.ServiceUpdater, service *entities.Service, identity *edge.CurrentIdentity) error {
-	precedence := ziti.GetPrecedenceForLabel(identity.DefaultHostingPrecedence)
-	serviceState := health.NewServiceState(service.Name, precedence, identity.DefaultHostingCost, serviceUpdater)
-
-	var checkDefinitions []health.CheckDefinition
-
-	for _, checkDef := range service.ServerConfig.PortChecks {
-		checkDefinitions = append(checkDefinitions, checkDef)
-	}
-
-	for _, checkDef := range service.ServerConfig.HttpChecks {
-		checkDefinitions = append(checkDefinitions, checkDef)
-	}
-
-	if len(checkDefinitions) > 0 {
-		return self.healthCheckMgr.RegisterServiceChecks(serviceState, checkDefinitions)
-	}
-
-	return nil
-}
-
-type hostingContext struct {
-	service *entities.Service
-	options *ziti.ListenOptions
-	onClose func()
-}
-
-func (self *hostingContext) ServiceName() string {
-	return self.service.Name
-}
-
-func (self *hostingContext) ListenOptions() *ziti.ListenOptions {
-	return self.options
-}
-
-func (self *hostingContext) Dial(options map[string]interface{}) (net.Conn, error) {
-	config := self.service.ServerConfig
-	return net.Dial(config.Protocol, config.Hostname+":"+strconv.Itoa(config.Port))
-}
-
-func (self *hostingContext) SupportHalfClose() bool {
-	return !strings.Contains(self.service.ServerConfig.Protocol, "udp")
-}
-
-func (self *hostingContext) OnClose() {
-	self.onClose()
-}

tunnel/provider.go

@@ -2,6 +2,7 @@ package tunnel
 
 import (
 	"encoding/json"
+	"github.com/openziti/edge/health"
 	"github.com/openziti/sdk-golang/ziti"
 	"github.com/openziti/sdk-golang/ziti/edge"
 	"github.com/sirupsen/logrus"
@@ -14,7 +15,10 @@ type HostingContext interface {
 	ListenOptions() *ziti.ListenOptions
 	Dial(options map[string]interface{}) (net.Conn, error)
 	SupportHalfClose() bool
+	GetHealthChecks() []health.CheckDefinition
+	GetInitialHealthState() (ziti.Precedence, uint16)
 	OnClose()
+	SetCloseCallback(func())
 }
 
 type HostControl interface {
@@ -22,6 +26,7 @@ type HostControl interface {
 	UpdateCost(cost uint16) error
 	UpdatePrecedence(precedence edge.Precedence) error
 	UpdateCostAndPrecedence(cost uint16, precedence edge.Precedence) error
+	SendHealthEvent(pass bool) error
 }
 
 type FabricProvider interface {