Add metrics on proxy service created (#144)

* Add metrics on proxy service created

* Fix lint

Author: temporal-nick

metrics/prometheus_defs.go

@@ -29,8 +29,11 @@ var (
 
 	// /proxy/proxy.go
 
-	GRPCServerMetrics = GetStandardGRPCInterceptor("direction")
-	ProxyStartCount   = DefaultCounter("proxy_start_count", "Emitted once per startup")
+	GRPCServerMetrics     = GetStandardGRPCInterceptor("direction")
+	ProxyStartCount       = DefaultCounter("proxy_start_count", "Emitted once on Go process start")
+	ProxyServiceCreated   = DefaultCounterVec("proxy_service_created", "Emitted once per service start", "direction")
+	ProxyServiceStopped   = DefaultCounterVec("proxy_service_stopped", "Emitted on service shutdown", "direction")
+	ProxyServiceRestarted = DefaultCounterVec("proxy_service_restarted", "Emitted on service shutdown", "direction")
 
 	// /transport/grpc.go
 	// Gratuitous hack: Until https://github.com/grpc-ecosystem/go-grpc-middleware/issues/783 is addressed,
@@ -84,6 +87,9 @@ func init() {
 
 	prometheus.MustRegister(GRPCServerMetrics)
 	prometheus.MustRegister(ProxyStartCount)
+	prometheus.MustRegister(ProxyServiceCreated)
+	prometheus.MustRegister(ProxyServiceStopped)
+	prometheus.MustRegister(ProxyServiceRestarted)
 
 	prometheus.MustRegister(GRPCOutboundClientMetrics)
 	prometheus.MustRegister(GRPCInboundClientMetrics)

proxy/proxy.go

@@ -29,6 +29,7 @@ type (
 		logger       log.Logger
 		server       *TemporalAPIServer
 		transManager *transport.TransportManager
+		metricLabels prometheus.Labels
 		shutDownCh   chan struct{}
 	}
 
@@ -184,10 +185,11 @@ func (ps *ProxyServer) start() error {
 
 	go func() {
 		for {
+			metrics.ProxyServiceCreated.With(ps.metricLabels).Inc()
 			// If using mux transport underneath, Open call will be blocked until
 			// underlying connection is established.
 			// Also note: GRPC requires the client interceptors (like metrics) to be defined on the transport, not on the client.
-			clientTransport, err := ps.transManager.OpenClient(prometheus.Labels{"direction": ps.opts.directionLabel()}, clientConfig)
+			clientTransport, err := ps.transManager.OpenClient(clientConfig)
 			if err != nil {
 				ps.logger.Error("Open client transport is failed", tag.Error(err))
 				return
@@ -215,10 +217,12 @@ func (ps *ProxyServer) start() error {
 
 			select {
 			case <-ps.shutDownCh:
+				metrics.ProxyServiceStopped.With(ps.metricLabels).Inc()
 				ps.stopServer()
 				return
 			case <-retryCh:
 				// If any closable transport is closed, try to restart the proxy server.
+				metrics.ProxyServiceRestarted.With(ps.metricLabels).Inc()
 				ps.stopServer()
 			}
 		}
@@ -243,6 +247,7 @@ func newProxyServer(
 		opts:         opts,
 		transManager: transManager,
 		logger:       logger,
+		metricLabels: prometheus.Labels{"direction": opts.directionLabel()},
 		shutDownCh:   make(chan struct{}),
 	}
 }

proxy/test/echo_server.go

@@ -6,7 +6,6 @@ import (
 	"time"
 
 	"github.com/gogo/status"
-	"github.com/prometheus/client_golang/prometheus"
 	"go.temporal.io/api/workflowservice/v1"
 	"go.temporal.io/server/api/adminservice/v1"
 	replicationpb "go.temporal.io/server/api/replication/v1"
@@ -149,7 +148,7 @@ func newEchoServer(
 		logger.Fatal("Failed to create server transport", tag.Error(err))
 	}
 
-	clientTransport, err := tm.OpenClient(prometheus.Labels{}, clientConfig)
+	clientTransport, err := tm.OpenClient(clientConfig)
 	if err != nil {
 		logger.Fatal("Failed to create client transport", tag.Error(err))
 	}

transport/transport.go

@@ -4,7 +4,6 @@ import (
 	"fmt"
 
 	grpcprom "github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus"
-	prometheus "github.com/prometheus/client_golang/prometheus"
 	"go.temporal.io/server/common/log"
 	"google.golang.org/grpc"
 
@@ -66,7 +65,7 @@ func (tm *TransportManager) IsMuxActive(name string) bool {
 	return tm.muxConnManagers[name].status.Load() == int32(statusStarted)
 }
 
-func (tm *TransportManager) OpenClient(metricLabels prometheus.Labels, clientConfig config.ProxyClientConfig) (ClientTransport, error) {
+func (tm *TransportManager) OpenClient(clientConfig config.ProxyClientConfig) (ClientTransport, error) {
 	if clientConfig.Type == config.MuxTransport {
 		return tm.openMuxTransport(clientConfig.MuxTransportName)
 	}