add HTTP and GRPC health check endpoints (#1258)

* add HTTP and GRPC health check endpoints

Signed-off-by: Bob Callaway <[email protected]>

* update prometheus check, add readiness probe, add nosec

Signed-off-by: Bob Callaway <[email protected]>

* bump k8s to v1.24

Signed-off-by: Bob Callaway <[email protected]>

* export struct to quiet lint, update issuers with cluster.local, expose grpc

Signed-off-by: Bob Callaway <[email protected]>

* bump setup-kind, remove grpc readiness

Signed-off-by: Bob Callaway <[email protected]>

* re-enable readiness probe for grpc

Signed-off-by: Bob Callaway <[email protected]>

---------

Signed-off-by: Bob Callaway <[email protected]>

Author: bobcallaway

.github/workflows/verify-k8s.yml

@@ -73,9 +73,9 @@ jobs:
       - uses: ko-build/setup-ko@ace48d793556083a76f1e3e6068850c1f4a369aa # v0.6
 
       - name: Setup Cluster
-        uses: chainguard-dev/actions/setup-kind@7d1eb557f464d97e5fe5177807a9226141eb9308 # main
+        uses: chainguard-dev/actions/setup-kind@f5a6616ce43b6ffabeddb87480a13721fffb3588 # main
         with:
-          k8s-version: v1.22.x
+          k8s-version: 1.24.x
           registry-authority: ${{ env.REGISTRY_NAME }}:${{ env.REGISTRY_PORT }}
 
       - name: Generate temporary CA files
@@ -121,6 +121,7 @@ jobs:
             server.yaml: |-
               host: 0.0.0.0
               port: 5555
+              grpc-port: 5554
               ca: fileca
               fileca-cert: /etc/fulcio-secret/cert.pem
               fileca-key: /etc/fulcio-secret/key.pem

cmd/app/grpc.go

@@ -42,6 +42,7 @@ import (
 	"github.com/spf13/viper"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials"
+	health "google.golang.org/grpc/health/grpc_health_v1"
 )
 
 const (
@@ -180,6 +181,8 @@ func createGRPCServer(cfg *config.FulcioConfig, ctClient *ctclient.LogClient, ba
 	myServer := grpc.NewServer(serverOpts...)
 
 	grpcCAServer := server.NewGRPCCAServer(ctClient, baseca, ip)
+
+	health.RegisterHealthServer(myServer, grpcCAServer)
 	// Register your gRPC service implementations.
 	gw.RegisterCAServer(myServer, grpcCAServer)
 
@@ -239,6 +242,10 @@ func (g *grpcServer) startUnixListener() {
 	}()
 }
 
+func (g *grpcServer) ExposesGRPCTLS() bool {
+	return viper.IsSet("grpc-tls-certificate") && viper.IsSet("grpc-tls-key")
+}
+
 func createLegacyGRPCServer(cfg *config.FulcioConfig, v2Server gw.CAServer) (*grpcServer, error) {
 	logger, opts := log.SetupGRPCLogging()
 

cmd/app/http.go

@@ -17,6 +17,7 @@ package app
 
 import (
 	"context"
+	"crypto/tls"
 	"errors"
 	"fmt"
 	"net/http"
@@ -32,7 +33,9 @@ import (
 	"github.com/sigstore/fulcio/pkg/log"
 	"github.com/sigstore/fulcio/pkg/server"
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/credentials/insecure"
+	health "google.golang.org/grpc/health/grpc_health_v1"
 	"google.golang.org/grpc/metadata"
 	"google.golang.org/protobuf/proto"
 )
@@ -48,10 +51,22 @@ func extractOIDCTokenFromAuthHeader(_ context.Context, req *http.Request) metada
 }
 
 func createHTTPServer(ctx context.Context, serverEndpoint string, grpcServer, legacyGRPCServer *grpcServer) httpServer {
+	opts := []grpc.DialOption{}
+	if grpcServer.ExposesGRPCTLS() {
+		/* #nosec G402 */ // InsecureSkipVerify is only used for the HTTP server to call the TLS-enabled grpc endpoint.
+		opts = append(opts, grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{InsecureSkipVerify: true})))
+	} else {
+		opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials()))
+	}
+	cc, err := grpc.Dial(grpcServer.grpcServerEndpoint, opts...)
+	if err != nil {
+		log.Logger.Fatal(err)
+	}
+
 	mux := runtime.NewServeMux(runtime.WithMetadata(extractOIDCTokenFromAuthHeader),
-		runtime.WithForwardResponseOption(setResponseCodeModifier))
+		runtime.WithForwardResponseOption(setResponseCodeModifier),
+		runtime.WithHealthzEndpoint(health.NewHealthClient(cc)))
 
-	opts := []grpc.DialOption{grpc.WithTransportCredentials(insecure.NewCredentials())}
 	if err := gw.RegisterCAHandlerFromEndpoint(ctx, mux, grpcServer.grpcServerEndpoint, opts); err != nil {
 		log.Logger.Fatal(err)
 	}

config/deployment.yaml

@@ -65,6 +65,15 @@ spec:
           readOnly: true
         - name: oidc-info
           mountPath: /var/run/fulcio
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 5555
+          initialDelaySeconds: 5
+        readinessProbe:
+          grpc:
+            port: 5554
+          initialDelaySeconds: 5
         resources:
           requests:
             memory: "1G"

docker-compose.yml

@@ -39,7 +39,7 @@ services:
       - ~/.config/gcloud:/root/.config/gcloud/:z # for GCP authentication
       - ./config/config.jsn:/etc/fulcio-config/config.json:z
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:5555/ping"]
+      test: ["CMD", "curl", "-f", "http://localhost:5555/healthz"]
       interval: 10s
       timeout: 3s
       retries: 3
@@ -62,7 +62,7 @@ services:
     volumes:
       - ./config/dex:/etc/config/:ro
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:8888/auth/healthz"]
+      test: ["CMD", "wget", "-O", "/dev/null", "http://localhost:8888/auth/healthz"]
       interval: 10s
       timeout: 3s
       retries: 3

pkg/server/grpc_server.go

@@ -32,18 +32,21 @@ import (
 	"github.com/sigstore/fulcio/pkg/log"
 	"github.com/sigstore/sigstore/pkg/cryptoutils"
 	"google.golang.org/grpc/codes"
+	health "google.golang.org/grpc/health/grpc_health_v1"
 	"google.golang.org/grpc/metadata"
+	"google.golang.org/grpc/status"
 )
 
-type grpcCAServer struct {
+type GRPCCAServer struct {
 	fulciogrpc.UnimplementedCAServer
+	health.HealthServer
 	ct *ctclient.LogClient
 	ca certauth.CertificateAuthority
 	identity.IssuerPool
 }
 
-func NewGRPCCAServer(ct *ctclient.LogClient, ca certauth.CertificateAuthority, ip identity.IssuerPool) fulciogrpc.CAServer {
-	return &grpcCAServer{
+func NewGRPCCAServer(ct *ctclient.LogClient, ca certauth.CertificateAuthority, ip identity.IssuerPool) *GRPCCAServer {
+	return &GRPCCAServer{
 		ct:         ct,
 		ca:         ca,
 		IssuerPool: ip,
@@ -54,7 +57,7 @@ const (
 	MetadataOIDCTokenKey = "oidcidentitytoken"
 )
 
-func (g *grpcCAServer) CreateSigningCertificate(ctx context.Context, request *fulciogrpc.CreateSigningCertificateRequest) (*fulciogrpc.SigningCertificate, error) {
+func (g *GRPCCAServer) CreateSigningCertificate(ctx context.Context, request *fulciogrpc.CreateSigningCertificateRequest) (*fulciogrpc.SigningCertificate, error) {
 	logger := log.ContextLogger(ctx)
 
 	// OIDC token either is passed in gRPC field or was extracted from HTTP headers
@@ -228,7 +231,7 @@ func (g *grpcCAServer) CreateSigningCertificate(ctx context.Context, request *fu
 	return result, nil
 }
 
-func (g *grpcCAServer) GetTrustBundle(ctx context.Context, _ *fulciogrpc.GetTrustBundleRequest) (*fulciogrpc.TrustBundle, error) {
+func (g *GRPCCAServer) GetTrustBundle(ctx context.Context, _ *fulciogrpc.GetTrustBundleRequest) (*fulciogrpc.TrustBundle, error) {
 	trustBundle, err := g.ca.TrustBundle(ctx)
 	if err != nil {
 		return nil, handleFulcioGRPCError(ctx, codes.Internal, err, retrieveTrustBundleCAError)
@@ -252,7 +255,7 @@ func (g *grpcCAServer) GetTrustBundle(ctx context.Context, _ *fulciogrpc.GetTrus
 	return resp, nil
 }
 
-func (g *grpcCAServer) GetConfiguration(ctx context.Context, _ *fulciogrpc.GetConfigurationRequest) (*fulciogrpc.Configuration, error) {
+func (g *GRPCCAServer) GetConfiguration(ctx context.Context, _ *fulciogrpc.GetConfigurationRequest) (*fulciogrpc.Configuration, error) {
 	cfg := config.FromContext(ctx)
 	if cfg == nil {
 		err := errors.New("configuration not loaded")
@@ -263,3 +266,11 @@ func (g *grpcCAServer) GetConfiguration(ctx context.Context, _ *fulciogrpc.GetCo
 		Issuers: cfg.ToIssuers(),
 	}, nil
 }
+
+func (g *GRPCCAServer) Check(_ context.Context, _ *health.HealthCheckRequest) (*health.HealthCheckResponse, error) {
+	return &health.HealthCheckResponse{Status: health.HealthCheckResponse_SERVING}, nil
+}
+
+func (g *GRPCCAServer) Watch(_ *health.HealthCheckRequest, _ health.Health_WatchServer) error {
+	return status.Error(codes.Unimplemented, "unimplemented")
+}

test/prometheus/main.go

@@ -75,31 +75,27 @@ func checkLatency(latency *dto.MetricFamily) error {
 	if *latency.Type != *dto.MetricType_HISTOGRAM.Enum() {
 		return fmt.Errorf("Wrong type, wanted %+v, got: %+v", dto.MetricType_HISTOGRAM.Enum(), latency.Type)
 	}
-	if len(latency.Metric) != 1 {
-		return fmt.Errorf("Got multiple entries, or none for metric, wanted one, got: %+v", latency.Metric)
-	}
-	// Make sure there's a 'post' and it's a 201.
-	var code string
-	var method string
-	for _, value := range latency.Metric[0].Label {
-		if *value.Name == "code" {
-			code = *value.Value
+
+	for _, metric := range latency.Metric {
+		var code string
+		var method string
+		for _, value := range metric.Label {
+			if *value.Name == "code" {
+				code = *value.Value
+			}
+			if *value.Name == "method" {
+				method = *value.Value
+			}
 		}
-		if *value.Name == "method" {
-			method = *value.Value
+		if code == "201" && method == "post" {
+			if *metric.Histogram.SampleCount != 1 {
+				return fmt.Errorf("Unexpected samplecount, wanted 1, got %d", *metric.Histogram.SampleCount)
+			}
+			return nil
 		}
 	}
-	if code != "201" {
-		return fmt.Errorf("unexpected code, wanted 201, got %s", code)
-	}
-	if method != "post" {
-		return fmt.Errorf("unexpected method, wanted post, got %s", method)
-	}
 
-	if *latency.Metric[0].Histogram.SampleCount != 1 {
-		return fmt.Errorf("Unexpected samplecount, wanted 1, got %d", *latency.Metric[0].Histogram.SampleCount)
-	}
-	return nil
+	return fmt.Errorf("Got multiple entries, or none for metric, wanted one, got: %+v", latency.Metric)
 }
 
 func checkCertCount(certCount *dto.MetricFamily) error {