diff --git a/Dockerfile b/Dockerfile index 53ab003210..a781407d2d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -75,7 +75,7 @@ ARG UI_TAG ARG UI_RELEASE RUN apk add --update --no-cache \ sqlite=3.44.2-r0 \ - postgresql16-client=16.6-r0 \ + postgresql16-client=16.8-r0 \ curl=8.12.1-r0 \ jq=1.7.1-r0 WORKDIR /firefly diff --git a/doc-site/docs/reference/config.md b/doc-site/docs/reference/config.md index 89c794f420..5cf982be27 100644 --- a/doc-site/docs/reference/config.md +++ b/doc-site/docs/reference/config.md @@ -407,30 +407,70 @@ title: Configuration Reference ## metrics +|Key|Description|Type|Default Value| +|---|-----------|----|-------------| +|address|Deprecated - use monitoring.address instead|`int`|`127.0.0.1` +|enabled|Deprecated - use monitoring.enabled instead|`boolean`|`true` +|path|Deprecated - use monitoring.metricsPath instead|`string`|`/metrics` +|port|Deprecated - use monitoring.port instead|`int`|`6000` +|publicURL|Deprecated - use monitoring.publicURL instead|URL `string`|`` +|readTimeout|Deprecated - use monitoring.readTimeout instead|[`time.Duration`](https://pkg.go.dev/time#Duration)|`15s` +|shutdownTimeout|The maximum amount of time to wait for any open HTTP requests to finish before shutting down the HTTP server|[`time.Duration`](https://pkg.go.dev/time#Duration)|`10s` +|writeTimeout|Deprecated - use monitoring.writeTimeout instead|[`time.Duration`](https://pkg.go.dev/time#Duration)|`15s` + +## metrics.auth + +|Key|Description|Type|Default Value| +|---|-----------|----|-------------| +|type|The auth plugin to use for server side authentication of requests|`string`|`` + +## metrics.auth.basic + +|Key|Description|Type|Default Value| +|---|-----------|----|-------------| +|passwordfile|The path to a .htpasswd file to use for authenticating requests. Passwords should be hashed with bcrypt.|`string`|`` + +## metrics.tls + +|Key|Description|Type|Default Value| +|---|-----------|----|-------------| +|ca|The TLS certificate authority in PEM format (this option is ignored if caFile is also set)|`string`|`` +|caFile|The path to the CA file for TLS on this API|`string`|`` +|cert|The TLS certificate in PEM format (this option is ignored if certFile is also set)|`string`|`` +|certFile|The path to the certificate file for TLS on this API|`string`|`` +|clientAuth|Enables or disables client auth for TLS on this API|`string`|`` +|enabled|Enables or disables TLS on this API|`boolean`|`false` +|insecureSkipHostVerify|When to true in unit test development environments to disable TLS verification. Use with extreme caution|`boolean`|`` +|key|The TLS certificate key in PEM format (this option is ignored if keyFile is also set)|`string`|`` +|keyFile|The path to the private key file for TLS on this API|`string`|`` +|requiredDNAttributes|A set of required subject DN attributes. Each entry is a regular expression, and the subject certificate must have a matching attribute of the specified type (CN, C, O, OU, ST, L, STREET, POSTALCODE, SERIALNUMBER are valid attributes)|`map[string]string`|`` + +## monitoring + |Key|Description|Type|Default Value| |---|-----------|----|-------------| |address|The IP address on which the metrics HTTP API should listen|`int`|`127.0.0.1` |enabled|Enables the metrics API|`boolean`|`true` -|path|The path from which to serve the Prometheus metrics|`string`|`/metrics` +|metricsPath|The path from which to serve the Prometheus metrics|`string`|`/metrics` |port|The port on which the metrics HTTP API should listen|`int`|`6000` |publicURL|The fully qualified public URL for the metrics API. This is used for building URLs in HTTP responses and in OpenAPI Spec generation|URL `string`|`` |readTimeout|The maximum time to wait when reading from an HTTP connection|[`time.Duration`](https://pkg.go.dev/time#Duration)|`15s` |shutdownTimeout|The maximum amount of time to wait for any open HTTP requests to finish before shutting down the HTTP server|[`time.Duration`](https://pkg.go.dev/time#Duration)|`10s` |writeTimeout|The maximum time to wait when writing to an HTTP connection|[`time.Duration`](https://pkg.go.dev/time#Duration)|`15s` -## metrics.auth +## monitoring.auth |Key|Description|Type|Default Value| |---|-----------|----|-------------| |type|The auth plugin to use for server side authentication of requests|`string`|`` -## metrics.auth.basic +## monitoring.auth.basic |Key|Description|Type|Default Value| |---|-----------|----|-------------| |passwordfile|The path to a .htpasswd file to use for authenticating requests. Passwords should be hashed with bcrypt.|`string`|`` -## metrics.tls +## monitoring.tls |Key|Description|Type|Default Value| |---|-----------|----|-------------| diff --git a/internal/apiserver/metrics_server.go b/internal/apiserver/metrics_server.go index 7e80ba7c25..e7653e91e1 100644 --- a/internal/apiserver/metrics_server.go +++ b/internal/apiserver/metrics_server.go @@ -1,4 +1,4 @@ -// Copyright © 2022 Kaleido, Inc. +// Copyright © 2025 Kaleido, Inc. // // SPDX-License-Identifier: Apache-2.0 // @@ -21,11 +21,17 @@ import ( ) const ( - MetricsEnabled = "enabled" - MetricsPath = "path" + Enabled = "enabled" + DeprecatedMetricsPath = "path" + MetricsPath = "metricsPath" ) -func initMetricsConfig(config config.Section) { - config.AddKnownKey(MetricsEnabled, true) +func initDeprecatedMetricsConfig(config config.Section) { + config.AddKnownKey(Enabled, true) + config.AddKnownKey(DeprecatedMetricsPath, "/metrics") +} + +func initMonitoringConfig(config config.Section) { + config.AddKnownKey(Enabled, true) config.AddKnownKey(MetricsPath, "/metrics") } diff --git a/internal/apiserver/server.go b/internal/apiserver/server.go index ca1671a051..7639995b9b 100644 --- a/internal/apiserver/server.go +++ b/internal/apiserver/server.go @@ -1,4 +1,4 @@ -// Copyright © 2024 Kaleido, Inc. +// Copyright © 2025 Kaleido, Inc. // // SPDX-License-Identifier: Apache-2.0 // @@ -42,10 +42,11 @@ import ( ) var ( - spiConfig = config.RootSection("spi") - apiConfig = config.RootSection("http") - metricsConfig = config.RootSection("metrics") - corsConfig = config.RootSection("cors") + spiConfig = config.RootSection("spi") + apiConfig = config.RootSection("http") + deprecatedMetricsConfig = config.RootSection("metrics") + monitoringConfig = config.RootSection("monitoring") + corsConfig = config.RootSection("cors") ) // Server is the external interface for the API Server @@ -55,31 +56,35 @@ type Server interface { type apiServer struct { // Defaults set with config - apiTimeout time.Duration - apiMaxTimeout time.Duration - metricsEnabled bool - ffiSwaggerGen FFISwaggerGen - apiPublicURL string - dynamicPublicURLHeader string - defaultNamespace string + apiTimeout time.Duration + apiMaxTimeout time.Duration + deprecatedMetricsEnabled bool + monitoringEnabled bool + ffiSwaggerGen FFISwaggerGen + apiPublicURL string + dynamicPublicURLHeader string + defaultNamespace string } func InitConfig() { httpserver.InitHTTPConfig(apiConfig, 5000) httpserver.InitHTTPConfig(spiConfig, 5001) - httpserver.InitHTTPConfig(metricsConfig, 6000) + httpserver.InitHTTPConfig(deprecatedMetricsConfig, 6000) + httpserver.InitHTTPConfig(monitoringConfig, 6000) httpserver.InitCORSConfig(corsConfig) - initMetricsConfig(metricsConfig) + initDeprecatedMetricsConfig(deprecatedMetricsConfig) + initMonitoringConfig(monitoringConfig) } func NewAPIServer() Server { as := &apiServer{ - apiTimeout: config.GetDuration(coreconfig.APIRequestTimeout), - apiMaxTimeout: config.GetDuration(coreconfig.APIRequestMaxTimeout), - dynamicPublicURLHeader: config.GetString(coreconfig.APIDynamicPublicURLHeader), - defaultNamespace: config.GetString(coreconfig.NamespacesDefault), - metricsEnabled: config.GetBool(coreconfig.MetricsEnabled), - ffiSwaggerGen: &ffiSwaggerGen{}, + apiTimeout: config.GetDuration(coreconfig.APIRequestTimeout), + apiMaxTimeout: config.GetDuration(coreconfig.APIRequestMaxTimeout), + dynamicPublicURLHeader: config.GetString(coreconfig.APIDynamicPublicURLHeader), + defaultNamespace: config.GetString(coreconfig.NamespacesDefault), + deprecatedMetricsEnabled: config.GetBool(coreconfig.DeprecatedMetricsEnabled), + monitoringEnabled: config.GetBool(coreconfig.MonitoringEnabled), + ffiSwaggerGen: &ffiSwaggerGen{}, } as.apiPublicURL = as.getPublicURL(apiConfig, "") return as @@ -110,15 +115,21 @@ func (as *apiServer) Serve(ctx context.Context, mgr namespace.Manager) (err erro } else if config.GetBool(coreconfig.LegacyAdminEnabled) { log.L(ctx).Warnf("Your config includes an 'admin' section, which should be renamed to 'spi' - SPI server will not be enabled until this is corrected") } + serverName := "metrics" + mConfig := deprecatedMetricsConfig + if as.monitoringEnabled { + serverName = "monitoring" + mConfig = monitoringConfig + } - if as.metricsEnabled { - metricsHTTPServer, err := httpserver.NewHTTPServer(ctx, "metrics", as.createMetricsMuxRouter(), metricsErrChan, metricsConfig, corsConfig, &httpserver.ServerOptions{ + if as.deprecatedMetricsEnabled || as.monitoringEnabled { + monitoringServer, err := httpserver.NewHTTPServer(ctx, serverName, as.createMonitoringMuxRouter(), metricsErrChan, mConfig, corsConfig, &httpserver.ServerOptions{ MaximumRequestTimeout: as.apiMaxTimeout, }) if err != nil { return err } - go metricsHTTPServer.ServeHTTP(ctx) + go monitoringServer.ServeHTTP(ctx) } return as.waitForServerStop(httpErrChan, spiErrChan, metricsErrChan) @@ -345,7 +356,7 @@ func (as *apiServer) createMuxRouter(ctx context.Context, mgr namespace.Manager) r := mux.NewRouter() hf := as.handlerFactory() - if as.metricsEnabled { + if as.deprecatedMetricsEnabled || as.monitoringEnabled { r.Use(metrics.GetRestServerInstrumentation().Middleware) } @@ -433,7 +444,7 @@ func (as *apiServer) spiWSHandler(mgr namespace.Manager) http.HandlerFunc { func (as *apiServer) createAdminMuxRouter(mgr namespace.Manager) *mux.Router { r := mux.NewRouter() - if as.metricsEnabled { + if as.deprecatedMetricsEnabled || as.monitoringEnabled { r.Use(metrics.GetAdminServerInstrumentation().Middleware) } hf := as.handlerFactory() @@ -468,12 +479,20 @@ func (as *apiServer) createAdminMuxRouter(mgr namespace.Manager) *mux.Router { return r } -func (as *apiServer) createMetricsMuxRouter() *mux.Router { +func (as *apiServer) createMonitoringMuxRouter() *mux.Router { r := mux.NewRouter() - - r.Path(config.GetString(coreconfig.MetricsPath)).Handler(promhttp.InstrumentMetricHandler(metrics.Registry(), + metricsPath := config.GetString(coreconfig.DeprecatedMetricsPath) + if as.monitoringEnabled { + metricsPath = config.GetString(coreconfig.MonitoringMetricsPath) + } + r.Path(metricsPath).Handler(promhttp.InstrumentMetricHandler(metrics.Registry(), promhttp.HandlerFor(metrics.Registry(), promhttp.HandlerOpts{}))) - + hf := ffapi.HandlerFactory{} + r.HandleFunc("/livez", hf.APIWrapper(func(res http.ResponseWriter, req *http.Request) (status int, err error) { + // a simple liveness check + return http.StatusOK, nil + })) + r.NotFoundHandler = hf.APIWrapper(as.notFoundHandler) return r } diff --git a/internal/apiserver/server_test.go b/internal/apiserver/server_test.go index 7855a033ef..444cb13e92 100644 --- a/internal/apiserver/server_test.go +++ b/internal/apiserver/server_test.go @@ -1,4 +1,4 @@ -// Copyright © 2021 Kaleido, Inc. +// Copyright © 2025 Kaleido, Inc. // // SPDX-License-Identifier: Apache-2.0 // @@ -86,7 +86,7 @@ func TestStartStopServer(t *testing.T) { InitConfig() apiConfig.Set(httpserver.HTTPConfPort, 0) spiConfig.Set(httpserver.HTTPConfPort, 0) - metricsConfig.Set(httpserver.HTTPConfPort, 0) + monitoringConfig.Set(httpserver.HTTPConfPort, 0) config.Set(coreconfig.UIPath, "test") config.Set(coreconfig.SPIEnabled, true) ctx, cancel := context.WithCancel(context.Background()) @@ -105,7 +105,7 @@ func TestStartLegacyAdminConfig(t *testing.T) { InitConfig() apiConfig.Set(httpserver.HTTPConfPort, 0) spiConfig.Set(httpserver.HTTPConfPort, 0) - metricsConfig.Set(httpserver.HTTPConfPort, 0) + monitoringConfig.Set(httpserver.HTTPConfPort, 0) config.Set(coreconfig.UIPath, "test") config.Set(coreconfig.LegacyAdminEnabled, true) ctx, cancel := context.WithCancel(context.Background()) @@ -170,8 +170,8 @@ func TestStartMetricsFail(t *testing.T) { coreconfig.Reset() metrics.Clear() InitConfig() - metricsConfig.Set(httpserver.HTTPConfAddress, "...://") - config.Set(coreconfig.MetricsEnabled, true) + monitoringConfig.Set(httpserver.HTTPConfAddress, "...://") + config.Set(coreconfig.MonitoringEnabled, true) ctx, cancel := context.WithCancel(context.Background()) cancel() // server will immediately shut down as := NewAPIServer() @@ -575,3 +575,17 @@ func TestContractAPIDefaultNS(t *testing.T) { assert.NoError(t, err) assert.Equal(t, 200, res.StatusCode()) } + +func TestMonitoringServerRoutes(t *testing.T) { + _, _, as := newTestServer() + s := httptest.NewServer(as.createMonitoringMuxRouter()) + defer s.Close() + + res, err := http.Get(fmt.Sprintf("http://%s/livez", s.Listener.Addr())) + assert.NoError(t, err) + assert.Equal(t, 200, res.StatusCode) + + res, err = http.Get(fmt.Sprintf("http://%s/metrics", s.Listener.Addr())) + assert.NoError(t, err) + assert.Equal(t, 200, res.StatusCode) +} diff --git a/internal/coreconfig/coreconfig.go b/internal/coreconfig/coreconfig.go index 401f69c1ac..ad6428ee89 100644 --- a/internal/coreconfig/coreconfig.go +++ b/internal/coreconfig/coreconfig.go @@ -1,4 +1,4 @@ -// Copyright © 2024 Kaleido, Inc. +// Copyright © 2025 Kaleido, Inc. // // SPDX-License-Identifier: Apache-2.0 // @@ -287,10 +287,14 @@ var ( MessageWriterBatchTimeout = ffc("message.writer.batchTimeout") // MessageWriterBatchMaxInserts MessageWriterBatchMaxInserts = ffc("message.writer.batchMaxInserts") - // MetricsEnabled determines whether metrics will be instrumented and if the metrics server will be enabled or not - MetricsEnabled = ffc("metrics.enabled") + // MetricsEnabled - deprecated, use monitoring.enabled + DeprecatedMetricsEnabled = ffc("metrics.enabled") + // MetricsPath - deprecated, use monitoring.metricsPath + DeprecatedMetricsPath = ffc("metrics.path") + // Monitoring determines whether monitoring routes will be enabled, which contains metrics instruments + MonitoringEnabled = ffc("monitoring.enabled") // MetricsPath determines what path to serve the Prometheus metrics from - MetricsPath = ffc("metrics.path") + MonitoringMetricsPath = ffc("monitoring.metricsPath") // NamespacesDefault is the default namespace - must be in the predefines list NamespacesDefault = ffc("namespaces.default") // NamespacesPredefined is a list of namespaces to ensure exists, without requiring a broadcast from the network diff --git a/internal/coremsgs/en_config_descriptions.go b/internal/coremsgs/en_config_descriptions.go index 1015b75710..3995499d93 100644 --- a/internal/coremsgs/en_config_descriptions.go +++ b/internal/coremsgs/en_config_descriptions.go @@ -1,4 +1,4 @@ -// Copyright © 2024 Kaleido, Inc. +// Copyright © 2025 Kaleido, Inc. // // SPDX-License-Identifier: Apache-2.0 // @@ -320,13 +320,21 @@ var ( ConfigTransactionWriterBatchTimeout = ffc("config.transaction.writer.batchTimeout", "How long to wait for more transactions to arrive before flushing the batch", i18n.TimeDurationType) ConfigTransactionWriterCount = ffc("config.transaction.writer.count", "The number of message writer workers", i18n.IntType) - ConfigMetricsAddress = ffc("config.metrics.address", "The IP address on which the metrics HTTP API should listen", i18n.IntType) - ConfigMetricsEnabled = ffc("config.metrics.enabled", "Enables the metrics API", i18n.BooleanType) - ConfigMetricsPath = ffc("config.metrics.path", "The path from which to serve the Prometheus metrics", i18n.StringType) - ConfigMetricsPort = ffc("config.metrics.port", "The port on which the metrics HTTP API should listen", i18n.IntType) - ConfigMetricsPublicURL = ffc("config.metrics.publicURL", "The fully qualified public URL for the metrics API. This is used for building URLs in HTTP responses and in OpenAPI Spec generation", urlStringType) - ConfigMetricsReadTimeout = ffc("config.metrics.readTimeout", "The maximum time to wait when reading from an HTTP connection", i18n.TimeDurationType) - ConfigMetricsWriteTimeout = ffc("config.metrics.writeTimeout", "The maximum time to wait when writing to an HTTP connection", i18n.TimeDurationType) + DeprecatedConfigMetricsAddress = ffc("config.metrics.address", "Deprecated - use monitoring.address instead", i18n.IntType) + DeprecatedConfigMetricsEnabled = ffc("config.metrics.enabled", "Deprecated - use monitoring.enabled instead", i18n.BooleanType) + DeprecatedConfigMetricsPath = ffc("config.metrics.path", "Deprecated - use monitoring.metricsPath instead", i18n.StringType) + DeprecatedConfigMetricsPort = ffc("config.metrics.port", "Deprecated - use monitoring.port instead", i18n.IntType) + DeprecatedConfigMetricsPublicURL = ffc("config.metrics.publicURL", "Deprecated - use monitoring.publicURL instead", urlStringType) + DeprecatedConfigMetricsReadTimeout = ffc("config.metrics.readTimeout", "Deprecated - use monitoring.readTimeout instead", i18n.TimeDurationType) + DeprecatedConfigMetricsWriteTimeout = ffc("config.metrics.writeTimeout", "Deprecated - use monitoring.writeTimeout instead", i18n.TimeDurationType) + + ConfigMetricsAddress = ffc("config.monitoring.address", "The IP address on which the metrics HTTP API should listen", i18n.IntType) + ConfigMetricsEnabled = ffc("config.monitoring.enabled", "Enables the metrics API", i18n.BooleanType) + ConfigMetricsPath = ffc("config.monitoring.metricsPath", "The path from which to serve the Prometheus metrics", i18n.StringType) + ConfigMetricsPort = ffc("config.monitoring.port", "The port on which the metrics HTTP API should listen", i18n.IntType) + ConfigMetricsPublicURL = ffc("config.monitoring.publicURL", "The fully qualified public URL for the metrics API. This is used for building URLs in HTTP responses and in OpenAPI Spec generation", urlStringType) + ConfigMetricsReadTimeout = ffc("config.monitoring.readTimeout", "The maximum time to wait when reading from an HTTP connection", i18n.TimeDurationType) + ConfigMetricsWriteTimeout = ffc("config.monitoring.writeTimeout", "The maximum time to wait when writing to an HTTP connection", i18n.TimeDurationType) ConfigNamespacesDefault = ffc("config.namespaces.default", "The default namespace - must be in the predefined list", i18n.StringType) ConfigNamespacesPredefined = ffc("config.namespaces.predefined", "A list of namespaces to ensure exists, without requiring a broadcast from the network", "List "+i18n.StringType) diff --git a/internal/events/webhooks/webhooks_test.go b/internal/events/webhooks/webhooks_test.go index f38ba4e0a7..958e21ba64 100644 --- a/internal/events/webhooks/webhooks_test.go +++ b/internal/events/webhooks/webhooks_test.go @@ -481,7 +481,23 @@ func TestRequestWithBodyReplyEndToEndWithTLS(t *testing.T) { }() server.Handler = r - go server.ListenAndServeTLS(publicKeyFile.Name(), privateKeyFile.Name()) + go func() { + if err := server.ListenAndServeTLS(publicKeyFile.Name(), privateKeyFile.Name()); err != nil && err != http.ErrServerClosed { + log.Fatalf("ListenAndServeTLS(): %v", err) + } + }() + + // Wait for the server to be ready + for { + conn, err := tls.Dial("tcp", server.Addr, &tls.Config{ + InsecureSkipVerify: true, + }) + if err == nil { + conn.Close() + break + } + time.Sleep(10 * time.Millisecond) + } // Build a TLS config for the client and set on the subscription object cert, err := tls.LoadX509KeyPair(publicKeyFile.Name(), privateKeyFile.Name()) diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 3169306b9b..35f3d5b19a 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -1,4 +1,4 @@ -// Copyright © 2024 Kaleido, Inc. +// Copyright © 2025 Kaleido, Inc. // // SPDX-License-Identifier: Apache-2.0 // @@ -54,7 +54,7 @@ type metricsManager struct { func NewMetricsManager(ctx context.Context) Manager { mm := &metricsManager{ ctx: ctx, - metricsEnabled: config.GetBool(coreconfig.MetricsEnabled), + metricsEnabled: config.GetBool(coreconfig.DeprecatedMetricsEnabled) || config.GetBool(coreconfig.MonitoringEnabled), timeMap: make(map[string]time.Time), } diff --git a/internal/namespace/manager.go b/internal/namespace/manager.go index 65a82714a2..b19ed57029 100644 --- a/internal/namespace/manager.go +++ b/internal/namespace/manager.go @@ -1,4 +1,4 @@ -// Copyright © 2024 Kaleido, Inc. +// Copyright © 2025 Kaleido, Inc. // // SPDX-License-Identifier: Apache-2.0 // @@ -161,7 +161,7 @@ func stringSlicesEqual(a, b []string) bool { func NewNamespaceManager() Manager { nm := &namespaceManager{ namespaces: make(map[string]*namespace), - metricsEnabled: config.GetBool(coreconfig.MetricsEnabled), + metricsEnabled: config.GetBool(coreconfig.DeprecatedMetricsEnabled) || config.GetBool(coreconfig.MonitoringEnabled), tokenBroadcastNames: make(map[string]string), watchConfig: viper.WatchConfig,