Skip to content

Commit 8b2913e

Browse files
vipul-21pchaigno
authored andcommitted
feat(sdp): Add metrics for standalone dns proxy
Add a dedicated Prometheus metrics subsystem for the standalone DNS proxy (SDP), exposing error counters for observability: - cilium_agent_connection_errors: gRPC client creation and policy stream failures - fqdn_mapping_sync_errors: FQDN mapping sync connection and request errors - retrieve_dns_rules_errors: policy stream send/recv failures - proxy_bootstrap_errors: DNS proxy listener startup failures The metrics are served on :9961. Note: Since the SDP DaemonSet runs with hostNetwork: true and uses a surge-based rolling update strategy (maxSurge: 2, maxUnavailable: 0), the metrics HTTP server sets SO_REUSEADDR and SO_REUSEPORT on its listener socket. This allows the new pod to bind the metrics port while the old pod is still terminating, avoiding bind failures during upgrades. Signed-off-by: Vipul Singh <singhvipul@microsoft.com>
1 parent 881d4d5 commit 8b2913e

8 files changed

Lines changed: 205 additions & 4 deletions

File tree

standalone-dns-proxy/cmd/root.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import (
2626
"github.com/cilium/cilium/standalone-dns-proxy/pkg/defaults"
2727
"github.com/cilium/cilium/standalone-dns-proxy/pkg/lookup"
2828
"github.com/cilium/cilium/standalone-dns-proxy/pkg/messagehandler"
29+
"github.com/cilium/cilium/standalone-dns-proxy/pkg/metrics"
2930
sdpshell "github.com/cilium/cilium/standalone-dns-proxy/pkg/shell"
3031
)
3132

@@ -50,6 +51,9 @@ var (
5051
// includes the message handler for receiving messages from the proxy and sending messages to the gRPC client which in turn sends them to the cilium agent
5152
messagehandler.Cell,
5253

54+
// Prometheus metrics registry and HTTP server for standalone DNS proxy
55+
metrics.Cell,
56+
5357
// Shell for inspecting the standalone DNS proxy. Listens on the Unix domain socket.
5458
shell.ServerCell(defaults.ShellSockPath),
5559

@@ -131,6 +135,7 @@ type standaloneDNSProxyParams struct {
131135
DNSProxier proxy.DNSProxier
132136
DNSRulesTable statedb.RWTable[client.DNSRules]
133137
DB *statedb.DB
138+
Metrics *metrics.Metrics
134139
}
135140

136141
type hooksParams struct {

standalone-dns-proxy/cmd/standalonednsproxy.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ import (
1919
"github.com/cilium/cilium/pkg/revert"
2020
"github.com/cilium/cilium/pkg/time"
2121
"github.com/cilium/cilium/standalone-dns-proxy/pkg/client"
22+
sdpmetrics "github.com/cilium/cilium/standalone-dns-proxy/pkg/metrics"
2223
)
2324

2425
// ReadinessStatusProvider is an interface for checking the readiness status
@@ -44,6 +45,9 @@ type StandaloneDNSProxy struct {
4445
db *statedb.DB
4546
jobGroup job.Group
4647

48+
// metrics tracks errors for the standalone DNS proxy
49+
metrics *sdpmetrics.Metrics
50+
4751
// readinessStatus tracks the readiness status of this standalone DNS proxy instance
4852
readinessStatus atomic.Bool
4953
}
@@ -71,6 +75,7 @@ func NewStandaloneDNSProxy(params standaloneDNSProxyParams) *StandaloneDNSProxy
7175
db: params.DB,
7276
dnsRulesTable: params.DNSRulesTable,
7377
jobGroup: params.JobGroup,
78+
metrics: params.Metrics,
7479
}
7580
}
7681

@@ -115,6 +120,7 @@ func (sdp *StandaloneDNSProxy) WatchConnection(ctx context.Context, _ cell.Healt
115120
// Start the DNS proxy once the connection is established
116121
if err := sdp.dnsProxier.Listen(sdp.proxyPort); err != nil {
117122
sdp.logger.Error("Failed to start DNS proxy", logfields.Error, err)
123+
sdp.metrics.ProxyBootstrapError.Inc()
118124
return err
119125
}
120126
return nil

standalone-dns-proxy/cmd/standalonednsproxy_test.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ import (
2929
"github.com/cilium/cilium/standalone-dns-proxy/pkg/client"
3030
"github.com/cilium/cilium/standalone-dns-proxy/pkg/lookup"
3131
"github.com/cilium/cilium/standalone-dns-proxy/pkg/messagehandler"
32+
"github.com/cilium/cilium/standalone-dns-proxy/pkg/metrics"
3233
)
3334

3435
var (
@@ -82,6 +83,7 @@ func setupTestEnv(t *testing.T) *StandaloneDNSProxy {
8283
ToFQDNsProxyPort: 1001,
8384
}
8485
},
86+
metrics.NewMetrics,
8587
NewStandaloneDNSProxy,
8688
NewReadinessStatusProvider,
8789
)),

standalone-dns-proxy/pkg/client/cell.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import (
1414
"github.com/cilium/cilium/pkg/logging"
1515
"github.com/cilium/cilium/pkg/logging/logfields"
1616
"github.com/cilium/cilium/pkg/time"
17+
"github.com/cilium/cilium/standalone-dns-proxy/pkg/metrics"
1718
)
1819

1920
const (
@@ -46,6 +47,7 @@ type clientParams struct {
4647
DNSRulesTable statedb.RWTable[DNSRules]
4748
IPtoEndpointTable statedb.RWTable[IPtoEndpointInfo]
4849
PrefixToIdentityTable statedb.RWTable[PrefixToIdentity]
50+
Metrics *metrics.Metrics
4951
}
5052

5153
// newGRPCClient creates a new gRPC connection handler client for standalone DNS proxy

standalone-dns-proxy/pkg/client/client.go

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ import (
3333
"github.com/cilium/cilium/pkg/policy/types"
3434
"github.com/cilium/cilium/pkg/time"
3535
"github.com/cilium/cilium/pkg/u8proto"
36+
sdpmetrics "github.com/cilium/cilium/standalone-dns-proxy/pkg/metrics"
3637

3738
pb "github.com/cilium/cilium/api/v1/standalone-dns-proxy"
3839
)
@@ -235,6 +236,9 @@ type GRPCClient struct {
235236

236237
// grpc client connection to the Cilium agent
237238
client *grpc.ClientConn
239+
240+
// metrics tracks errors for the standalone DNS proxy gRPC client
241+
metrics *sdpmetrics.Metrics
238242
}
239243

240244
// createGRPCClient creates a new gRPC connection handler client for standalone DNS proxy
@@ -248,6 +252,7 @@ func createGRPCClient(params clientParams) *GRPCClient {
248252
dnsRulesTable: params.DNSRulesTable,
249253
ipToEndpointTable: params.IPtoEndpointTable,
250254
prefixToIdentityTable: params.PrefixToIdentityTable,
255+
metrics: params.Metrics,
251256
}
252257
}
253258

@@ -260,6 +265,7 @@ func (c *GRPCClient) InitClient() error {
260265
)
261266
if err != nil {
262267
c.logger.Error("Client creation failed", logfields.Error, err)
268+
c.metrics.CiliumAgentConnection.WithLabelValues(sdpmetrics.LabelErrorClientCreation).Inc()
263269
return err
264270
}
265271

@@ -280,6 +286,7 @@ func (c *GRPCClient) createPolicyStream(ctx context.Context) error {
280286
stream, err := fqdnClient.StreamPolicyState(context.Background())
281287
if err != nil {
282288
c.logger.Error("Failed to open policy stream", logfields.Error, err)
289+
c.metrics.CiliumAgentConnection.WithLabelValues(sdpmetrics.LabelErrorOpenPolicyStream).Inc()
283290
return err
284291
}
285292
defer stream.CloseSend()
@@ -290,6 +297,7 @@ func (c *GRPCClient) createPolicyStream(ctx context.Context) error {
290297
state, err := stream.Recv()
291298
if err != nil {
292299
c.logger.Error("Policy stream recv failed", logfields.Error, err)
300+
c.metrics.RetrieveDNSRules.WithLabelValues(sdpmetrics.LabelErrorPolicyStreamRecv).Inc()
293301
return err
294302
}
295303
response := &pb.PolicyStateResponse{
@@ -303,6 +311,7 @@ func (c *GRPCClient) createPolicyStream(ctx context.Context) error {
303311
}
304312
if sendErr := stream.Send(response); sendErr != nil {
305313
c.logger.Error("Policy stream ACK send failed", logfields.Error, sendErr)
314+
c.metrics.RetrieveDNSRules.WithLabelValues(sdpmetrics.LabelErrorPolicyStreamSend).Inc()
306315
return sendErr
307316
}
308317
c.connected.Store(true)
@@ -336,10 +345,14 @@ func (c *GRPCClient) NotifyOnMsg(msg *pb.FQDNMapping) error {
336345
client := pb.NewFQDNDataClient(c.client)
337346

338347
_, err := client.UpdateMappingRequest(context.Background(), msg)
339-
if err != nil && isConnectionError(err) {
340-
c.logger.Error("Connection error during UpdateMappingRequest", logfields.Error, err)
341-
// Return nil as standalone dns proxy can still continue to handle the DNS requests
342-
return nil
348+
if err != nil {
349+
if isConnectionError(err) {
350+
c.logger.Error("Connection error during UpdateMappingRequest", logfields.Error, err)
351+
c.metrics.FQDNMappingSync.WithLabelValues(sdpmetrics.LabelErrorMappingSyncConnection).Inc()
352+
// Return nil as standalone dns proxy can still continue to handle the DNS requests
353+
return nil
354+
}
355+
c.metrics.FQDNMappingSync.WithLabelValues(sdpmetrics.LabelErrorMappingSyncRequest).Inc()
343356
}
344357
return err
345358
}

standalone-dns-proxy/pkg/client/client_test.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ import (
2929
"github.com/cilium/cilium/pkg/testutils"
3030
"github.com/cilium/cilium/pkg/time"
3131
"github.com/cilium/cilium/pkg/u8proto"
32+
"github.com/cilium/cilium/standalone-dns-proxy/pkg/metrics"
3233

3334
pb "github.com/cilium/cilium/api/v1/standalone-dns-proxy"
3435
)
@@ -125,6 +126,7 @@ func setupClientAndServer(t *testing.T) (ConnectionHandler, *mockFqdnDataServer,
125126
return newMockDialConfig(lis)
126127
},
127128
newGRPCClient),
129+
cell.Provide(metrics.NewMetrics),
128130
cell.Invoke(func(_c ConnectionHandler) {
129131
connHandler = _c
130132
}),
Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
// SPDX-License-Identifier: Apache-2.0
2+
// Copyright Authors of Cilium
3+
4+
package metrics
5+
6+
import (
7+
"github.com/cilium/hive/cell"
8+
"github.com/spf13/pflag"
9+
10+
"github.com/cilium/cilium/pkg/metrics"
11+
)
12+
13+
const (
14+
// SDPPrometheusServeAddr is the flag name for the standalone DNS proxy
15+
// prometheus metrics serve address.
16+
SDPPrometheusServeAddr = "sdp-prometheus-serve-addr"
17+
)
18+
19+
// Cell provides the modular metrics registry and metric HTTP server
20+
// for the standalone DNS proxy.
21+
var Cell = cell.Module(
22+
"sdp-metrics",
23+
"Standalone DNS Proxy Metrics",
24+
25+
cell.Config(defaultConfig),
26+
cell.Provide(func(conf Config) metrics.RegistryConfig {
27+
return metrics.RegistryConfig{
28+
PrometheusServeAddr: conf.SDPPrometheusServeAddr,
29+
}
30+
}),
31+
metrics.NewCell("sdp"),
32+
metrics.Metric(NewMetrics),
33+
cell.Invoke(initializeMetrics),
34+
)
35+
36+
type Config struct {
37+
SDPPrometheusServeAddr string
38+
}
39+
40+
var defaultConfig = Config{
41+
SDPPrometheusServeAddr: ":9961",
42+
}
43+
44+
func (def Config) Flags(flags *pflag.FlagSet) {
45+
flags.String(SDPPrometheusServeAddr, def.SDPPrometheusServeAddr, "Address to serve Prometheus metrics for the standalone DNS proxy")
46+
}
Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
// SPDX-License-Identifier: Apache-2.0
2+
// Copyright Authors of Cilium
3+
4+
package metrics
5+
6+
import (
7+
"fmt"
8+
"log/slog"
9+
"net"
10+
"regexp"
11+
"syscall"
12+
13+
"github.com/cilium/hive/cell"
14+
"github.com/prometheus/client_golang/prometheus"
15+
"github.com/prometheus/client_golang/prometheus/collectors"
16+
"golang.org/x/sys/unix"
17+
18+
"github.com/cilium/cilium/pkg/metrics"
19+
"github.com/cilium/cilium/pkg/metrics/metric"
20+
)
21+
22+
const (
23+
sdpNamespace = "standalone_dns_proxy"
24+
25+
// Error labels for CiliumAgentConnection metric.
26+
LabelErrorClientCreation = "client_creation" // gRPC client creation failed
27+
LabelErrorOpenPolicyStream = "open_policy_stream" // opening the policy stream failed
28+
29+
// Error labels for RetrieveDNSRules metric.
30+
LabelErrorPolicyStreamSend = "policy_stream_send" // sending ACK on the policy stream failed
31+
LabelErrorPolicyStreamRecv = "policy_stream_recv" // receiving from the policy stream failed
32+
33+
// Error labels for FQDNMappingSync metric.
34+
LabelErrorMappingSyncConnection = "mapping_sync_connection" // FQDN mapping sync failed due to connection error
35+
LabelErrorMappingSyncRequest = "mapping_sync_request" // FQDN mapping sync request failed
36+
)
37+
38+
// goCustomCollectorsRX tracks enabled go runtime metrics.
39+
var goCustomCollectorsRX = regexp.MustCompile(`^/sched/latencies:seconds`)
40+
41+
// Metrics contains all metrics for the standalone DNS proxy.
42+
type Metrics struct {
43+
// CiliumAgentConnection tracks total errors while connecting to cilium-agent.
44+
CiliumAgentConnection metric.Vec[metric.Counter]
45+
46+
// FQDNMappingSync tracks errors while syncing FQDN mappings.
47+
FQDNMappingSync metric.Vec[metric.Counter]
48+
49+
// RetrieveDNSRules tracks errors while retrieving DNS rules.
50+
RetrieveDNSRules metric.Vec[metric.Counter]
51+
52+
// ProxyBootstrapError tracks errors that occurred during proxy bootstrap.
53+
ProxyBootstrapError metric.Counter
54+
}
55+
56+
// NewMetrics creates the SDP metrics.
57+
func NewMetrics() *Metrics {
58+
return &Metrics{
59+
CiliumAgentConnection: metric.NewCounterVec(metric.CounterOpts{
60+
Namespace: sdpNamespace,
61+
Name: "cilium_agent_connection_errors",
62+
Help: "Total errors while connecting to cilium-agent",
63+
}, []string{metrics.LabelError}),
64+
FQDNMappingSync: metric.NewCounterVec(metric.CounterOpts{
65+
Namespace: sdpNamespace,
66+
Name: "fqdn_mapping_sync_errors",
67+
Help: "Total errors while syncing FQDN mappings",
68+
}, []string{metrics.LabelError}),
69+
RetrieveDNSRules: metric.NewCounterVec(metric.CounterOpts{
70+
Namespace: sdpNamespace,
71+
Name: "retrieve_dns_rules_errors",
72+
Help: "Total errors while retrieving DNS rules",
73+
}, []string{metrics.LabelError}),
74+
ProxyBootstrapError: metric.NewCounter(metric.CounterOpts{
75+
Namespace: sdpNamespace,
76+
Name: "proxy_bootstrap_errors",
77+
Help: "Number of errors occurred during proxy bootstrap",
78+
}),
79+
}
80+
}
81+
82+
type initParams struct {
83+
cell.In
84+
85+
Logger *slog.Logger
86+
Registry *metrics.Registry
87+
88+
Metrics []metric.WithMetadata `group:"hive-metrics"`
89+
}
90+
91+
func initializeMetrics(p initParams) {
92+
p.Registry.MustRegister(collectors.NewGoCollector(
93+
collectors.WithGoCollectorRuntimeMetrics(
94+
collectors.GoRuntimeMetricsRule{Matcher: goCustomCollectorsRX},
95+
),
96+
))
97+
98+
p.Registry.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{
99+
Namespace: sdpNamespace,
100+
}))
101+
102+
for _, m := range p.Metrics {
103+
p.Registry.MustRegister(m.(prometheus.Collector))
104+
}
105+
106+
p.Registry.AddServerRuntimeHooks("sdp-prometheus-server", nil, net.ListenConfig{
107+
Control: setsockoptReusePort,
108+
})
109+
}
110+
111+
// setsockoptReusePort sets SO_REUSEPORT on the socket to allow the new SDP pod
112+
// to bind the metrics port while the old pod is still terminating during a
113+
// surge-based rolling update.
114+
func setsockoptReusePort(network, address string, c syscall.RawConn) error {
115+
var soerr error
116+
if err := c.Control(func(su uintptr) {
117+
s := int(su)
118+
if err := unix.SetsockoptInt(s, unix.SOL_SOCKET, unix.SO_REUSEPORT, 1); err != nil {
119+
soerr = fmt.Errorf("failed to setsockopt(SO_REUSEPORT): %w", err)
120+
}
121+
}); err != nil {
122+
return err
123+
}
124+
return soerr
125+
}

0 commit comments

Comments
 (0)