Skip to content

Commit 9ee3d3c

Browse files
dinofizzdinofizz
committed
Switch to JSON for access logs
1 parent 3bc9d49 commit 9ee3d3c

File tree

9 files changed

+863
-634
lines changed

9 files changed

+863
-634
lines changed

‎Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
# Image URL to use all building/pushing image targets
22
IMG ?= ${ACC}.dkr.ecr.eu-west-1.amazonaws.com/monzo/egress-operator:manager-$(shell git rev-parse --short head)
33
# Produce CRDs that work back to Kubernetes 1.11 (no version conversion)
4-
CRD_OPTIONS ?= "crd:trivialVersions=false"
4+
CRD_OPTIONS ?= "crd:trivialVersions=true"
55

66
# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
77
ifeq (,$(shell go env GOBIN))

‎controllers/configmap.go‎

Lines changed: 47 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@ package controllers
33
import (
44
"context"
55
"fmt"
6+
//structpb "github.com/golang/protobuf/ptypes/struct"
7+
"google.golang.org/protobuf/types/known/anypb"
68
"google.golang.org/protobuf/types/known/durationpb"
79
"hash/fnv"
810
"strconv"
@@ -13,16 +15,16 @@ import (
1315
envoycorev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
1416
envoyendpoint "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
1517
envoylistener "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
16-
filev3 "github.com/envoyproxy/go-control-plane/envoy/extensions/access_loggers/file/v3"
18+
streamv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/access_loggers/stream/v3"
1719
aggregatev3 "github.com/envoyproxy/go-control-plane/envoy/extensions/clusters/aggregate/v3"
1820
tcpproxyv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/tcp_proxy/v3"
1921
udpproxyv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/udp/udp_proxy/v3"
20-
"github.com/golang/protobuf/ptypes/wrappers"
21-
"google.golang.org/protobuf/types/known/wrapperspb"
22-
2322
"github.com/golang/protobuf/jsonpb"
2423
"github.com/golang/protobuf/ptypes"
2524
"github.com/golang/protobuf/ptypes/duration"
25+
"github.com/golang/protobuf/ptypes/wrappers"
26+
"google.golang.org/protobuf/types/known/structpb"
27+
"google.golang.org/protobuf/types/known/wrapperspb"
2628
corev1 "k8s.io/api/core/v1"
2729
apierrs "k8s.io/apimachinery/pkg/api/errors"
2830
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -35,6 +37,34 @@ import (
3537

3638
// +kubebuilder:rbac:namespace=egress-operator-system,groups=core,resources=configmaps,verbs=get;list;watch;create;patch
3739

40+
var (
41+
logFields = &structpb.Struct{
42+
Fields: map[string]*structpb.Value{
43+
"authority": {Kind: &structpb.Value_StringValue{StringValue: "%REQ(:AUTHORITY)%"}},
44+
"bytes_received": {Kind: &structpb.Value_StringValue{StringValue: "%BYTES_RECEIVED%"}},
45+
"bytes_sent": {Kind: &structpb.Value_StringValue{StringValue: "%BYTES_SENT%"}},
46+
"connection_termination_details": {Kind: &structpb.Value_StringValue{StringValue: "%CONNECTION_TERMINATION_DETAILS%"}},
47+
"downstream_local_address": {Kind: &structpb.Value_StringValue{StringValue: "%DOWNSTREAM_LOCAL_ADDRESS%"}},
48+
"downstream_remote_address": {Kind: &structpb.Value_StringValue{StringValue: "%DOWNSTREAM_REMOTE_ADDRESS%"}},
49+
"duration": {Kind: &structpb.Value_StringValue{StringValue: "%DURATION%"}},
50+
"method": {Kind: &structpb.Value_StringValue{StringValue: "%REQ(:METHOD)%"}},
51+
"path": {Kind: &structpb.Value_StringValue{StringValue: "%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%"}},
52+
"protocol": {Kind: &structpb.Value_StringValue{StringValue: "%PROTOCOL%"}},
53+
"requested_server_name": {Kind: &structpb.Value_StringValue{StringValue: "%REQUESTED_SERVER_NAME%"}},
54+
"response_code": {Kind: &structpb.Value_StringValue{StringValue: "%RESPONSE_CODE%"}},
55+
"response_code_details": {Kind: &structpb.Value_StringValue{StringValue: "%RESPONSE_CODE_DETAILS%"}},
56+
"response_flags": {Kind: &structpb.Value_StringValue{StringValue: "%RESPONSE_FLAGS%"}},
57+
"start_time": {Kind: &structpb.Value_StringValue{StringValue: "%START_TIME%"}},
58+
"upstream_cluster": {Kind: &structpb.Value_StringValue{StringValue: "%UPSTREAM_CLUSTER%"}},
59+
"upstream_host": {Kind: &structpb.Value_StringValue{StringValue: "%UPSTREAM_HOST%"}},
60+
"upstream_local_address": {Kind: &structpb.Value_StringValue{StringValue: "%UPSTREAM_LOCAL_ADDRESS%"}},
61+
"upstream_service_time": {Kind: &structpb.Value_StringValue{StringValue: "%RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)%"}},
62+
"upstream_transport_failure_reason": {Kind: &structpb.Value_StringValue{StringValue: "%UPSTREAM_TRANSPORT_FAILURE_REASON%"}},
63+
"user_agent": {Kind: &structpb.Value_StringValue{StringValue: "%REQ(USER-AGENT)%"}},
64+
},
65+
}
66+
)
67+
3868
func (r *ExternalServiceReconciler) reconcileConfigMap(ctx context.Context, req ctrl.Request, es *egressv1.ExternalService, desired *corev1.ConfigMap) error {
3969
if err := ctrl.SetControllerReference(es, desired, r.Scheme); err != nil {
4070
return err
@@ -86,9 +116,6 @@ func adminPort(es *egressv1.ExternalService) int32 {
86116
panic("couldn't find a port for admin listener")
87117
}
88118

89-
const accessLogFormat = `[%START_TIME%] %BYTES_RECEIVED% %BYTES_SENT% %DURATION% "%DOWNSTREAM_REMOTE_ADDRESS%" "%UPSTREAM_HOST%" "%UPSTREAM_CLUSTER%"
90-
`
91-
92119
func envoyConfig(es *egressv1.ExternalService) (string, error) {
93120
config := bootstrap.Bootstrap{
94121
Node: &envoycorev3.Node{
@@ -197,16 +224,21 @@ func envoyConfig(es *egressv1.ExternalService) (string, error) {
197224
var listener *envoylistener.Listener
198225
switch protocol {
199226
case envoycorev3.SocketAddress_TCP:
200-
accessConfig, err := ptypes.MarshalAny(&filev3.FileAccessLog{
201-
AccessLogFormat: &filev3.FileAccessLog_Format{
202-
Format: accessLogFormat,
227+
accessConfig, err := anypb.New(&streamv3.StdoutAccessLog{
228+
AccessLogFormat: &streamv3.StdoutAccessLog_LogFormat{
229+
LogFormat: &envoycorev3.SubstitutionFormatString{
230+
Format: &envoycorev3.SubstitutionFormatString_JsonFormat{
231+
JsonFormat: logFields,
232+
},
233+
OmitEmptyValues: true,
234+
ContentType: "application/json; charset=UTF-8",
235+
JsonFormatOptions: nil,
236+
},
203237
},
204-
Path: "/dev/stdout",
205238
})
206-
207-
filterConfig, err := ptypes.MarshalAny(&tcpproxyv3.TcpProxy{
239+
filterConfig, err := anypb.New(&tcpproxyv3.TcpProxy{
208240
AccessLog: []*accesslogfilterv3.AccessLog{{
209-
Name: "envoy.file_access_log",
241+
Name: "envoy.stdout_access_log",
210242
ConfigType: &accesslogfilterv3.AccessLog_TypedConfig{TypedConfig: accessConfig},
211243
}},
212244
StatPrefix: "tcp_proxy",
@@ -236,7 +268,7 @@ func envoyConfig(es *egressv1.ExternalService) (string, error) {
236268
}}}}},
237269
}
238270
case envoycorev3.SocketAddress_UDP:
239-
filterConfig, err := ptypes.MarshalAny(&udpproxyv3.UdpProxyConfig{
271+
filterConfig, err := anypb.New(&udpproxyv3.UdpProxyConfig{
240272
StatPrefix: "udp_proxy",
241273
RouteSpecifier: &udpproxyv3.UdpProxyConfig_Cluster{
242274
Cluster: name,

‎controllers/configmap_test.go‎

Lines changed: 65 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
package controllers
22

33
import (
4+
bootstrap "github.com/envoyproxy/go-control-plane/envoy/config/bootstrap/v3"
5+
"github.com/google/go-cmp/cmp"
6+
"sigs.k8s.io/yaml"
47
"testing"
58

6-
"github.com/google/go-cmp/cmp"
79
corev1 "k8s.io/api/core/v1"
810
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
911

@@ -97,12 +99,34 @@ staticResources:
9799
typedConfig:
98100
'@type': type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy
99101
accessLog:
100-
- name: envoy.file_access_log
102+
- name: envoy.stdout_access_log
101103
typedConfig:
102-
'@type': type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog
103-
format: |
104-
[%START_TIME%] %BYTES_RECEIVED% %BYTES_SENT% %DURATION% "%DOWNSTREAM_REMOTE_ADDRESS%" "%UPSTREAM_HOST%" "%UPSTREAM_CLUSTER%"
105-
path: /dev/stdout
104+
'@type': type.googleapis.com/envoy.extensions.access_loggers.stream.v3.StdoutAccessLog
105+
logFormat:
106+
contentType: application/json; charset=UTF-8
107+
jsonFormat:
108+
authority: '%REQ(:AUTHORITY)%'
109+
bytes_received: '%BYTES_RECEIVED%'
110+
bytes_sent: '%BYTES_SENT%'
111+
connection_termination_details: '%CONNECTION_TERMINATION_DETAILS%'
112+
downstream_local_address: '%DOWNSTREAM_LOCAL_ADDRESS%'
113+
downstream_remote_address: '%DOWNSTREAM_REMOTE_ADDRESS%'
114+
duration: '%DURATION%'
115+
method: '%REQ(:METHOD)%'
116+
path: '%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%'
117+
protocol: '%PROTOCOL%'
118+
requested_server_name: '%REQUESTED_SERVER_NAME%'
119+
response_code: '%RESPONSE_CODE%'
120+
response_code_details: '%RESPONSE_CODE_DETAILS%'
121+
response_flags: '%RESPONSE_FLAGS%'
122+
start_time: '%START_TIME%'
123+
upstream_cluster: '%UPSTREAM_CLUSTER%'
124+
upstream_host: '%UPSTREAM_HOST%'
125+
upstream_local_address: '%UPSTREAM_LOCAL_ADDRESS%'
126+
upstream_service_time: '%RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)%'
127+
upstream_transport_failure_reason: '%UPSTREAM_TRANSPORT_FAILURE_REASON%'
128+
user_agent: '%REQ(USER-AGENT)%'
129+
omitEmptyValues: true
106130
cluster: foo_TCP_101
107131
statPrefix: tcp_proxy
108132
name: foo_TCP_101
@@ -209,12 +233,34 @@ staticResources:
209233
typedConfig:
210234
'@type': type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy
211235
accessLog:
212-
- name: envoy.file_access_log
236+
- name: envoy.stdout_access_log
213237
typedConfig:
214-
'@type': type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog
215-
format: |
216-
[%START_TIME%] %BYTES_RECEIVED% %BYTES_SENT% %DURATION% "%DOWNSTREAM_REMOTE_ADDRESS%" "%UPSTREAM_HOST%" "%UPSTREAM_CLUSTER%"
217-
path: /dev/stdout
238+
'@type': type.googleapis.com/envoy.extensions.access_loggers.stream.v3.StdoutAccessLog
239+
logFormat:
240+
contentType: application/json; charset=UTF-8
241+
jsonFormat:
242+
authority: '%REQ(:AUTHORITY)%'
243+
bytes_received: '%BYTES_RECEIVED%'
244+
bytes_sent: '%BYTES_SENT%'
245+
connection_termination_details: '%CONNECTION_TERMINATION_DETAILS%'
246+
downstream_local_address: '%DOWNSTREAM_LOCAL_ADDRESS%'
247+
downstream_remote_address: '%DOWNSTREAM_REMOTE_ADDRESS%'
248+
duration: '%DURATION%'
249+
method: '%REQ(:METHOD)%'
250+
path: '%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%'
251+
protocol: '%PROTOCOL%'
252+
requested_server_name: '%REQUESTED_SERVER_NAME%'
253+
response_code: '%RESPONSE_CODE%'
254+
response_code_details: '%RESPONSE_CODE_DETAILS%'
255+
response_flags: '%RESPONSE_FLAGS%'
256+
start_time: '%START_TIME%'
257+
upstream_cluster: '%UPSTREAM_CLUSTER%'
258+
upstream_host: '%UPSTREAM_HOST%'
259+
upstream_local_address: '%UPSTREAM_LOCAL_ADDRESS%'
260+
upstream_service_time: '%RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)%'
261+
upstream_transport_failure_reason: '%UPSTREAM_TRANSPORT_FAILURE_REASON%'
262+
user_agent: '%REQ(USER-AGENT)%'
263+
omitEmptyValues: true
218264
cluster: foo_TCP_101
219265
statPrefix: tcp_proxy
220266
name: foo_TCP_101
@@ -246,7 +292,15 @@ staticResources:
246292
for _, tt := range tests {
247293
t.Run(tt.name, func(t *testing.T) {
248294
tt.args.es.Spec.EnvoyClusterMaxConnections = tt.args.maxConns
295+
got, _ := envoyConfig(tt.args.es)
296+
297+
var x bootstrap.Bootstrap
298+
err := yaml.Unmarshal([]byte(got), &x)
299+
if err != nil {
300+
t.Error()
301+
}
249302
if got, _ := envoyConfig(tt.args.es); got != tt.want {
303+
250304
t.Errorf("envoyConfig() = %v, want %v", got, tt.want)
251305
t.Error(cmp.Diff(got, tt.want))
252306
}

‎controllers/testdata/expected_udp_tcp.yaml‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
admin:
2+
accessLogPath: /dev/stdout
3+
address:
4+
socketAddress:
5+
address: 0.0.0.0
6+
portValue: 11000
7+
node:
8+
cluster: foo
9+
staticResources:
10+
clusters:
11+
- connectTimeout: 1s
12+
dnsLookupFamily: V4_ONLY
13+
loadAssignment:
14+
clusterName: foo_UDP_100
15+
endpoints:
16+
- lbEndpoints:
17+
- endpoint:
18+
address:
19+
socketAddress:
20+
address: google.com
21+
portValue: 100
22+
protocol: UDP
23+
name: foo_UDP_100
24+
type: LOGICAL_DNS
25+
upstreamConnectionOptions:
26+
tcpKeepalive:
27+
keepaliveInterval: 5
28+
keepaliveProbes: 3
29+
keepaliveTime: 30
30+
- connectTimeout: 1s
31+
dnsLookupFamily: V4_ONLY
32+
loadAssignment:
33+
clusterName: foo_TCP_101
34+
endpoints:
35+
- lbEndpoints:
36+
- endpoint:
37+
address:
38+
socketAddress:
39+
address: google.com
40+
portValue: 101
41+
name: foo_TCP_101
42+
type: LOGICAL_DNS
43+
upstreamConnectionOptions:
44+
tcpKeepalive:
45+
keepaliveInterval: 5
46+
keepaliveProbes: 3
47+
keepaliveTime: 30
48+
listeners:
49+
- address:
50+
socketAddress:
51+
address: 0.0.0.0
52+
portValue: 100
53+
protocol: UDP
54+
filterChains:
55+
- filters:
56+
- name: envoy.filters.udp_listener.udp_proxy
57+
typedConfig:
58+
"@type": type.googleapis.com/envoy.extensions.filters.udp.udp_proxy.v3.UdpProxyConfig
59+
cluster: foo_UDP_100
60+
statPrefix: udp_proxy
61+
name: foo_UDP_100
62+
- address:
63+
socketAddress:
64+
address: 0.0.0.0
65+
portValue: 101
66+
filterChains:
67+
- filters:
68+
- name: envoy.tcp_proxy
69+
typedConfig:
70+
'@type': type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy
71+
accessLog:
72+
- name: envoy.stdout_access_log
73+
typedConfig:
74+
'@type': type.googleapis.com/envoy.extensions.access_loggers.stream.v3.StdoutAccessLog
75+
logFormat:
76+
contentType: "application/json; charset=UTF-8"
77+
jsonFormat:
78+
authority: '%REQ(:AUTHORITY)%'
79+
bytes_received: '%BYTES_RECEIVED%'
80+
bytes_sent: '%BYTES_SENT%'
81+
connection_termination_details: '%CONNECTION_TERMINATION_DETAILS%'
82+
downstream_local_address: '%DOWNSTREAM_LOCAL_ADDRESS%'
83+
downstream_remote_address: '%DOWNSTREAM_REMOTE_ADDRESS%'
84+
duration: '%DURATION%'
85+
method: '%REQ(:METHOD)%'
86+
path: '%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%'
87+
protocol: '%PROTOCOL%'
88+
requested_server_name: '%REQUESTED_SERVER_NAME%'
89+
response_code: '%RESPONSE_CODE%'
90+
response_code_details: '%RESPONSE_CODE_DETAILS%'
91+
response_flags: '%RESPONSE_FLAGS%'
92+
start_time: '%START__TIME%'
93+
upstream_cluster: '%UPSTREAM_CLUSTER%'
94+
upstream_host: '%UPSTREAM_HOST%'
95+
upstream_local_address: '%UPSTREAM_LOCAL_ADDRESS%'
96+
upstream_service_time: '%RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)%'
97+
upstream_transport_failure_reason: '%UPSTREAM_TRANSPORT_FAILURE_REASON%'
98+
user_agent: '%REQ(USER-AGENT)%'
99+
omitEmptyValues: True
100+
cluster: "foo_TCP_101"
101+
statPrefix: "tcp_proxy"
102+
name: foo_TCP_101

0 commit comments

Comments
 (0)