Skip to content

Commit e4e6bf7

Browse files
i3149i3149
authored
Adding a vrf field for flow consumed by ktrans directly (#901)
* Adding a vrf field for flow consumed by ktrans directly * Bumping goflow version to 2.6 * Actually adding the vrfs to the proto config
1 parent 5768610 commit e4e6bf7

4 files changed

Lines changed: 53 additions & 25 deletions

File tree

‎go.mod‎

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -39,11 +39,11 @@ require (
3939
github.com/liamg/furious v0.0.0-20191231090757-c295c872d6c1
4040
github.com/linkedin/goavro/v2 v2.10.1
4141
github.com/montanaflynn/stats v0.7.0
42-
github.com/netsampler/goflow2/v2 v2.1.3
43-
github.com/oschwald/geoip2-golang v1.9.0
42+
github.com/netsampler/goflow2/v2 v2.2.6
43+
github.com/oschwald/geoip2-golang v1.13.0
4444
github.com/pkg/errors v0.9.1
4545
github.com/prometheus-community/pro-bing v0.3.0
46-
github.com/prometheus/client_golang v1.19.0
46+
github.com/prometheus/client_golang v1.23.2
4747
github.com/prometheus/prometheus v0.46.0
4848
github.com/redis/go-redis/v9 v9.18.0
4949
github.com/sasha-s/go-hll v0.0.0-20180522065212-c6eb27aee351
@@ -130,7 +130,7 @@ require (
130130
github.com/jmespath/go-jmespath v0.4.0 // indirect
131131
github.com/josharian/intern v1.0.0 // indirect
132132
github.com/kevinburke/ssh_config v1.5.0 // indirect
133-
github.com/klauspost/compress v1.16.7 // indirect
133+
github.com/klauspost/compress v1.18.0 // indirect
134134
github.com/klauspost/cpuid/v2 v2.3.0 // indirect
135135
github.com/kylelemons/godebug v1.1.0 // indirect
136136
github.com/libp2p/go-reuseport v0.4.0 // indirect
@@ -139,19 +139,20 @@ require (
139139
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
140140
github.com/modern-go/reflect2 v1.0.2 // indirect
141141
github.com/mostlygeek/arp v0.0.0-20170424181311-541a2129847a // indirect
142+
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
142143
github.com/oklog/ulid v1.3.1 // indirect
143144
github.com/onsi/ginkgo v1.16.2 // indirect
144145
github.com/onsi/gomega v1.34.1 // indirect
145146
github.com/opentracing/opentracing-go v1.2.0 // indirect
146-
github.com/oschwald/maxminddb-golang v1.11.0 // indirect
147+
github.com/oschwald/maxminddb-golang v1.13.0 // indirect
147148
github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 // indirect
148149
github.com/pierrec/lz4/v4 v4.1.17 // indirect
149150
github.com/pjbgf/sha1cd v0.5.0 // indirect
150151
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
151152
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
152-
github.com/prometheus/client_model v0.5.0 // indirect
153-
github.com/prometheus/common v0.48.0 // indirect
154-
github.com/prometheus/procfs v0.12.0 // indirect
153+
github.com/prometheus/client_model v0.6.2 // indirect
154+
github.com/prometheus/common v0.66.1 // indirect
155+
github.com/prometheus/procfs v0.16.1 // indirect
155156
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
156157
github.com/sergi/go-diff v1.4.0 // indirect
157158
github.com/sirupsen/logrus v1.9.3 // indirect
@@ -167,6 +168,7 @@ require (
167168
go.opentelemetry.io/otel/trace v1.43.0 // indirect
168169
go.opentelemetry.io/proto/otlp v1.10.0 // indirect
169170
go.uber.org/atomic v1.11.0 // indirect
171+
go.yaml.in/yaml/v2 v2.4.2 // indirect
170172
golang.org/x/crypto v0.49.0 // indirect
171173
golang.org/x/oauth2 v0.35.0 // indirect
172174
golang.org/x/sync v0.20.0 // indirect

‎go.sum‎

Lines changed: 20 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -693,8 +693,8 @@ github.com/klauspost/compress v1.13.5/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47e
693693
github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
694694
github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
695695
github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
696-
github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I=
697-
github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
696+
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
697+
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
698698
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
699699
github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
700700
github.com/klauspost/cpuid/v2 v2.1.0/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=
@@ -763,9 +763,11 @@ github.com/montanaflynn/stats v0.7.0 h1:r3y12KyNxj/Sb/iOE46ws+3mS1+MZca1wlHQFPsY
763763
github.com/montanaflynn/stats v0.7.0/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow=
764764
github.com/mostlygeek/arp v0.0.0-20170424181311-541a2129847a h1:AfneHvfmYgUIcgdUrrDFklLdEzQAvG9AKRTe1x1mx/0=
765765
github.com/mostlygeek/arp v0.0.0-20170424181311-541a2129847a/go.mod h1:jZxafo9CAqaKFQE4zitrg5QNlA6CXUsjwXPlIppF3tk=
766+
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
767+
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
766768
github.com/ncw/swift v1.0.52/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM=
767-
github.com/netsampler/goflow2/v2 v2.1.3 h1:glfeG2hIzFlAmEz236nZIAWi848AB+p1pJnuQqiyLkI=
768-
github.com/netsampler/goflow2/v2 v2.1.3/go.mod h1:94ZaxfHuUwG6KviCxMWuZIvz0UGu657StE/g83hlS8A=
769+
github.com/netsampler/goflow2/v2 v2.2.6 h1:pm+UEykIYV+lGJzrunYLNehoQtHlipJTp3lFhBpUkj0=
770+
github.com/netsampler/goflow2/v2 v2.2.6/go.mod h1:sKVfADpYP0NvkEeMRMh8/vz6T1yq4z1JZqO+N0+RGqk=
769771
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
770772
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
771773
github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
@@ -784,10 +786,10 @@ github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k=
784786
github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY=
785787
github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs=
786788
github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
787-
github.com/oschwald/geoip2-golang v1.9.0 h1:uvD3O6fXAXs+usU+UGExshpdP13GAqp4GBrzN7IgKZc=
788-
github.com/oschwald/geoip2-golang v1.9.0/go.mod h1:BHK6TvDyATVQhKNbQBdrj9eAvuwOMi2zSFXizL3K81Y=
789-
github.com/oschwald/maxminddb-golang v1.11.0 h1:aSXMqYR/EPNjGE8epgqwDay+P30hCBZIveY0WZbAWh0=
790-
github.com/oschwald/maxminddb-golang v1.11.0/go.mod h1:YmVI+H0zh3ySFR3w+oz8PCfglAFj3PuCmui13+P9zDg=
789+
github.com/oschwald/geoip2-golang v1.13.0 h1:Q44/Ldc703pasJeP5V9+aFSZFmBN7DKHbNsSFzQATJI=
790+
github.com/oschwald/geoip2-golang v1.13.0/go.mod h1:P9zG+54KPEFOliZ29i7SeYZ/GM6tfEL+rgSn03hYuUo=
791+
github.com/oschwald/maxminddb-golang v1.13.0 h1:R8xBorY71s84yO06NgTmQvqvTvlS/bnYZrrWX1MElnU=
792+
github.com/oschwald/maxminddb-golang v1.13.0/go.mod h1:BU0z8BfFVhi1LQaonTwwGQlsHUEu9pWNdMfmq4ztm0o=
791793
github.com/pborman/getopt v0.0.0-20180729010549-6fdd0a2c7117/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o=
792794
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
793795
github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE=
@@ -816,15 +818,15 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI
816818
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
817819
github.com/prometheus-community/pro-bing v0.3.0 h1:SFT6gHqXwbItEDJhTkzPWVqU6CLEtqEfNAPp47RUON4=
818820
github.com/prometheus-community/pro-bing v0.3.0/go.mod h1:p9dLb9zdmv+eLxWfCT6jESWuDrS+YzpPkQBgysQF8a0=
819-
github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU=
820-
github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k=
821+
github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
822+
github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
821823
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
822-
github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw=
823-
github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI=
824-
github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE=
825-
github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc=
826-
github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=
827-
github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
824+
github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
825+
github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
826+
github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs=
827+
github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA=
828+
github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=
829+
github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
828830
github.com/prometheus/prometheus v0.46.0 h1:9JSdXnsuT6YsbODEhSQMwxNkGwPExfmzqG73vCMk/Kw=
829831
github.com/prometheus/prometheus v0.46.0/go.mod h1:10L5IJE5CEsjee1FnOcVswYXlPIscDWWt3IJ2UDYrz4=
830832
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM=
@@ -989,6 +991,8 @@ go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
989991
go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
990992
go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM=
991993
go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw=
994+
go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
995+
go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
992996
gocloud.dev v0.26.0/go.mod h1:mkUgejbnbLotorqDyvedJO20XcZNTynmSeVSQS9btVg=
993997
golang.org/x/crypto v0.0.0-20180723164146-c126467f60eb/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
994998
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=

‎pkg/inputs/flow/flow.go‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -108,8 +108,12 @@ func NewFlowSource(ctx context.Context, proto FlowSource, maxBatchSize int, log
108108
}
109109

110110
kt.SetConfig(config)
111+
configm, err := config.Compile()
112+
if err != nil {
113+
return nil, err
114+
}
111115

112-
flowProducer, err := protoproducer.CreateProtoProducer(config, protoproducer.CreateSamplingSystem)
116+
flowProducer, err := protoproducer.CreateProtoProducer(configm, protoproducer.CreateSamplingSystem)
113117
if err != nil {
114118
return nil, err
115119
}

‎pkg/inputs/flow/specials.go‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,16 @@ func loadDefault(cfg *ktranslate.FlowInputConfig) *producer.ProducerConfig {
1717
Index: 99,
1818
Type: "varint",
1919
},
20+
producer.ProtobufFormatterConfig{
21+
Name: "ingressVRFID",
22+
Index: 100,
23+
Type: "varint",
24+
},
25+
producer.ProtobufFormatterConfig{
26+
Name: "egressVRFID",
27+
Index: 101,
28+
Type: "varint",
29+
},
2030
},
2131
},
2232
IPFIX: producer.IPFIXProducerConfig{
@@ -25,6 +35,14 @@ func loadDefault(cfg *ktranslate.FlowInputConfig) *producer.ProducerConfig {
2535
Type: 61,
2636
Destination: "flow_direction",
2737
},
38+
producer.NetFlowMapField{
39+
Type: 234,
40+
Destination: "ingressVRFID",
41+
},
42+
producer.NetFlowMapField{
43+
Type: 235,
44+
Destination: "egressVRFID",
45+
},
2846
},
2947
},
3048
NetFlowV9: producer.NetFlowV9ProducerConfig{

0 commit comments

Comments
 (0)