Skip to content

Commit d50825f

Browse files
akpwakpw
committed
Add Firewall Address Lists collector, #222 
1 parent e4bcf0c commit d50825f

8 files changed

Lines changed: 278 additions & 9 deletions

File tree

‎deploy/kubernetes/secret.yaml‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -67,12 +67,12 @@ stringData:
6767
6868
username = username # RouterOS user, needs to have 'read' and 'api' permissions
6969
password = password
70-
credentials_file = ""
70+
credentials_file = "" # To use an external file in YAML format for both username and password, specify the path here
7171
7272
use_ssl = False # enables connection via API-SSL servis
7373
no_ssl_certificate = False # enables API_SSL connect without router SSL certificate
7474
ssl_certificate_verify = False # turns SSL certificate verification on / off
75-
ssl_check_hostname = True # check if the hostname matches the peer cert’s hostname
75+
ssl_check_hostname = True # check if the hostname matches the peer cert’s hostname
7676
ssl_ca_file = "" # path to the certificate authority file to validate against, leave empty to use system store
7777
plaintext_login = True # for legacy RouterOS versions below 6.43 use False
7878
@@ -90,12 +90,14 @@ stringData:
9090
pool = True # IPv4 Pool metrics
9191
firewall = True # IPv4 Firewall rules traffic metrics
9292
neighbor = True # IPv4 Reachable Neighbors
93+
address_list = None # Firewall Address List metrics, a comma-separated list of names
9394
dns = False # DNS stats
9495
9596
ipv6_route = False # IPv6 Routes metrics
9697
ipv6_pool = False # IPv6 Pool metrics
9798
ipv6_firewall = False # IPv6 Firewall rules traffic metrics
9899
ipv6_neighbor = False # IPv6 Reachable Neighbors
100+
ipv6_address_list = None # IPv6 Firewall Address List metrics, a comma-separated list of names
99101
100102
poe = True # POE metrics
101103
monitor = True # Interface monitor metrics
@@ -121,11 +123,11 @@ stringData:
121123
122124
bfd = False # BFD sessions metrics
123125
bgp = False # BGP sessions metrics
124-
routing_stats = False # Routing process stats
126+
routing_stats = False # Routing process stats
125127
certificate = False # Certificates metrics
126128
127129
container = False # Containers metrics
128-
130+
129131
remote_dhcp_entry = None # An MKTXP entry to provide for remote DHCP info / resolution
130132
remote_capsman_entry = None # An MKTXP entry to provide for remote capsman info
131133

‎mktxp/cli/config/config.py‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ class CollectorKeys:
5454
GRE_COLLECTOR = 'GRECollector'
5555
IPIP_COLLECTOR = 'IPIPCollector'
5656
IPSEC_COLLECTOR = 'IPSecCollector'
57+
ADDRESS_LIST_COLLECTOR = 'AddressListCollector'
5758
LTE_COLLECTOR = 'LTECollector'
5859
SWITCH_PORT_COLLECTOR = 'SwitchPortCollector'
5960
MKTXP_COLLECTOR = 'MKTXPCollector'
@@ -91,12 +92,14 @@ class MKTXPConfigKeys:
9192
FE_ROUTE_KEY = 'route'
9293
FE_DHCP_POOL_KEY = 'pool'
9394
FE_FIREWALL_KEY = 'firewall'
95+
FE_ADDRESS_LIST_KEY = 'address_list'
9496
FE_NEIGHBOR_KEY = 'neighbor'
9597
FE_DNS_KEY = 'dns'
9698

9799
FE_IPV6_ROUTE_KEY = 'ipv6_route'
98100
FE_IPV6_DHCP_POOL_KEY = 'ipv6_pool'
99101
FE_IPV6_FIREWALL_KEY = 'ipv6_firewall'
102+
FE_IPV6_ADDRESS_LIST_KEY = 'ipv6_address_list'
100103
FE_IPV6_NEIGHBOR_KEY = 'ipv6_neighbor'
101104

102105
FE_MONITOR_KEY = 'monitor'
@@ -170,6 +173,8 @@ class MKTXPConfigKeys:
170173
DEFAULT_API_SSL_PORT = 8729
171174
DEFAULT_FE_REMOTE_DHCP_ENTRY = 'None'
172175
DEFAULT_FE_REMOTE_CAPSMAN_ENTRY = 'None'
176+
DEFAULT_FE_ADDRESS_LIST_KEY = 'None'
177+
DEFAULT_FE_IPV6_ADDRESS_LIST_KEY = 'None'
173178
DEFAULT_MKTXP_PORT = 49090
174179
DEFAULT_MKTXP_SOCKET_TIMEOUT = 2
175180
DEFAULT_MKTXP_INITIAL_DELAY = 120
@@ -195,7 +200,7 @@ class MKTXPConfigKeys:
195200
SYSTEM_BOOLEAN_KEYS_YES = {MKTXP_PERSISTENT_ROUTER_CONNECTION_POOL}
196201
SYSTEM_BOOLEAN_KEYS_NO = {MKTXP_BANDWIDTH_KEY, MKTXP_VERBOSE_MODE, MKTXP_FETCH_IN_PARALLEL, MKTXP_COMPACT_CONFIG, MKTXP_PROMETHEUS_HEADERS_DEDUPLICATION}
197202

198-
STR_KEYS = (HOST_KEY, USER_KEY, PASSWD_KEY, CREDENTIALS_FILE_KEY, SSL_CA_FILE, FE_REMOTE_DHCP_ENTRY, FE_REMOTE_CAPSMAN_ENTRY)
203+
STR_KEYS = (HOST_KEY, USER_KEY, PASSWD_KEY, CREDENTIALS_FILE_KEY, SSL_CA_FILE, FE_REMOTE_DHCP_ENTRY, FE_REMOTE_CAPSMAN_ENTRY, FE_ADDRESS_LIST_KEY, FE_IPV6_ADDRESS_LIST_KEY)
199204
INT_KEYS = ()
200205
MKTXP_INT_KEYS = (PORT_KEY, MKTXP_SOCKET_TIMEOUT, MKTXP_INITIAL_DELAY, MKTXP_MAX_DELAY,
201206
MKTXP_INC_DIV, MKTXP_BANDWIDTH_TEST_INTERVAL, MKTXP_MIN_COLLECT_INTERVAL,
@@ -216,8 +221,8 @@ class ConfigEntry:
216221
MKTXPConfigKeys.FE_MONITOR_KEY, MKTXPConfigKeys.FE_WIRELESS_KEY, MKTXPConfigKeys.FE_WIRELESS_CLIENTS_KEY,
217222
MKTXPConfigKeys.FE_IP_CONNECTIONS_KEY, MKTXPConfigKeys.FE_CONNECTION_STATS_KEY, MKTXPConfigKeys.FE_CAPSMAN_KEY, MKTXPConfigKeys.FE_CAPSMAN_CLIENTS_KEY, MKTXPConfigKeys.FE_POE_KEY,
218223
MKTXPConfigKeys.FE_NETWATCH_KEY, MKTXPConfigKeys.MKTXP_USE_COMMENTS_OVER_NAMES, MKTXPConfigKeys.FE_PUBLIC_IP_KEY,
219-
MKTXPConfigKeys.FE_ROUTE_KEY, MKTXPConfigKeys.FE_DHCP_POOL_KEY, MKTXPConfigKeys.FE_FIREWALL_KEY, MKTXPConfigKeys.FE_NEIGHBOR_KEY, MKTXPConfigKeys.FE_DNS_KEY,
220-
MKTXPConfigKeys.FE_IPV6_ROUTE_KEY, MKTXPConfigKeys.FE_IPV6_DHCP_POOL_KEY, MKTXPConfigKeys.FE_IPV6_FIREWALL_KEY, MKTXPConfigKeys.FE_IPV6_NEIGHBOR_KEY,
224+
MKTXPConfigKeys.FE_ROUTE_KEY, MKTXPConfigKeys.FE_DHCP_POOL_KEY, MKTXPConfigKeys.FE_FIREWALL_KEY, MKTXPConfigKeys.FE_ADDRESS_LIST_KEY, MKTXPConfigKeys.FE_NEIGHBOR_KEY, MKTXPConfigKeys.FE_DNS_KEY,
225+
MKTXPConfigKeys.FE_IPV6_ROUTE_KEY, MKTXPConfigKeys.FE_IPV6_DHCP_POOL_KEY, MKTXPConfigKeys.FE_IPV6_FIREWALL_KEY, MKTXPConfigKeys.FE_IPV6_ADDRESS_LIST_KEY, MKTXPConfigKeys.FE_IPV6_NEIGHBOR_KEY,
221226
MKTXPConfigKeys.FE_USER_KEY, MKTXPConfigKeys.FE_QUEUE_KEY, MKTXPConfigKeys.FE_REMOTE_DHCP_ENTRY, MKTXPConfigKeys.FE_REMOTE_CAPSMAN_ENTRY, MKTXPConfigKeys.FE_CHECK_FOR_UPDATES, MKTXPConfigKeys.FE_BFD_KEY, MKTXPConfigKeys.FE_BGP_KEY,
222227
MKTXPConfigKeys.FE_KID_CONTROL_DEVICE, MKTXPConfigKeys.FE_KID_CONTROL_DYNAMIC, MKTXPConfigKeys.FE_EOIP_KEY, MKTXPConfigKeys.FE_GRE_KEY, MKTXPConfigKeys.FE_IPIP_KEY, MKTXPConfigKeys.FE_LTE_KEY, MKTXPConfigKeys.FE_IPSEC_KEY, MKTXPConfigKeys.FE_SWITCH_PORT_KEY,
223228
MKTXPConfigKeys.FE_ROUTING_STATS_KEY, MKTXPConfigKeys.FE_CERTIFICATE_KEY, MKTXPConfigKeys.FE_CONTAINER_KEY
@@ -372,10 +377,10 @@ def _system_entry(self):
372377
def _read_from_disk(self):
373378
''' (Force-)Read conf data from disk
374379
'''
375-
self.config = ConfigObj(self.usr_conf_data_path, indent_type = ' ')
380+
self.config = ConfigObj(self.usr_conf_data_path, indent_type = ' ', encoding='utf-8')
376381
self.config.preserve_comments = True
377382

378-
self._config = ConfigObj(self.mktxp_conf_path, indent_type = ' ')
383+
self._config = ConfigObj(self.mktxp_conf_path, indent_type = ' ', encoding='utf-8')
379384
self._config.preserve_comments = True
380385

381386
def _create_os_path(self, os_path, resource_path):
@@ -573,6 +578,8 @@ def _default_value_for_key(self, key, value=None):
573578
MKTXPConfigKeys.SSL_CA_FILE: lambda _: MKTXPConfigKeys.DEFAULT_SSL_CA_FILE,
574579
MKTXPConfigKeys.FE_REMOTE_DHCP_ENTRY: lambda _: MKTXPConfigKeys.DEFAULT_FE_REMOTE_DHCP_ENTRY,
575580
MKTXPConfigKeys.FE_REMOTE_CAPSMAN_ENTRY: lambda _: MKTXPConfigKeys.DEFAULT_FE_REMOTE_CAPSMAN_ENTRY,
581+
MKTXPConfigKeys.FE_ADDRESS_LIST_KEY: lambda _: MKTXPConfigKeys.DEFAULT_FE_ADDRESS_LIST_KEY,
582+
MKTXPConfigKeys.FE_IPV6_ADDRESS_LIST_KEY: lambda _: MKTXPConfigKeys.DEFAULT_FE_IPV6_ADDRESS_LIST_KEY,
576583
MKTXPConfigKeys.MKTXP_SOCKET_TIMEOUT: lambda _: MKTXPConfigKeys.DEFAULT_MKTXP_SOCKET_TIMEOUT,
577584
MKTXPConfigKeys.MKTXP_INITIAL_DELAY: lambda _: MKTXPConfigKeys.DEFAULT_MKTXP_INITIAL_DELAY,
578585
MKTXPConfigKeys.MKTXP_MAX_DELAY: lambda _: MKTXPConfigKeys.DEFAULT_MKTXP_MAX_DELAY,

‎mktxp/cli/config/mktxp.conf‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,12 +47,14 @@
4747
pool = True # IPv4 Pool metrics
4848
firewall = True # IPv4 Firewall rules traffic metrics
4949
neighbor = True # IPv4 Reachable Neighbors
50+
address_list = None # Firewall Address List metrics, a comma-separated list of names
5051
dns = False # DNS stats
5152

5253
ipv6_route = False # IPv6 Routes metrics
5354
ipv6_pool = False # IPv6 Pool metrics
5455
ipv6_firewall = False # IPv6 Firewall rules traffic metrics
5556
ipv6_neighbor = False # IPv6 Reachable Neighbors
57+
ipv6_address_list = None # IPv6 Firewall Address List metrics, a comma-separated list of names
5658

5759
poe = True # POE metrics
5860
monitor = True # Interface monitor metrics

‎mktxp/collector/address_list_collector.py‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
# coding=utf8
2+
## Copyright (c) 2020 Arseniy Kuznetsov
3+
##
4+
## This program is free software; you can redistribute it and/or
5+
## modify it under the terms of the GNU General Public License
6+
## as published by the Free Software Foundation; either version 2
7+
## of the License, or (at your option) any later version.
8+
##
9+
## This program is distributed in the hope that it will be useful,
10+
## but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
## GNU General Public License for more details.
13+
14+
15+
from mktxp.collector.base_collector import BaseCollector
16+
from mktxp.flow.processor.output import BaseOutputProcessor
17+
from mktxp.cli.config.config import MKTXPConfigKeys
18+
from mktxp.datasource.address_list_ds import AddressListMetricsDataSource
19+
from collections import defaultdict
20+
21+
class AddressListCollector(BaseCollector):
22+
'''Address List collector'''
23+
24+
@staticmethod
25+
def collect(router_entry):
26+
metric_labels = ['list', 'address', 'dynamic', 'timeout', 'disabled', 'comment']
27+
reduced_metric_labels = ['list', 'address', 'dynamic', 'disabled', 'comment']
28+
translation_table = {
29+
'dynamic': lambda value: '1' if value == 'true' else '0',
30+
'disabled': lambda value: '1' if value == 'true' else '0',
31+
'timeout': lambda value: BaseOutputProcessor.parse_timedelta_milliseconds(value) if value else '0',
32+
'comment': lambda value: value if value else ''
33+
}
34+
35+
# IPv4
36+
address_list_names = AddressListCollector._get_list_names(router_entry.config_entry.address_list)
37+
if address_list_names:
38+
ipv4_records = AddressListMetricsDataSource.metric_records(
39+
router_entry,
40+
address_list_names,
41+
'ip',
42+
metric_labels = metric_labels,
43+
translation_table = translation_table
44+
)
45+
if ipv4_records:
46+
yield BaseCollector.gauge_collector('firewall_address_list', 'IPv4 Firewall Address List Entry',
47+
ipv4_records, 'timeout', reduced_metric_labels)
48+
49+
count_records = AddressListCollector._get_count_records(router_entry, ipv4_records)
50+
yield BaseCollector.gauge_collector('firewall_address_list_entries_count', 'Number of entries in a IPv4 firewall address list',
51+
count_records, 'count', ['list', MKTXPConfigKeys.ROUTERBOARD_NAME, MKTXPConfigKeys.ROUTERBOARD_ADDRESS])
52+
53+
# IPv6
54+
ipv6_address_list_names = AddressListCollector._get_list_names(router_entry.config_entry.ipv6_address_list)
55+
if ipv6_address_list_names:
56+
ipv6_records = AddressListMetricsDataSource.metric_records(
57+
router_entry,
58+
ipv6_address_list_names,
59+
'ipv6',
60+
metric_labels=metric_labels,
61+
translation_table = translation_table
62+
)
63+
if ipv6_records:
64+
yield BaseCollector.gauge_collector('firewall_address_list_ipv6', 'Firewall IPv6 Address List Entry',
65+
ipv6_records, 'timeout', reduced_metric_labels)
66+
67+
count_records = AddressListCollector._get_count_records(router_entry, ipv6_records)
68+
yield BaseCollector.gauge_collector('firewall_address_list_entries_count_ipv6', 'Number of entries in a IPv6 firewall address list',
69+
count_records, 'count', ['list', MKTXPConfigKeys.ROUTERBOARD_NAME, MKTXPConfigKeys.ROUTERBOARD_ADDRESS])
70+
71+
@staticmethod
72+
def _get_list_names(config_value):
73+
if not config_value:
74+
return []
75+
if isinstance(config_value, str):
76+
if config_value.lower() == 'none':
77+
return []
78+
return [name.strip() for name in config_value.split(',') if name.strip()]
79+
if isinstance(config_value, list):
80+
return [name for name in config_value if name]
81+
return []
82+
83+
@staticmethod
84+
def _get_count_records(router_entry, records):
85+
counts = defaultdict(int)
86+
for record in records:
87+
counts[record['list']] += 1
88+
89+
count_records = []
90+
for list_name, count in counts.items():
91+
count_records.append({
92+
MKTXPConfigKeys.ROUTERBOARD_NAME: router_entry.router_id[MKTXPConfigKeys.ROUTERBOARD_NAME],
93+
MKTXPConfigKeys.ROUTERBOARD_ADDRESS: router_entry.router_id[MKTXPConfigKeys.ROUTERBOARD_ADDRESS],
94+
'list': list_name, 'count': count})
95+
return count_records

‎mktxp/datasource/address_list_ds.py‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
# coding=utf8
2+
## Copyright (c) 2020 Arseniy Kuznetsov
3+
##
4+
## This program is free software; you can redistribute it and/or
5+
## modify it under the terms of the GNU General Public License
6+
## as published by the Free Software Foundation; either version 2
7+
## of the License, or (at your option) any later version.
8+
##
9+
## This program is distributed in the hope that it will be useful,
10+
## but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
## GNU General Public License for more details.
13+
14+
15+
from mktxp.datasource.base_ds import BaseDSProcessor
16+
17+
18+
class AddressListMetricsDataSource:
19+
"""Address List Metrics data provider"""
20+
@staticmethod
21+
def metric_records(router_entry, address_lists, ip_version, *, metric_labels=None, translation_table=None):
22+
if metric_labels is None:
23+
metric_labels = []
24+
25+
all_records = []
26+
try:
27+
path = f"/{ip_version}/firewall/address-list"
28+
resource = router_entry.api_connection.router_api().get_resource(path)
29+
for list_name in address_lists:
30+
records = resource.get(list=list_name)
31+
all_records.extend(records)
32+
33+
return BaseDSProcessor.trimmed_records(router_entry, router_records=all_records,
34+
metric_labels=metric_labels, translation_table=translation_table)
35+
except Exception as exc:
36+
print(f'Error getting Address List info from router {router_entry.router_name}@{router_entry.config_entry.hostname}: {exc}')
37+
return None

‎mktxp/flow/collector_registry.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
from mktxp.collector.health_collector import HealthCollector
2222
from mktxp.collector.identity_collector import IdentityCollector
2323
from mktxp.collector.ipsec_collector import IPSecCollector
24+
from mktxp.collector.address_list_collector import AddressListCollector
2425
from mktxp.collector.public_ip_collector import PublicIPAddressCollector
2526
from mktxp.collector.neighbor_collector import NeighborCollector
2627
from mktxp.collector.monitor_collector import MonitorCollector
@@ -71,6 +72,7 @@ def __init__(self):
7172
self.register(CollectorKeys.DHCP_COLLECTOR, DHCPCollector.collect)
7273
self.register(CollectorKeys.IP_CONNECTION_COLLECTOR, IPConnectionCollector.collect)
7374
self.register(CollectorKeys.IPSEC_COLLECTOR, IPSecCollector.collect)
75+
self.register(CollectorKeys.ADDRESS_LIST_COLLECTOR, AddressListCollector.collect)
7476
self.register(CollectorKeys.POOL_COLLECTOR, PoolCollector.collect)
7577
self.register(CollectorKeys.INTERFACE_COLLECTOR, InterfaceCollector.collect)
7678

‎mktxp/flow/router_entry.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ def __init__(self, router_name):
6161
CollectorKeys.POOL_COLLECTOR: 0,
6262
CollectorKeys.IP_CONNECTION_COLLECTOR: 0,
6363
CollectorKeys.IPSEC_COLLECTOR: 0,
64+
CollectorKeys.ADDRESS_LIST_COLLECTOR: 0,
6465
CollectorKeys.INTERFACE_COLLECTOR: 0,
6566
CollectorKeys.FIREWALL_COLLECTOR: 0,
6667
CollectorKeys.MONITOR_COLLECTOR: 0,

0 commit comments

Comments
 (0)