Skip to content

Commit 47d250b

Browse files
grafoleangrafolean
committed
Merge branch 'feature/vars-in-output-path' into 'master'
Support for vars in output path template See merge request grafolean/grafolean-collector-snmp!9
2 parents 8baac34 + c596e60 commit 47d250b

File tree

4 files changed

+166
-43
lines changed

4 files changed

+166
-43
lines changed

‎Pipfile

+1
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ easysnmp = {editable = true,git = "http://github.com/grafolean/easysnmp"}
1616
mathjspy = "*"
1717
numpy = "*"
1818
redis = "*"
19+
python-slugify = "*"
1920

2021
[requires]
2122
python_version = "3.6"

‎Pipfile.lock

+54-34
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎snmpcollector.py

+56-4
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,11 @@
77
from colors import color
88
import requests
99
import redis
10+
import re
1011

1112
from easysnmp import Session, SNMPVariable
1213
from mathjspy import MathJS
14+
from slugify import slugify
1315

1416
from collector import Collector
1517

@@ -27,6 +29,10 @@ class NoValueForOid(Exception):
2729
pass
2830

2931

32+
class InvalidOutputPath(Exception):
33+
pass
34+
35+
3036
REDIS_HOST = os.environ.get('REDIS_HOST', '127.0.0.1')
3137
r = redis.Redis(host=REDIS_HOST)
3238

@@ -72,7 +78,38 @@ def _convert_counters_to_values(results, now, counter_ident_prefix):
7278
return new_results
7379

7480

75-
def _apply_expression_to_results(snmp_results, methods, expression, output_path):
81+
def _construct_output_path(template, addressable_results, oid_index):
82+
# make sure that only valid characters are in the template:
83+
if not re.match(r'^([.0-9a-zA-Z_-]+|[{][^}]+[}])+$', template):
84+
raise InvalidOutputPath("Invalid output path template, could not parse")
85+
result_parts = []
86+
for between_dots in template.split('.'):
87+
OUTPUT_PATH_REGEX = r'([0-9a-zA-Z_-]+|[{][^}]+[}])' # split parts with curly braces from those without
88+
for part in re.findall(OUTPUT_PATH_REGEX, between_dots):
89+
if part[0] != '{':
90+
result_parts.append(part)
91+
continue
92+
# expression parsing is currently a bit limited, we only replace {$1} to {$N} and {$index}
93+
if part[1] != '$':
94+
raise InvalidOutputPath("Only simple substitutions are currently supported (like 'abc.{$2}.{$index}.def') - was expecting '$' after '{'.")
95+
expression = part[2:-1]
96+
if expression == 'index':
97+
result_parts.append(oid_index)
98+
else:
99+
if not expression.isdigit():
100+
raise InvalidOutputPath("Only simple substitutions are currently supported (like 'abc.{$2}.{$index}.def') - was expecting either 'index' or a number after '$'.")
101+
i = int(expression) - 1
102+
if not 0 <= i < len(addressable_results):
103+
raise InvalidOutputPath(f"Could not create output path - the number after '$' should be between 1 and {len(addressable_results)} inclusive.")
104+
v = addressable_results[i][oid_index]
105+
clean_value = slugify(v.value, regex_pattern=r'[^0-9a-zA-Z_-]+', lowercase=False)
106+
result_parts.append(clean_value)
107+
108+
result_parts.append('.')
109+
return ''.join(result_parts)[:-1]
110+
111+
112+
def _apply_expression_to_results(snmp_results, methods, expression, output_path_template):
76113
if 'walk' in methods:
77114
"""
78115
- determine which oid indexes are used
@@ -91,6 +128,7 @@ def _apply_expression_to_results(snmp_results, methods, expression, output_path)
91128

92129
result = []
93130
for oid_index in walk_indexes:
131+
known_output_paths = set()
94132
try:
95133
mjs = MathJS()
96134
for i, r in enumerate(addressable_results):
@@ -99,24 +137,38 @@ def _apply_expression_to_results(snmp_results, methods, expression, output_path)
99137
raise NoValueForOid()
100138
if v.value is None: # no value (probably the first time we're asking for a counter)
101139
raise NoValueForOid()
102-
mjs.set('${}'.format(i + 1), float(v.value))
140+
var_name = f'${i + 1}'
141+
if var_name in expression: # not all values are used - some might be used by output_path
142+
mjs.set(var_name, float(v.value))
103143
value = mjs.eval(expression)
144+
145+
output_path = _construct_output_path(output_path_template, addressable_results, oid_index)
146+
if output_path in known_output_paths:
147+
raise InvalidOutputPath("The same path was already constructed from a previous result, please include {$index} in the output path template, or make sure it is unique!")
148+
known_output_paths.add(output_path)
104149
result.append({
105-
'p': f'{output_path}.{oid_index}',
150+
'p': output_path,
106151
'v': value,
107152
})
108153
except NoValueForOid:
109154
log.warning(f'Missing value for oid index: {oid_index}')
155+
except InvalidOutputPath as ex:
156+
log.warning(f'Invalid output path for oid index [{oid_index}]: {str(ex)}')
110157
return result
111158

112159
else:
113160
try:
161+
dummy_oid_index = '0'
162+
addressable_results = [{dummy_oid_index: v} for v in snmp_results]
114163
mjs = MathJS()
115164
for i, v in enumerate(snmp_results):
116165
if v.value is None: # no value (probably the first time we're asking for a counter)
117166
raise NoValueForOid()
118-
mjs.set('${}'.format(i + 1), float(v.value))
167+
var_name = f'${i + 1}'
168+
if var_name in expression: # not all values are used - some might be used by output_path
169+
mjs.set(var_name, float(v.value))
119170
value = mjs.eval(expression)
171+
output_path = _construct_output_path(output_path_template, addressable_results, dummy_oid_index)
120172
return [
121173
{'p': output_path, 'v': value},
122174
]

‎test_snmpcollector.py

+55-5
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
from easysnmp import SNMPVariable
2+
import pytest
23

3-
from snmpcollector import _apply_expression_to_results, _convert_counters_to_values
4+
from snmpcollector import _apply_expression_to_results, _convert_counters_to_values, _construct_output_path
45

56

67
def test_apply_expression_snmpget():
@@ -38,7 +39,7 @@ def test_apply_expression_snmpwalk():
3839
]
3940
methods = ['walk' if isinstance(x, list) else 'get' for x in results]
4041
expression = '$1'
41-
output_path = 'snmp.test123.asdf'
42+
output_path = 'snmp.test123.asdf.{$index}'
4243
expected_result = [
4344
{ 'p': 'snmp.test123.asdf.1', 'v': 60000.0 },
4445
{ 'p': 'snmp.test123.asdf.2', 'v': 61000.0 },
@@ -57,7 +58,7 @@ def test_apply_expression_expression_add():
5758
]
5859
methods = ['walk' if isinstance(x, list) else 'get' for x in results]
5960
expression = '$1 + $2'
60-
output_path = 'snmp.test123.asdf'
61+
output_path = 'snmp.test123.asdf.{$index}'
6162
expected_result = [
6263
{ 'p': 'snmp.test123.asdf.1', 'v': 60500.0 },
6364
{ 'p': 'snmp.test123.asdf.2', 'v': 61500.0 },
@@ -79,7 +80,7 @@ def test_apply_expression_snmpwalk_missing_value_walk():
7980
]
8081
methods = ['walk' if isinstance(x, list) else 'get' for x in results]
8182
expression = '$1 / $2'
82-
output_path = 'snmp.test123.asdf'
83+
output_path = 'snmp.test123.asdf.{$index}'
8384
expected_result = [
8485
{ 'p': 'snmp.test123.asdf.1', 'v': 6000.0 },
8586
{ 'p': 'snmp.test123.asdf.2', 'v': 6100.0 },
@@ -97,6 +98,19 @@ def test_apply_expression_snmpwalk_missing_value_get():
9798
expected_result = []
9899
assert _apply_expression_to_results(results, methods, expression, output_path) == expected_result
99100

101+
def test_apply_expression_snmpget_output_path():
102+
results = [
103+
SNMPVariable(oid='1.3.6.1.4.1.2021.13.16.2.1.3', oid_index='1', value='60000', snmp_type='GAUGE'),
104+
SNMPVariable(oid='1.3.6.1.4.1.2021.13.16.2.2.2', oid_index='1', value='asdf.QWER', snmp_type='STRING'),
105+
]
106+
methods = ['walk' if isinstance(x, list) else 'get' for x in results]
107+
expression = '$1'
108+
output_path = 'snmp.{$2}.aaa{$2}bbb.asdf'
109+
expected_result = [
110+
{ 'p': 'snmp.asdf-QWER.aaaasdf-QWERbbb.asdf', 'v': 60000.0 },
111+
]
112+
assert _apply_expression_to_results(results, methods, expression, output_path) == expected_result
113+
100114
def test_convert_counters_no_counters_no_change():
101115
""" If there are no counters, nothing should change """
102116
now = 1234567890.123456
@@ -130,7 +144,7 @@ def test_convert_counters_counter():
130144
assert _convert_counters_to_values(results_2, now + 1.0 + 3.0, "ASDF/1234") == expected_2
131145

132146
def test_convert_counters_overflow():
133-
""" First expression should be empty, next ones should work """
147+
""" Counter overflow should cause value to be discarded """
134148
now = 1234567890.123456
135149

136150
results_0 = [SNMPVariable(oid='.1.3.6.1.2.1.2.2.1.17', oid_index='1', value='123000.0', snmp_type='COUNTER')]
@@ -148,3 +162,39 @@ def test_convert_counters_overflow():
148162
results_3 = [SNMPVariable(oid='.1.3.6.1.2.1.2.2.1.17', oid_index='1', value='2000.0', snmp_type='COUNTER')]
149163
expected_3 = [SNMPVariable(oid='.1.3.6.1.2.1.2.2.1.17', oid_index='1', value='500.0', snmp_type='COUNTER_PER_S')]
150164
assert _convert_counters_to_values(results_3, now + 1.0 + 3.0 + 2.0, "ASDF/1234") == expected_3
165+
166+
167+
output_path_test_results_get = [
168+
{'0': SNMPVariable(oid='.1.3.6.1.2.1.2.2.1.13', oid_index='0', value='123000.0', snmp_type='COUNTER')},
169+
{'0': SNMPVariable(oid='.1.3.6.1.2.1.2.2.1.14', oid_index='0', value='aaa', snmp_type='STRING')},
170+
{'0': SNMPVariable(oid='.1.3.6.1.2.1.2.2.1.15', oid_index='0', value='Core 6', snmp_type='STRING')},
171+
{'0': SNMPVariable(oid='.1.3.6.1.2.1.2.2.1.16', oid_index='0', value='6', snmp_type='GAUGE')},
172+
]
173+
output_path_test_results_walk = [
174+
{
175+
'1': SNMPVariable(oid='.1.3.6.1.2.1.2.2.1.13', oid_index='1', value='123000.0', snmp_type='COUNTER'),
176+
'2': SNMPVariable(oid='.1.3.6.1.2.1.2.2.1.13', oid_index='2', value='234000.0', snmp_type='COUNTER'),
177+
},
178+
{
179+
'1': SNMPVariable(oid='.1.3.6.1.2.1.2.2.1.14', oid_index='1', value='WWW', snmp_type='STRING'),
180+
'2': SNMPVariable(oid='.1.3.6.1.2.1.2.2.1.14', oid_index='2', value='qqq', snmp_type='STRING'),
181+
},
182+
{
183+
'1': SNMPVariable(oid='.1.3.6.1.2.1.2.2.1.15', oid_index='1', value='Core-3', snmp_type='STRING'),
184+
'2': SNMPVariable(oid='.1.3.6.1.2.1.2.2.1.15', oid_index='2', value='Core 6', snmp_type='STRING'),
185+
},
186+
{
187+
'1': SNMPVariable(oid='.1.3.6.1.2.1.2.2.1.16', oid_index='0', value='6', snmp_type='GAUGE'), # this value came from SNMP GET
188+
'2': SNMPVariable(oid='.1.3.6.1.2.1.2.2.1.16', oid_index='0', value='6', snmp_type='GAUGE'),
189+
},
190+
]
191+
@pytest.mark.parametrize("template,addressable_results,oid_index,expected", [
192+
('123{$2}dd.{$index}BBB', output_path_test_results_get, '0', '123aaadd.0BBB',),
193+
('123{$2}dd.{$index}BBB', output_path_test_results_walk, '1', '123WWWdd.1BBB',),
194+
('123{$2}dd.{$index}BBB', output_path_test_results_walk, '2', '123qqqdd.2BBB',),
195+
('asdf.123.{$3}', output_path_test_results_walk, '2', 'asdf.123.Core-6',),
196+
('Asdf.123.{$3}.{$index}', output_path_test_results_walk, '2', 'Asdf.123.Core-6.2',),
197+
('{$3}.{$index}', output_path_test_results_walk, '1', 'Core-3.1',), # '.' gets replaced by '-'
198+
])
199+
def test_construct_output_path(template, addressable_results, oid_index, expected):
200+
assert expected == _construct_output_path(template, addressable_results, oid_index)

0 commit comments

Comments
 (0)