diff --git a/core/services/modules/shadowbook/extendTools.py b/core/services/modules/shadowbook/extendTools.py index db03304..77d0068 100644 --- a/core/services/modules/shadowbook/extendTools.py +++ b/core/services/modules/shadowbook/extendTools.py @@ -3,9 +3,26 @@ from itertools import imap, groupby from operator import itemgetter +from re import compile +from netaddr import IPNetwork + +CIDR_re = compile(r'^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/([0-9]|[1-2][0-9]|3[0-2]))$') def unique_justseen(iterable, key=None): "List unique elements, preserving order. Remember only the element just seen." # unique_justseen('AAAABBBCCDAABBB') --> A B C D A B # unique_justseen('ABBCcAD', str.lower) --> A B C A D return imap(next, imap(itemgetter(1), groupby(iterable, key))) + +def transform_cidr(iterable, key=None): + iterable_transformed = [] + + for dict in iterable: + if CIDR_re.match(dict.get(key)): + for ip in IPNetwork(dict.get(key)): + dict['ip'] = str(ip) + iterable_transformed.append(dict) + else: + iterable_transformed.append(dict) + + return iterable_transformed diff --git a/core/services/modules/shadowbook/processFeed.py b/core/services/modules/shadowbook/processFeed.py index 46ead75..59bd8e8 100644 --- a/core/services/modules/shadowbook/processFeed.py +++ b/core/services/modules/shadowbook/processFeed.py @@ -60,6 +60,8 @@ def main(listSessions, cfgPath): #the function will return a csvDictReader which we convert to dict afterward parsedPage = csvParser(feedPage, cfgPath) parsedPage = list(extendTools.unique_justseen(list(parsedPage), key = operator.itemgetter(source.coreIntelligence))) + if source.coreIntelligence == 'ip': + parsedPage = extendTools.transform_cidr(parsedPage, key = source.coreIntelligence) #parsedPage will look like: #[{'domain': 'stie.pbsoedirman.com', 'original_reference-why_it_was_listed': 'spamhaus.org', u'extra': ['20160324'], 'nextvalidation': '', 'blank': '', 'type': 'malware'}, {'domain': 'thecommercialalliance.com', 'original_reference-why_it_was_listed': 'spamhaus.org', u'extra': ['20160324'], 'nextvalidation': '', 'blank': '', 'type': 'botnet'}]