Move trap/syslog filter code from EventManagerBase into new module.

This will aid with the Python 3 conversion.

ZEN-35265

Author: jpeacock-zenoss

src/Products/ZenEvents/EventManagerBase.py

@@ -0,0 +1,30 @@
+import unicodedata
+
+import six
+
+
+def load_trap_filters():
+    import os.path
+
+    path = os.path.join(os.path.dirname(__file__), "trap_filters.txt")
+    with open(path) as f:
+        return f.read()
+
+
+def load_syslog_parsers():
+    import json
+    import os.path
+
+    path = os.path.join(os.path.dirname(__file__), "syslog_parsers.json")
+    with open(path) as f:
+        return json.load(f, object_pairs_hook=_as_bytes)
+
+
+def _as_bytes(pairs):
+    return {str(k): _encode(v) for k, v in pairs}
+
+
+def _encode(value):
+    if isinstance(value, six.text_type):
+        return unicodedata.normalize("NFKD", value).encode("ascii", "ignore")
+    return value

src/Products/ZenEvents/filters.py

@@ -0,0 +1,87 @@
+[
+    {
+        "description": "generic mark",
+        "expr": "^(?P<summary>-- (?P<eventClassKey>MARK) --)",
+        "keep": true
+    },
+    {
+        "description": "Cisco UCS: 2010 Oct 19 15:47:45 CDT: snmpd: SNMP Operation (GET) failed. Reason:2 reqId (257790979) errno (42) error index (1)",
+        "expr": "^: \\d{4} \\w{3}\\s+\\d{1,2}\\s+\\d{1,2}:\\d\\d:\\d\\d \\w{3}: %(?P<eventClassKey>[^:]+): (?P<summary>.*)",
+        "keep": true
+    },
+    {
+        "description": "ntsyslog windows msg",
+        "expr": "^(?P<component>.+)\\[(?P<ntseverity>\\D+)\\] (?P<ntevid>\\d+) (?P<summary>.*)",
+        "keep": true
+    },
+    {
+        "description": "cisco msg with card indicator",
+        "expr": "%CARD-\\S+:(SLOT\\d+) %(?P<eventClassKey>\\S+): (?P<summary>.*)",
+        "keep": true
+    },
+    {
+        "description": "cisco standard msg",
+        "expr": "%(?P<eventClassKey>(?P<component>\\S+)-(?P<overwriteSeverity>\\d)-\\S+): *(?P<summary>.*)",
+        "keep": true
+    },
+    {
+        "description": "Cisco ACS",
+        "expr": "^(?P<ipAddress>\\S+)\\s+(?P<summary>(?P<eventClassKey>(CisACS_\\d\\d|CSCOacs)_\\S+)\\s+(?P<eventKey>\\S+)\\s.*)",
+        "keep": true
+    },
+    {
+        "description": "netscreen device msg",
+        "expr": "device_id=\\S+\\s+\\[\\S+\\](?P<eventClassKey>\\S+\\d+):\\s+(?P<summary>.*)\\s+\\((?P<originalTime>\\d\\d\\d\\d-\\d\\d-\\d\\d \\d\\d:\\d\\d:\\d\\d)\\)",
+        "keep": true
+    },
+    {
+        "description": "NetApp: [deviceName: 10/100/1000/e1a:warning]: Client 10.0.0.101 (xid 4251521131) is trying to access an unexported mount (fileid 64, snapid 0, generation 6111516 and flags 0x0 on volume 0xc97d89a [No volume name available])",
+        "expr": "^\\[[^:]+: (?P<component>[^:]+)[^\\]]+\\]: (?P<summary>.*)",
+        "keep": true
+    },
+    {
+        "description": "unix syslog with pid",
+        "expr": "(?P<component>\\S+)\\[(?P<pid>\\d+)\\]:\\s*(?P<summary>.*)",
+        "keep": true
+    },
+    {
+        "description": "unix syslog without pid",
+        "expr": "(?P<component>\\S+): (?P<summary>.*)",
+        "keep": true
+    },
+    {
+        "description": "adtran devices",
+        "expr": "^(?P<deviceModel>[^\\[]+)\\[(?P<deviceManufacturer>ADTRAN)\\]:(?P<component>[^\\|]+\\|\\d+\\|\\d+)\\|(?P<summary>.*)",
+        "keep": true
+    },
+    {
+        "description": "fortigate devices",
+        "expr": "^date=.+ (?P<summary>devname=.+ log_id=(?P<eventClassKey>\\d+) type=(?P<component>\\S+).+)",
+        "keep": true
+    },
+    {
+        "description": "proprietary message passing system",
+        "expr": "^(?P<component>\\S+)(\\.|\\s)[A-Z]{3} \\d \\S+ \\d\\d:\\d\\d:\\d\\d-\\d\\d:\\d\\d:\\d\\d \\d{5} \\d{2} \\d{5} \\S+ \\d{4} \\d{3,5} (- )*(?P<summary>.*) \\d{4} \\d{4}",
+        "keep": true
+    },
+    {
+        "description": "Cisco port state logging info",
+        "expr": "^Process (?P<process_id>\\d+), Nbr (?P<device>\\d+\\.\\d+\\.\\d+\\.\\d+) on (?P<interface>\\w+/\\d+) from (?P<start_state>\\w+) to (?P<end_state>\\w+), (?P<summary>.+)",
+        "keep": true
+    },
+    {
+        "description": "Cisco VPN Concentrator: 54884 05/25/2009 13:41:14.060 SEV=3 HTTP/42 RPT=4623 Error on socket accept.",
+        "expr": "^\\d+ \\d+\\/\\d+\\/\\d+ \\d+:\\d+:\\d+\\.\\d+ SEV=\\d+ (?P<eventClassKey>\\S+) RPT=\\d+ (?P<summary>.*)",
+        "keep": true
+    },
+    {
+        "description": "Dell Storage Array: 2626:48:VolExec:27-Aug-2009 13:15:58.072049:VE_VolSetWorker.hh:75:WARNING:43.3.2:Volume volumeName has reached 96 percent of its reported size and is currently using 492690MB.",
+        "expr": "^\\d+:\\d+:(?P<component>[^:]+):\\d+-\\w{3}-\\d{4} \\d{2}:\\d{2}:\\d{2}\\.\\d+:[^:]+:\\d+:\\w+:(?P<eventClassKey>[^:]+):(?P<summary>.*)",
+        "keep": true
+    },
+    {
+        "description": "1-Oct-2009 23:00:00.383809:snapshotDelete.cc:290:INFO:8.2.5:Successfully deleted snapshot \"UNVSQLCLUSTERTEMPDB-2009-09-30-23:00:14.11563\"",
+        "expr": "^\\d+-\\w{3}-\\d{4} \\d{2}:\\d{2}:\\d{2}\\.\\d+:[^:]+:\\d+:\\w+:(?P<eventClassKey>[^:]+):(?P<summary>.*)",
+        "keep": true
+    }
+]

src/Products/ZenEvents/syslog_parsers.json

@@ -0,0 +1,14 @@
+# Format: [COLLECTOR REGEX] include|exclude v1|v2 <version-specific options>
+# Include all generic SNMP V1 Traps 0-5
+include v1 0
+include v1 1
+include v1 2
+include v1 3
+include v1 4
+include v1 5
+
+# Include all enterprise-specific SNMP V1 traps
+include v1 *
+
+# Include all SNMP V2 traps
+include v2 *

src/Products/ZenEvents/trap_filters.txt

@@ -20,7 +20,7 @@
     Parsers,
     parse_MSG,
 )
-from Products.ZenEvents.EventManagerBase import EventManagerBase
+from Products.ZenEvents.filters import load_syslog_parsers
 
 
 class TestGetEventClassKeyValue(TestCase):
@@ -59,10 +59,12 @@ def test_default(t):
 
 
 class TestParseMSG(TestCase):
+    _syslog_parsers = load_syslog_parsers()
+
     def setUp(t):
         logging.getLogger().setLevel(logging.CRITICAL + 10)
         t.parsers = Parsers(t.sendEvent)
-        t.parsers.update(EventManagerBase.syslogParsers)
+        t.parsers.update(t._syslog_parsers)
 
     def tearDown(t):
         del t.parsers
@@ -90,7 +92,7 @@ def testCheckFortigate(t):
         key = "987654321"
         comp = "myComponent"
         msg = (
-            "date=xxxx devname=blue log_id={} type={} " "blah blah blah"
+            "date=xxxx devname=blue log_id={} type={} blah blah blah"
         ).format(key, comp)
         fields, index, drop = parse_MSG(msg, t.parsers)
         t.assertFalse(drop)

src/Products/ZenEvents/zensyslog/tests/test_processor.py

@@ -15,7 +15,6 @@
 
 from Products.ZenHub.interfaces import TRANSFORM_CONTINUE, TRANSFORM_DROP
 
-from Products.ZenEvents.EventManagerBase import EventManagerBase
 from Products.ZenEvents.zensyslog.transformer import (
     FilterRules,
     SyslogMsgFilter,
@@ -32,7 +31,7 @@ def tearDown(t):
     def testDefaultFilterRules(self):
         app = Mock()
         rules = FilterRules(app)
-        rules.update(EventManagerBase.syslogMsgEvtFieldFilterRules)
+        rules.update({})
         self.assertEquals(app.sendEvent.called, False)
 
     def testBadFilter(self):

src/Products/ZenEvents/zensyslog/tests/test_transformer.py

@@ -4,7 +4,7 @@
 from struct import pack
 from unittest import TestCase
 
-from ..decode import decode_snmp_value
+from Products.ZenEvents.zentrap.decode import decode_snmp_value
 
 
 class DecodersUnitTest(TestCase):

src/Products/ZenEvents/zentrap/tests/test_decode.py

@@ -15,7 +15,7 @@
 
 from unittest import TestCase
 
-from ..filterspec import (
+from Products.ZenEvents.zentrap.filterspec import (
     BaseFilterDefinition,
     FilterSpecification,
     GenericTrapFilterDefinition,

src/Products/ZenEvents/zentrap/tests/test_filterspec.py

@@ -6,10 +6,10 @@
 
 from mock import Mock
 
-from ..handlers import ReplayTrapHandler
-from ..net import FakePacket, SNMPv1, SNMPv2
-from ..oidmap import OidMap
-from ..processors import (
+from Products.ZenEvents.zentrap.handlers import ReplayTrapHandler
+from Products.ZenEvents.zentrap.net import FakePacket, SNMPv1, SNMPv2
+from Products.ZenEvents.zentrap.oidmap import OidMap
+from Products.ZenEvents.zentrap.processors import (
     LegacyVarbindProcessor,
     DirectVarbindProcessor,
     MixedVarbindProcessor,

src/Products/ZenEvents/zentrap/tests/test_handlers.py

@@ -4,7 +4,7 @@
 
 from mock import Mock
 
-from ..oidmap import OidMap
+from Products.ZenEvents.zentrap.oidmap import OidMap
 
 
 class TestOidMap(TestCase):