Skip to content

Commit 61920ba

Browse files
author
kien
committed
feat: check ticket data
1 parent 7afa78f commit 61920ba

10 files changed

+368
-131
lines changed

‎Dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ WORKDIR /unsupervised_log_detection
44
COPY ./requirement.txt /tmp/requirement.txt
55
RUN pip3 install --no-cache-dir -r /tmp/requirement.txt
66
COPY . .
7-
RUN pip3 install /unsupervised_log_detection/syslog_rust-0.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl rich
7+
RUN pip3 install /unsupervised_log_detection/syslog_rust-0.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl rich /unsupervised_log_detection/tag_template_rust-0.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
88
RUN sed -i 's/self\.max_buffer_size = max_buffer_size or 104857600/self\.max_buffer_size = max_buffer_size or 2097152000/' /usr/local/lib/python3.9/site-packages/tornado/iostream.py
99
EXPOSE 3001
1010
CMD ["panel", "serve", "/unsupervised_log_detection/app_panel.py", "--address", "0.0.0.0", "--port", "3001", "--websocket-max-message-size", "2097152000", "--allow-websocket-origin","*"]

‎app_panel.py‎

Lines changed: 220 additions & 128 deletions
Large diffs are not rendered by default.

‎model_26_Dec_2024/dictionary/template_dictionary.dict‎

67.3 KB
Binary file not shown.

‎model_26_Dec_2024/model/template_lda.model‎

14.6 KB
Binary file not shown.

‎model_26_Dec_2024/model/template_lda.model.expElogbeta.npy‎

3.54 MB
Binary file not shown.

‎model_26_Dec_2024/model/template_lda.model.id2word‎

67.3 KB
Binary file not shown.

‎model_26_Dec_2024/model/template_lda.model.state‎

3.55 MB
Binary file not shown.

‎preprocessing.py‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,11 @@
22
import pandas as pd
33
import numpy as np
44
import gensim
5-
from nltk.stem import WordNetLemmatizer, SnowballStemmer
5+
from nltk.stem import WordNetLemmatizer
66
from nltk.stem.porter import *
77
import math
88
from functools import lru_cache
9+
import preprocessing_dict
910

1011

1112
@lru_cache(maxsize=10000)
@@ -15,7 +16,11 @@ def lemmatize_stemming(text):
1516
def preprocess(text):
1617
result = []
1718
for token in gensim.utils.simple_preprocess(text):
18-
result.append(lemmatize_stemming(token))
19+
if token not in preprocessing_dict.specific_words:
20+
result.append(lemmatize_stemming(token))
21+
else:
22+
for tok in gensim.utils.simple_preprocess(preprocessing_dict.specific_words[token]):
23+
result.append(lemmatize_stemming(tok))
1924
return result
2025

2126
def calculate_score(data, vectorizer, entropy):

‎preprocessing_dict.py‎

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
specific_words = {
2+
'del': 'delete',
3+
'err': 'error',
4+
'pkt': 'packet',
5+
'nh': 'next hop',
6+
'aggr': 'aggregate',
7+
'ucast': 'unicast',
8+
'auth': 'authentication',
9+
'sendto': 'send to',
10+
'nh-id': 'next hop id',
11+
'obj': 'object',
12+
'dyn': 'dynamic',
13+
'phy': 'physical',
14+
'addr': 'address',
15+
'rev': 'revision',
16+
'msg': 'message',
17+
'var': 'variable',
18+
'diag': 'diagnostic',
19+
'kern': 'kernel',
20+
'sndcc': 'send buffer size in bytes',
21+
'rcvcc': 'receive buffer size in bytes',
22+
'snd': 'send',
23+
'una': 'unacknowledged',
24+
'nxt': 'next',
25+
'wnd': 'window',
26+
'rcv': 'receive',
27+
'adv': 'advertised',
28+
'hw': 'hardware',
29+
'tnl': 'tunnel',
30+
'exp': 'experimental',
31+
'dscp': 'differentiated services code point',
32+
'rt': 'route',
33+
'rtt': 'route trip time',
34+
'dc': 'destination class',
35+
'sc': 'source class',
36+
'rpf': 'reverse path forwarding',
37+
'cos': 'class of service',
38+
'dfw': 'dynamic firewall',
39+
'stat': 'statistic',
40+
'ctl': 'control',
41+
'h/w': 'hardware',
42+
'ptr': 'pointer',
43+
'qh': 'queue head',
44+
'conn': 'connection',
45+
'inactivesqtd': 'inactive sqtd',
46+
'param': 'parameter',
47+
'params': 'parameters',
48+
'recv': 'receive',
49+
'rnh': 'routing next hop',
50+
'ptp': 'precision time protocol',
51+
'rpm': 'Real Time Performance Monitoring',
52+
'rts': 'request to send',
53+
'proc': 'process',
54+
'idl': 'interconnect description language',
55+
'rtsock': 'routing socket',
56+
'intf': 'interface',
57+
'trans': 'transaction',
58+
'ifl': 'logical interface',
59+
'dfwdlib': 'dfwd library',
60+
'altnext': 'alternative next',
61+
'ifd': 'interface device',
62+
'sntpsync': 'simple network time protocol sync',
63+
'qtd': 'Queue Element Transfer Descriptor',
64+
'rtc': 'real time clock',
65+
'ch': 'channel',
66+
'serv': 'service',
67+
'idx': 'index',
68+
'ifp': 'ingress filter processor',
69+
'tcam': 'ternary content addressable memory',
70+
'ack': 'acknowledge',
71+
'NNI': 'network to network interface',
72+
'pw-ifl': 'pseudowire logical interface',
73+
'if_pfe': 'interface pfe',
74+
'ifstate': 'interface state',
75+
'chk': 'check',
76+
'fw': 'firewall',
77+
'ifdpointchange': 'interface device point change',
78+
'tlv': 'type length value',
79+
'fwd': 'forward',
80+
'len': 'length',
81+
'bundlestate': 'bundle state',
82+
'bd': 'bundle',
83+
'inact': 'inactive',
84+
'endpt': 'endpoint',
85+
'eps': 'event per second',
86+
'dtc': 'distributed transaction coordinator',
87+
'cfg': 'config',
88+
'cerr': 'content error',
89+
'tdm': 'time division multiplexing',
90+
'inst': 'instance',
91+
'intfs': 'interfaces',
92+
'errmsg': 'error message',
93+
'sw': 'switch',
94+
'src': 'source',
95+
'dst': 'destination',
96+
'cmlc': 'chassis manager line card',
97+
'pkts': 'packets',
98+
'res': 'resolution',
99+
'desc': 'description',
100+
'tx': 'tranѕmitter',
101+
'rx': 'receiᴠer',
102+
'nak': 'not acknowledged',
103+
'dma': 'direct memory access',
104+
'na': 'neighbour advertisement',
105+
'ns': 'neighbour solicitation',
106+
'cfm': 'connectivity fault management',
107+
'reg': 'register',
108+
'jnh': 'junos next hop',
109+
'prl': 'packet replication layer',
110+
'acct': 'accounting',
111+
'pam': 'pluggable authentication module',
112+
'sm': 'security module',
113+
'dfe': 'decision feedback equalizer',
114+
'fbuf': 'firewall buffer',
115+
'vc': 'virtual circuit',
116+
'bw': 'bandwidth',
117+
'req': 'request',
118+
'ql': 'quality level',
119+
'pwr': 'power',
120+
'num': 'number',
121+
'putblk': 'put block',
122+
'iftlv': 'interface type length value',
123+
'nlri': 'Network layer reachability information',
124+
'rhost': 'routing host',
125+
'sysctlbyname': 'system control by name',
126+
'mhop': 'multi hop',
127+
'krt': 'kernel routing table',
128+
'rttable': 'routing table',
129+
'ipt': 'ip table',
130+
'sec': 'secondary',
131+
'rtb': 'route table',
132+
'vrf': 'virtual routing forwarding',
133+
'setsockopt': 'set socket operation',
134+
'sched': 'scheduler',
135+
'dyncfg': 'dynamic configugration',
136+
'nbr': 'neighbour',
137+
'ifdev': 'interface device',
138+
'chan': 'channel',
139+
'lostconn': 'lost connection',
140+
}

‎tag_template_rust-0.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl‎

1.88 MB
Binary file not shown.

0 commit comments

Comments
 (0)