-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathcli.py
More file actions
202 lines (161 loc) · 6.55 KB
/
Copy pathcli.py
File metadata and controls
202 lines (161 loc) · 6.55 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
from __future__ import annotations
import argparse
import json
import sys
from pipeline.storage import IOCStorage
INGESTORS = ("urlhaus", "feodo", "otx")
def build_ingestor(name: str, args: argparse.Namespace | None = None):
if name == "urlhaus":
from pipeline.ingestors.urlhaus import URLHausIngestor
return URLHausIngestor()
if name == "feodo":
from pipeline.ingestors.feodo import FeodoIngestor
return FeodoIngestor()
if name == "otx":
from pipeline.ingestors.otx import OTXIngestor
if args and args.otx_limit is not None:
return OTXIngestor(limit=args.otx_limit, max_page=args.otx_max_page)
return OTXIngestor()
raise ValueError(f"Unknown ingestor: {name}")
def ingest(args: argparse.Namespace) -> int:
storage = IOCStorage()
total = 0
selected = args.feeds or list(INGESTORS)
for name in selected:
try:
ingestor = build_ingestor(name, args)
records = ingestor.ingest()
stored = storage.upsert_iocs(records)
storage.log_feed_run(name, "success", stored)
total += stored
print(f"{name}: stored {stored} normalized records")
except Exception as exc:
storage.log_feed_run(name, "failed", 0, str(exc))
print(f"{name}: failed: {exc}", file=sys.stderr)
print(f"ingest complete: {total} records processed")
return 0
def score(args: argparse.Namespace) -> int:
from pipeline.scorer import ConfidenceScorer
storage = IOCStorage()
scorer = ConfidenceScorer(storage)
updated = scorer.score_all()
print(f"scored {updated} IOC records")
if args.verbose:
print("\nscore breakdown:")
rows = storage.top_iocs(args.top)
for row in rows:
breakdown = scorer.score_breakdown(row)
print(
f"{row['value']} ({row['type']}): "
f"recency={breakdown['recency']}, "
f"source={breakdown['source']}, "
f"corroboration={breakdown['corroboration']}, "
f"total={row['confidence_score']:.1f}"
)
return 0
def enrich(args: argparse.Namespace) -> int:
from pipeline.enricher import IOCEnricher
storage = IOCStorage()
updated = IOCEnricher(storage).enrich_high_confidence(
limit=args.limit, verbose=args.verbose, ioc_type=args.ioc_type
)
print(f"enriched {updated} high-confidence IOC records")
return 0
def export(args: argparse.Namespace) -> int:
from pipeline.exporter import IOCExporter
storage = IOCStorage()
exporter = IOCExporter(storage)
if args.wazuh:
try:
counts = exporter.export_wazuh(
ip_path=args.ip_list,
domain_path=args.domain_list,
reload_wazuh=args.reload,
)
except PermissionError as exc:
print(
f"export failed: permission denied writing Wazuh list path: {exc.filename}",
file=sys.stderr,
)
print(
"Use --ip-list/--domain-list for a local screenshot export, "
"or run with permissions to write /var/ossec.",
file=sys.stderr,
)
return 1
print(
f"exported {counts['ips']} IPs and {counts['domains']} domains to Wazuh lists"
)
else:
print(exporter.report(args.top))
return 0
def report(args: argparse.Namespace) -> int:
from pipeline.exporter import IOCExporter
storage = IOCStorage()
print(IOCExporter(storage).report(args.top))
return 0
def stats(args: argparse.Namespace) -> int:
storage = IOCStorage()
print(json.dumps(storage.stats(), indent=2))
return 0
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description="Threat intel IOC aggregation pipeline")
subparsers = parser.add_subparsers(dest="command", required=True)
ingest_parser = subparsers.add_parser("ingest", help="pull feeds, normalize, and store")
ingest_parser.add_argument(
"--feeds",
nargs="+",
choices=sorted(INGESTORS),
help="optional feed subset to ingest",
)
ingest_parser.add_argument(
"--otx-limit",
type=int,
help="maximum OTX pulses to fetch; keeps OTX screenshots fast",
)
ingest_parser.add_argument(
"--otx-max-page",
type=int,
default=1,
help="maximum OTX API pages to walk when --otx-limit is used",
)
ingest_parser.set_defaults(func=ingest)
score_parser = subparsers.add_parser("score", help="calculate confidence scores")
score_parser.add_argument("--verbose", action="store_true", help="print scoring components")
score_parser.add_argument("--top", type=int, default=5, help="verbose rows to print")
score_parser.set_defaults(func=score)
enrich_parser = subparsers.add_parser("enrich", help="enrich high-confidence IOCs")
enrich_parser.add_argument("--limit", type=int, help="maximum number of IOCs to enrich")
enrich_parser.add_argument("--verbose", action="store_true", help="print enrichment summaries")
enrich_parser.add_argument(
"--type",
dest="ioc_type",
choices=["ip", "domain", "url", "hash"],
help="only enrich high-confidence IOCs of this type",
)
enrich_parser.set_defaults(func=enrich)
export_parser = subparsers.add_parser("export", help="export to Wazuh or print report")
export_parser.add_argument("--wazuh", action="store_true", help="write Wazuh CDB lists")
export_parser.add_argument("--reload", action="store_true", help="reload Wazuh after export")
export_parser.add_argument(
"--ip-list",
help="IP CDB list output path; defaults to TIP_WAZUH_IP_LIST or /var/ossec",
)
export_parser.add_argument(
"--domain-list",
help="domain CDB list output path; defaults to TIP_WAZUH_DOMAIN_LIST or /var/ossec",
)
export_parser.add_argument("--top", type=int, default=20, help="report rows when not using Wazuh")
export_parser.set_defaults(func=export)
report_parser = subparsers.add_parser("report", help="print top IOCs by score")
report_parser.add_argument("--top", type=int, default=20)
report_parser.set_defaults(func=report)
stats_parser = subparsers.add_parser("stats", help="show database and feed run stats")
stats_parser.set_defaults(func=stats)
return parser
def main() -> int:
parser = build_parser()
args = parser.parse_args()
return args.func(args)
if __name__ == "__main__":
raise SystemExit(main())