Repository navigation
Expand file tree
/
Copy pathatlas.py
More file actions
381 lines (307 loc) · 14.3 KB
/
Copy pathatlas.py
File metadata and controls
381 lines (307 loc) · 14.3 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
# or more contributor license agreements. Licensed under the Elastic License
# 2.0; you may not use this file except in compliance with the Elastic License
# 2.0.
"""MITRE ATLAS info, versioned like ATT&CK (`atlas-v*.json.gz`)."""
from __future__ import annotations
import json
from collections import OrderedDict
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, cast
import requests
import yaml
from .utils import cached, clear_caches, get_etc_glob_path, get_etc_path, gzip_compress, read_gzip
if TYPE_CHECKING:
from pathlib import Path
ATLAS_JSON_GZ_PATTERN = "atlas-v*.json.gz"
ATLAS_DIST_BASE = "https://raw.githubusercontent.com/mitre-atlas/atlas-data/main/dist"
ATLAS_MANIFEST_URL = f"{ATLAS_DIST_BASE}/manifest.yaml"
ATLAS_URL_BASE = "https://atlas.mitre.org/{type}/{id}/"
# AML.T0000 vs AML.T0000.000 — sub-techniques contain two dots.
ATLAS_SUBTECHNIQUE_DOT_COUNT = 2
# Stack at which MITRE ATLAS threat mappings are shipped (emit transform gate).
# Keep in sync with stack_emit.MITRE_ATLAS_MIN_STACK.
MITRE_ATLAS_MIN_STACK_MAJOR_MINOR = (9, 6)
def _atlas_file_version_key(path: Path) -> tuple[int, ...]:
"""Sort key for an atlas-v*.json.gz filename (content version, e.g. 2026.08 or 5.1.0)."""
ver = path.name.split("-v", 1)[1][: -len(".json.gz")]
parts: list[int] = []
for part in ver.split("."):
try:
parts.append(int(part))
except ValueError:
parts.append(0)
return tuple(parts)
def get_atlas_file_path() -> Path:
"""Return the latest ATLAS data file (highest content version)."""
atlas_files = get_etc_glob_path([ATLAS_JSON_GZ_PATTERN])
if not atlas_files:
raise FileNotFoundError(f"Missing required {ATLAS_JSON_GZ_PATTERN} file")
return max(atlas_files, key=_atlas_file_version_key)
def get_atlas_file_path_for_version(version: str) -> Path:
"""Return the ATLAS data file whose content version matches `version`."""
wanted = str(version).lstrip("v")
atlas_files = get_etc_glob_path([ATLAS_JSON_GZ_PATTERN])
for path in atlas_files:
file_ver = path.name.split("-v", 1)[1][: -len(".json.gz")]
if file_ver == wanted:
return path
available = [p.name.split("-v", 1)[1][: -len(".json.gz")] for p in atlas_files]
raise FileNotFoundError(f"No ATLAS data file found for version {version!r}. Available: {available}")
def _current_atlas_version() -> str:
try:
path = get_atlas_file_path()
except FileNotFoundError:
return "unknown"
return path.name.split("-v", 1)[1][: -len(".json.gz")]
CURRENT_ATLAS_VERSION = _current_atlas_version()
def load_atlas_gz() -> dict[str, Any]:
"""Load the latest ATLAS JSON payload."""
return json.loads(read_gzip(get_atlas_file_path()))
@dataclass
class AtlasLookups:
"""Pre-built ATLAS lookup structures for a specific content version."""
version: str
tactics_map: dict[str, str]
tactic_id_to_detail: dict[str, dict[str, str]]
technique_lookup: OrderedDict[str, dict[str, Any]]
matrix: dict[str, list[str]]
def _mapping(value: Any) -> dict[str, Any]:
"""Return value when it is a dict, otherwise an empty mapping."""
return cast("dict[str, Any]", value) if isinstance(value, dict) else {}
def _sequence(value: Any) -> list[Any]:
"""Return value when it is a list, otherwise an empty list."""
return cast("list[Any]", value) if isinstance(value, list) else []
def _normalize_atlas_payload(raw: dict[str, Any]) -> dict[str, Any]:
"""Normalize v5 (legacy matrices list) and v6 (collection + dict maps) payloads."""
tactics_raw = raw.get("tactics")
if "collection" in raw and isinstance(tactics_raw, dict):
collection = _mapping(raw.get("collection"))
version = str(collection.get("version") or raw.get("format-version") or "unknown")
techniques_raw = raw.get("techniques")
techniques_map = _mapping(techniques_raw)
return {
"version": version,
"tactics": list(_mapping(tactics_raw).values()),
"techniques": list(techniques_map.values()),
"relationships": _mapping(raw.get("relationships")),
}
# v5 / legacy: version + matrices[].tactics / techniques
version = str(raw.get("version") or "unknown")
matrices = _sequence(raw.get("matrices"))
matrix_data: dict[str, Any] | None = None
for matrix in matrices:
matrix_obj = _mapping(matrix)
if matrix_obj.get("id") == "ATLAS":
matrix_data = matrix_obj
break
if matrix_data is None and matrices:
matrix_data = _mapping(matrices[0])
matrix_data = matrix_data or {}
return {
"version": version,
"tactics": _sequence(matrix_data.get("tactics")),
"techniques": _sequence(matrix_data.get("techniques")),
"relationships": {},
}
def _tactics_for_technique(
technique: dict[str, Any],
relationships: dict[str, Any],
known_tactic_ids: set[str],
) -> list[str]:
"""Return tactic IDs for a technique from v6 relationships or a v5 tactics field."""
tech_id = str(technique.get("id", ""))
rel = _mapping(relationships.get(tech_id))
achieved = [str(entry.get("target")) for entry in _sequence(rel.get("achieves")) if _mapping(entry).get("target")]
if achieved:
return [tid for tid in achieved if tid in known_tactic_ids]
return [str(tid) for tid in _sequence(technique.get("tactics")) if str(tid) in known_tactic_ids]
def _build_lookups(version: str, raw: dict[str, Any]) -> AtlasLookups:
"""Build ATLAS lookup structures from a normalized or raw ATLAS payload."""
normalized = _normalize_atlas_payload(raw)
version = version or str(normalized.get("version") or "unknown")
tactics_map: dict[str, str] = {}
tactic_id_to_detail: dict[str, dict[str, str]] = {}
for tactic in normalized["tactics"]:
tactic_id = str(tactic["id"])
tactic_name = str(tactic["name"])
tactics_map[tactic_name] = tactic_id
tactic_id_to_detail[tactic_id] = {
"id": tactic_id,
"name": tactic_name,
"reference": ATLAS_URL_BASE.format(type="tactics", id=tactic_id),
}
technique_lookup: dict[str, dict[str, Any]] = {}
matrix: dict[str, list[str]] = {name: [] for name in tactics_map}
relationships = _mapping(normalized.get("relationships"))
for technique in _sequence(normalized.get("techniques")):
technique_obj = _mapping(technique)
technique_id = str(technique_obj["id"])
tactic_ids = _tactics_for_technique(technique_obj, relationships, set(tactic_id_to_detail))
technique_lookup[technique_id] = {
"name": technique_obj["name"],
"id": technique_id,
"tactics": tactic_ids,
}
for tactic_id in tactic_ids:
tactic_name = tactic_id_to_detail[tactic_id]["name"]
if technique_id not in matrix[tactic_name]:
matrix[tactic_name].append(technique_id)
for val in matrix.values():
val.sort(key=lambda tid: technique_lookup.get(tid, {}).get("name", "").lower())
return AtlasLookups(
version=version,
tactics_map=tactics_map,
tactic_id_to_detail=tactic_id_to_detail,
technique_lookup=OrderedDict(sorted(technique_lookup.items())),
matrix=matrix,
)
@cached
def build_atlas_lookups_for_version(version: str) -> AtlasLookups:
"""Load and cache ATLAS lookup structures for a specific content version."""
path = get_atlas_file_path_for_version(version)
raw = json.loads(read_gzip(path))
return _build_lookups(version, raw)
def _empty_lookups() -> AtlasLookups:
return AtlasLookups(
version="unknown",
tactics_map={},
tactic_id_to_detail={},
technique_lookup=OrderedDict(),
matrix={},
)
def _load_latest_lookups() -> AtlasLookups:
if CURRENT_ATLAS_VERSION == "unknown":
return _empty_lookups()
try:
return build_atlas_lookups_for_version(CURRENT_ATLAS_VERSION)
except FileNotFoundError:
return _empty_lookups()
_latest = _load_latest_lookups()
tactics_map = _latest.tactics_map
technique_lookup = _latest.technique_lookup
matrix = _latest.matrix
tactics = list(tactics_map)
techniques = sorted({v["name"] for _, v in technique_lookup.items()})
technique_id_list = [t for t in technique_lookup if t.count(".") < ATLAS_SUBTECHNIQUE_DOT_COUNT]
sub_technique_id_list = [t for t in technique_lookup if t.count(".") >= ATLAS_SUBTECHNIQUE_DOT_COUNT]
def _latest_manifest_release() -> tuple[str, str]:
"""Return (content_version, dist-relative yaml path) for the newest v6 ATLAS release."""
response = requests.get(ATLAS_MANIFEST_URL, timeout=30)
response.raise_for_status()
loaded = yaml.safe_load(response.text)
if not isinstance(loaded, list) or not loaded:
raise ValueError("ATLAS manifest is empty or invalid")
releases = [_mapping(entry) for entry in cast("list[Any]", loaded) if isinstance(entry, dict)]
if not releases:
raise ValueError("ATLAS manifest is empty or invalid")
def _release_key(entry: dict[str, Any]) -> tuple[int, ...]:
rel = str(entry.get("release") or "0")
parts: list[int] = []
for part in rel.split("."):
try:
parts.append(int(part))
except ValueError:
parts.append(0)
return tuple(parts)
latest = max(releases, key=_release_key)
content_version = str(latest["release"])
versions = [_mapping(entry) for entry in _sequence(latest.get("versions")) if isinstance(entry, dict)]
v6 = next((entry for entry in versions if str(entry.get("format-version", "")).startswith("6.")), None)
if v6 is None and versions:
v6 = versions[0]
if v6 is None or not v6.get("path"):
raise ValueError(f"No ATLAS distribution path for release {content_version}")
return content_version, str(v6["path"])
def download_atlas_data(save: bool = True) -> dict[str, Any] | None:
"""Download the latest ATLAS YAML and optionally persist it as versioned json.gz."""
content_version, rel_path = _latest_manifest_release()
url = f"{ATLAS_DIST_BASE}/{rel_path.lstrip('/')}"
response = requests.get(url, timeout=60)
response.raise_for_status()
downloaded = yaml.safe_load(response.text)
if not isinstance(downloaded, dict):
raise TypeError("ATLAS download did not return a mapping")
atlas_data = cast("dict[str, Any]", downloaded)
if save:
compressed = gzip_compress(json.dumps(atlas_data, sort_keys=True, default=str))
new_path = get_etc_path([f"atlas-v{content_version}.json.gz"])
_ = new_path.write_bytes(compressed)
print(f"Downloaded ATLAS {content_version} to {new_path}")
return atlas_data
def refresh_atlas_data(save: bool = True) -> tuple[dict[str, Any] | None, bytes | None]:
"""Refresh ATLAS data from MITRE when a newer content version exists."""
try:
current_key = _atlas_file_version_key(get_atlas_file_path())
current_version = CURRENT_ATLAS_VERSION
except FileNotFoundError:
current_key = (0,)
current_version = "none"
content_version, rel_path = _latest_manifest_release()
latest_key = tuple(int(p) if p.isdigit() else 0 for p in str(content_version).split("."))
if current_key >= latest_key:
print(f"No versions newer than the current detected: {current_version}")
return None, None
url = f"{ATLAS_DIST_BASE}/{rel_path.lstrip('/')}"
response = requests.get(url, timeout=60)
response.raise_for_status()
atlas_data = yaml.safe_load(response.text)
compressed = gzip_compress(json.dumps(atlas_data, sort_keys=True, default=str))
if save:
new_path = get_etc_path([f"atlas-v{content_version}.json.gz"])
_ = new_path.write_bytes(compressed)
print(f"Saved ATLAS {content_version} to {new_path} (previous: {current_version})")
clear_caches()
return atlas_data, compressed
def load_atlas_yaml() -> dict[str, Any]:
"""Load ATLAS data (kept for callers that still expect YAML-shaped dicts)."""
return load_atlas_gz()
def canonical_technique_id(technique_id: str) -> str:
"""Normalize a tagged/authored ATLAS technique id to the AML.Txxxx form."""
tid = technique_id.strip()
if tid.upper().startswith("AML."):
return f"AML.{tid[4:]}" if tid.startswith("aml.") else tid
if tid.upper().startswith("T") and tid[1:2].isdigit():
return f"AML.{tid}"
return tid
def build_threat_map_entry(tactic_name: str, *technique_ids: str) -> dict[str, Any]:
"""Build rule threat map from ATLAS technique IDs."""
tactic_id = tactics_map.get(tactic_name)
if not tactic_id:
raise ValueError(f"Unknown ATLAS tactic: {tactic_name}")
tech_entries: dict[str, Any] = {}
def make_entry(_id: str) -> dict[str, Any]:
tech_info = technique_lookup.get(_id)
if not tech_info:
raise ValueError(f"Unknown ATLAS technique ID: {_id}")
return {
"id": _id,
"name": tech_info["name"],
"reference": ATLAS_URL_BASE.format(type="techniques", id=_id),
}
for raw_tid in technique_ids:
tid = canonical_technique_id(raw_tid)
if tid not in technique_lookup:
raise ValueError(f"Unknown ATLAS technique ID: {tid}")
tech_info = technique_lookup[tid]
tech_tactic_ids = tech_info.get("tactics", [])
if tactic_id not in tech_tactic_ids:
raise ValueError(f"ATLAS technique ID: {tid} does not fall under tactic: {tactic_name}")
# Sub-techniques are AML.T0000.000 (two dots)
if tid.count(".") >= ATLAS_SUBTECHNIQUE_DOT_COUNT:
parent_technique = tid.rsplit(".", 1)[0]
tech_entries.setdefault(parent_technique, make_entry(parent_technique))
tech_entries[parent_technique].setdefault("subtechnique", []).append(make_entry(tid))
else:
tech_entries.setdefault(tid, make_entry(tid))
entry: dict[str, Any] = {
"framework": "MITRE ATLAS",
"tactic": {
"id": tactic_id,
"name": tactic_name,
"reference": ATLAS_URL_BASE.format(type="tactics", id=tactic_id),
},
}
if tech_entries:
entry["technique"] = sorted(tech_entries.values(), key=lambda x: x["id"])
return entry