Repository navigation
Expand file tree
/
Copy pathversion_lock.py
More file actions
527 lines (433 loc) · 25.5 KB
/
Copy pathversion_lock.py
File metadata and controls
527 lines (433 loc) · 25.5 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
# or more contributor license agreements. Licensed under the Elastic License
# 2.0; you may not use this file except in compliance with the Elastic License
# 2.0.
"""Helper utilities to manage the version lock."""
from copy import deepcopy
from dataclasses import dataclass
from pathlib import Path
from typing import Any, ClassVar, Literal, cast
import click
from semver import Version
from .config import load_current_package_version, parse_rules_config
from .mixins import LockDataclassMixin, MarshmallowDataclassMixin
from .schemas import definitions
from .stack_emit import emit_epoch_key, parse_stack, resolve_stack_emit_entry, transforms_for_stack
from .utils import cached, next_version
RULES_CONFIG = parse_rules_config()
# This was the original version the lock was created under. This constant has been replaced by
# schemas.get_min_supported_stack_version to dynamically determine the minimum:
# - MIN_LOCK_VERSION_DEFAULT = Version("7.13.0")
@dataclass(frozen=True)
class BaseEntry:
rule_name: definitions.RuleName
sha256: definitions.Sha256
type: definitions.RuleType
version: definitions.PositiveInteger
@dataclass(frozen=True)
class PreviousEntry(BaseEntry):
# this is Optional for resiliency in already tagged branches missing this field. This means we should strictly
# validate elsewhere
max_allowable_version: int | None = None
@dataclass(frozen=True)
class StackEmitEntry:
"""Version/hash for a stack emit epoch (shared across minors that inherit the same transforms)."""
sha256: definitions.Sha256
version: definitions.PositiveInteger
transforms: list[str] | None = None
@dataclass(frozen=True)
class VersionLockFileEntry(MarshmallowDataclassMixin, BaseEntry):
"""Schema for a rule entry in the version lock."""
min_stack_version: definitions.SemVerMinorOnly | None = None
previous: dict[definitions.SemVerMinorOnly, PreviousEntry] | None = None
# Emit epochs for stack-conditional packaging transforms (MITRE remap, RI operator, …).
# Keyed by the newest applicable transform min_stack (major.minor), not every package minor.
stack_emit: dict[definitions.SemVerMinorOnly, StackEmitEntry] | None = None
@dataclass(frozen=True)
class VersionLockFile(LockDataclassMixin):
"""Schema for the full version lock file."""
data: dict[definitions.UUIDString | definitions.KNOWN_BAD_RULE_IDS, VersionLockFileEntry]
file_path: ClassVar[Path] = RULES_CONFIG.version_lock_file
def __contains__(self, rule_id: str) -> bool:
"""Check if a rule is in the map by comparing IDs."""
return rule_id in self.data
def __getitem__(
self, item: definitions.UUIDString | Literal["119c8877-8613-416d-a98a-96b6664ee73a5"]
) -> VersionLockFileEntry:
"""Return entries by rule id."""
if item not in self.data:
raise KeyError(item)
return self.data[item]
@dataclass(frozen=True)
class DeprecatedRulesEntry(MarshmallowDataclassMixin):
"""Schema for rule entry in the deprecated rules file."""
deprecation_date: definitions.Date | definitions.KNOWN_BAD_DEPRECATED_DATES
rule_name: definitions.RuleName
stack_version: definitions.SemVer
deprecated_reason: str | None = None # Optional; from rule [metadata] when maturity == "deprecated"
@dataclass(frozen=True)
class DeprecatedRulesFile(LockDataclassMixin):
"""Schema for the full deprecated rules file."""
data: dict[definitions.UUIDString | definitions.KNOWN_BAD_RULE_IDS, DeprecatedRulesEntry]
file_path: ClassVar[Path] = RULES_CONFIG.deprecated_rules_file
def __contains__(self, rule_id: str) -> bool:
"""Check if a rule is in the map by comparing IDs."""
return rule_id in self.data
def __getitem__(
self, item: definitions.UUIDString | Literal["119c8877-8613-416d-a98a-96b6664ee73a5"]
) -> DeprecatedRulesEntry:
"""Return entries by rule id."""
if item not in self.data:
raise KeyError(item)
return self.data[item]
@cached
def load_versions() -> dict[str, Any]:
"""Load and validate the default version.lock file."""
version_lock_file = VersionLockFile.load_from_file()
return version_lock_file.to_dict()
# for tagged branches which existed before the types were added and validation enforced, we will need to manually add
# them to allow them to pass validation. These will only ever currently be loaded via the RuleCollection.load_git_tag
# method, which is primarily for generating diffs across releases, so there is no risk to versioning
def add_rule_types_to_lock(lock_contents: dict[str, Any], rule_map: dict[str, Any]) -> dict[str, Any]:
"""Add the rule type to entries in the lock file,if missing."""
for rule_id, lock in lock_contents.items():
rule = rule_map.get(rule_id, {})
# this defaults to query if the rule is not found - it is just for validation so should not impact
rule_type = rule.get("rule", {}).get("type", "query")
# the type is a bit less important than the structure to pass validation
lock["type"] = rule_type
if "previous" in lock:
for prev_lock in lock["previous"].values():
prev_lock["type"] = rule_type
return lock_contents
class VersionLock:
"""Version handling for rule files and collections."""
def __init__( # noqa: PLR0913, PLR0917
self,
version_lock_file: Path | None = None,
deprecated_lock_file: Path | None = None,
version_lock: dict[str, Any] | None = None,
deprecated_lock: dict[str, Any] | None = None,
name: str | None = None,
invalidated: bool | None = False,
) -> None:
if invalidated:
err_msg = "This VersionLock configuration is not valid when configued to bypass_version_lock."
raise NotImplementedError(err_msg)
if not version_lock_file and not version_lock:
raise ValueError("Must provide version lock file or contents")
if not deprecated_lock_file and not deprecated_lock:
raise ValueError("Must provide deprecated lock file or contents")
self.name = name
self.version_lock_file = version_lock_file
self.deprecated_lock_file = deprecated_lock_file
if version_lock_file:
self.version_lock = VersionLockFile.load_from_file(version_lock_file)
else:
self.version_lock = VersionLockFile.from_dict({"data": version_lock})
if deprecated_lock_file:
self.deprecated_lock = DeprecatedRulesFile.load_from_file(deprecated_lock_file)
else:
self.deprecated_lock = DeprecatedRulesFile.from_dict({"data": deprecated_lock})
@staticmethod
def save_file(path: Path, lock_file: VersionLockFile | DeprecatedRulesFile) -> None:
lock_file.save_to_file(path)
print(f"Updated {path} file")
def get_locked_version(self, rule_id: str, min_stack_version: str | None = None) -> int | None:
if rule_id not in self.version_lock:
return None
latest_version_info = self.version_lock[rule_id]
if latest_version_info.previous and latest_version_info.previous.get(min_stack_version):
stack_version_info = latest_version_info.previous.get(min_stack_version)
else:
stack_version_info = latest_version_info
return stack_version_info.version
def get_locked_hash(self, rule_id: str, min_stack_version: str | None = None) -> str | None:
"""Get the version info matching the min_stack_version if present."""
if rule_id not in self.version_lock:
return None
latest_version_info = self.version_lock[rule_id]
if latest_version_info.previous and latest_version_info.previous.get(min_stack_version):
stack_version_info = latest_version_info.previous.get(min_stack_version)
else:
stack_version_info = latest_version_info
existing_sha256: str = stack_version_info.sha256
return existing_sha256
def get_shipped_version(
self,
rule_id: str,
feature_min_stack: str | None,
package_stack: str | Version,
) -> int | None:
"""Resolve the version to ship for package_stack (baseline or inherited stack_emit)."""
baseline = self.get_locked_version(rule_id, feature_min_stack)
if rule_id not in self.version_lock:
return baseline
stack_emit = self.version_lock[rule_id].to_dict().get("stack_emit")
emit = resolve_stack_emit_entry(stack_emit, parse_stack(package_stack))
if emit is not None:
return int(emit["version"])
return baseline
def get_stack_emit_hash(
self,
rule_id: str,
package_stack: str | Version,
) -> str | None:
"""Return the locked emit hash for the inherited epoch of package_stack, if any."""
if rule_id not in self.version_lock:
return None
raw = self.version_lock[rule_id].to_dict().get("stack_emit")
emit = resolve_stack_emit_entry(raw, parse_stack(package_stack))
if emit is None:
return None
return str(emit["sha256"])
def manage_versions( # noqa: PLR0912, PLR0915
self,
rules: Any, # type: ignore[reportRedeclaration]
exclude_version_update: bool = False,
save_changes: bool = False,
verbose: bool = True,
buffer_int: int = 100,
) -> tuple[list[definitions.UUIDString], list[str], list[str]]:
"""Update the contents of the version.lock file and optionally save changes."""
from .packaging import current_stack_version
from .rule import TOMLRule
from .rule_loader import RuleCollection # noqa: TC001
rules: RuleCollection = rules # noqa: PLW0127
version_lock_hash = self.version_lock.sha256()
lock_file_contents = deepcopy(self.version_lock.to_dict())
current_deprecated_lock = deepcopy(self.deprecated_lock.to_dict())
verbose_echo = click.echo if verbose else (lambda _: None) # type: ignore[reportUnknownVariableType]
already_deprecated = set(current_deprecated_lock)
deprecated_rules = set(rules.deprecated.id_map)
new_rules = {rule.id for rule in rules if rule.contents.saved_version is None} - deprecated_rules
changed_rules = {rule.id for rule in rules if rule.contents.is_dirty} - deprecated_rules
emit_changed_rules = (
{rule.id for rule in rules if rule.contents.is_emit_dirty} - deprecated_rules - changed_rules
)
# manage deprecated rules
newly_deprecated = deprecated_rules - already_deprecated
if not (new_rules or changed_rules or newly_deprecated or emit_changed_rules):
return list(changed_rules), list(new_rules), list(newly_deprecated)
verbose_echo("Rule changes detected!")
changes: list[str] = []
def log_changes(r: TOMLRule, route_taken: str, new_rule_version: Any, *msg: str) -> None:
new = [f" {route_taken}: {r.id}, new version: {new_rule_version}"]
new.extend([f" - {m}" for m in msg if m])
changes.extend(new)
package_stack = parse_stack(load_current_package_version())
epoch_key = emit_epoch_key(package_stack)
package_transforms = [t.id for t in transforms_for_stack(package_stack)]
for rule in rules:
if rule.contents.metadata.maturity == "production" or rule.id in newly_deprecated:
# assume that older stacks are always locked first
min_stack = Version.parse(rule.contents.get_supported_version(), optional_minor_and_patch=True)
lock_from_rule = rule.contents.lock_info(bump=not exclude_version_update)
lock_from_file = lock_file_contents.setdefault(rule.id, {})
# scenarios to handle, assuming older stacks are always locked first:
# 1) no breaking changes ever made or the first time a rule is created
# 2) on the latest, after a breaking change has been locked
# 3) on the latest stack, locking in a breaking change
# 4) on an old stack, after a breaking change has been made
latest_locked_stack_version = rule.contents.convert_supported_version(
lock_from_file.get("min_stack_version")
)
# strip version down to only major.minor to compare against lock file versioning
stripped_version = f"{min_stack.major}.{min_stack.minor}"
baseline_touched = False
if rule.id in new_rules or rule.id in changed_rules or rule.id in newly_deprecated:
baseline_touched = True
if not lock_from_file or min_stack == latest_locked_stack_version:
route = "A"
# 1) no breaking changes ever made or the first time a rule is created
# 2) on the latest, after a breaking change has been locked
# Preserve existing stack_emit channels — never drop higher-stack emit rows.
existing_stack_emit = lock_from_file.get("stack_emit")
# If a stack_emit entry already shipped the proposed version, the lock takes the
# next free version instead so no shipped version is used twice for the same stack version.
floor = next_version(lock_from_file, stripped_version) if lock_from_file else None
lock_from_file.update(lock_from_rule)
if existing_stack_emit is not None:
lock_from_file["stack_emit"] = existing_stack_emit
if floor is not None and not exclude_version_update and lock_from_file["version"] < floor:
lock_from_file["version"] = floor
new_version = lock_from_file["version"]
# add the min_stack_version to the lock if it's explicitly set
if rule.contents.metadata.min_stack_version is not None:
lock_from_file["min_stack_version"] = stripped_version
log_msg = f"min_stack_version added: {min_stack}"
log_changes(rule, route, new_version, log_msg)
elif min_stack > latest_locked_stack_version:
route = "B"
# 3) on the latest stack, locking in a breaking change
stripped_latest_locked_stack_version = (
f"{latest_locked_stack_version.major}.{latest_locked_stack_version.minor}"
)
# preserve buffer space to support forked version spacing
if exclude_version_update:
buffer_int -= 1
lock_from_rule["version"] = lock_from_file["version"] + buffer_int
previous_lock_info = {
"max_allowable_version": lock_from_rule["version"] - 1,
"rule_name": lock_from_file["rule_name"],
"sha256": lock_from_file["sha256"],
"version": lock_from_file["version"],
"type": lock_from_file["type"],
}
lock_from_file.setdefault("previous", {})
# move the current locked info into the previous section
lock_from_file["previous"][stripped_latest_locked_stack_version] = previous_lock_info
existing_stack_emit = lock_from_file.get("stack_emit")
# overwrite the "latest" part of the lock at the top level
lock_from_file.update(lock_from_rule, min_stack_version=stripped_version)
if existing_stack_emit is not None:
lock_from_file["stack_emit"] = existing_stack_emit
new_version = lock_from_rule["version"]
log_changes(
rule,
route,
new_version,
f"previous {stripped_latest_locked_stack_version} saved as \
version: {previous_lock_info['version']}",
f"current min_stack updated to {stripped_version}",
)
elif min_stack < latest_locked_stack_version:
route = "C"
# 4) on an old stack, after a breaking change has been made (updated fork)
if stripped_version not in lock_from_file.get("previous", {}):
raise ValueError(f"Expected {rule.id} @ v{stripped_version} in the rule lock")
previous_entry = lock_from_file["previous"][stripped_version]
max_allowable_version = previous_entry["max_allowable_version"]
info_from_rule = (lock_from_rule["sha256"], lock_from_rule["version"])
info_from_file = (previous_entry["sha256"], previous_entry["version"])
if lock_from_rule["version"] > max_allowable_version:
raise ValueError(
f"Forked rule: {rule.id} - {rule.name} has changes that will force it to "
f"exceed the max allowable version of {max_allowable_version}"
)
if info_from_rule != info_from_file:
# The fork cannot pass its max_allowable_version, so fail if the next free
# version is exceeding max_allowable_version.
floor = next_version(lock_from_file, stripped_version)
previous_entry.update(lock_from_rule)
if not exclude_version_update and previous_entry["version"] < floor:
if floor > max_allowable_version:
raise ValueError(
f"Forked rule: {rule.id} - {rule.name} needs version {floor} to clear its "
f"shipped emit versions, above the max allowable version of "
f"{max_allowable_version}"
)
previous_entry["version"] = floor
new_version = previous_entry["version"]
log_changes(
rule,
route,
"unchanged",
f"previous version {stripped_version} updated version to {new_version}",
)
# Still allow emit-channel updates below; do not continue past emit handling.
else:
raise RuntimeError("Unreachable code")
# Update stack_emit for the current package stack epoch when transforms apply.
# Older branches must not rewrite newer emit keys; they only touch their own epoch.
emit_rules = emit_changed_rules | changed_rules | new_rules
if epoch_key and package_transforms and (baseline_touched or rule.id in emit_rules):
emit_hash = rule.contents.get_emit_hash()
untransformed_hash = rule.contents.get_untransformed_hash()
stack_emit: dict[str, dict[str, Any]] = cast(
"dict[str, dict[str, Any]]", lock_from_file.get("stack_emit") or {}
)
# Transforms were a no-op for this rule — drop a stale epoch rather than
# comparing against the baseline lock hash (which strips related_integrations).
if emit_hash == untransformed_hash:
if epoch_key in stack_emit:
if exclude_version_update:
# a hash-only run (dev update-lock-versions) cannot bump, and dropping the epoch
# hands its stacks the baseline number next; keep the row until a versioned run
log_changes(
rule,
"stack_emit",
lock_from_file.get("version"),
f"epoch {epoch_key} kept: transforms are a no-op but dropping it needs a "
"version bump (run build-release --update-version-lock)",
)
continue
stack_emit = dict(stack_emit)
dropped_version = int(stack_emit.pop(epoch_key)["version"])
if stack_emit:
lock_from_file["stack_emit"] = stack_emit
else:
lock_from_file.pop("stack_emit", None)
log_changes(
rule,
"stack_emit",
lock_from_file.get("version"),
f"epoch {epoch_key} removed (transforms no-op)",
)
# these stacks ship the baseline from now on, so it must sit above the version the
# epoch already shipped to them
forks: dict[str, dict[str, Any]] = lock_from_file.get("previous") or {}
baseline: dict[str, Any] = forks.get(stripped_version) or lock_from_file
if int(baseline["version"]) <= dropped_version:
floor = next_version(lock_from_file, stripped_version)
baseline["version"] = max(dropped_version + 1, floor)
log_changes(
rule,
"stack_emit",
baseline["version"],
f"baseline moved past version {dropped_version} shipped by epoch {epoch_key}",
)
continue
if stack_emit.get(epoch_key, {}).get("sha256") == emit_hash:
continue
stack_emit = cast("dict[str, dict[str, Any]]", lock_from_file.setdefault("stack_emit", {}))
existing_emit = stack_emit.get(epoch_key)
if exclude_version_update and existing_emit:
new_emit_version = int(existing_emit["version"])
else:
# the epoch content changed, so its old number is taken too; go above everything shipped
new_emit_version = next_version(lock_from_file, stripped_version)
stack_emit[epoch_key] = {
"sha256": emit_hash,
"version": new_emit_version,
"transforms": package_transforms,
}
log_changes(
rule,
"stack_emit",
new_emit_version,
f"epoch {epoch_key} hash updated ({', '.join(package_transforms)})",
)
for rule in rules.deprecated:
if rule.id in newly_deprecated:
entry: dict[str, Any] = {
"rule_name": rule.name,
"stack_version": current_stack_version(),
"deprecation_date": rule.contents.metadata["deprecation_date"],
}
# deprecated_reason is set in the rule TOML [metadata]; copy into lock when present.
if rule.contents.metadata.get("deprecated_reason"):
entry["deprecated_reason"] = rule.contents.metadata["deprecated_reason"]
current_deprecated_lock[rule.id] = entry
if save_changes or verbose:
click.echo(f" - {len(changed_rules)} changed rules")
click.echo(f" - {len(new_rules)} new rules")
click.echo(f" - {len(newly_deprecated)} newly deprecated rules")
click.echo(f" - {len(emit_changed_rules)} stack-emit changed rules")
if not save_changes:
verbose_echo(
"run `build-release --update-version-lock` to update version.lock.json and deprecated_rules.json"
)
return list(changed_rules), list(new_rules), list(newly_deprecated)
click.echo("Detailed changes: \n" + "\n".join(changes))
# reset local version lock
self.version_lock = VersionLockFile.from_dict({"data": lock_file_contents})
self.deprecated_lock = DeprecatedRulesFile.from_dict({"data": current_deprecated_lock})
new_hash = self.version_lock.sha256()
if version_lock_hash != new_hash and self.version_lock_file:
self.save_file(self.version_lock_file, self.version_lock)
if newly_deprecated and self.deprecated_lock_file:
self.save_file(self.deprecated_lock_file, self.deprecated_lock)
return list(changed_rules), list(new_rules), list(newly_deprecated)
name = str(RULES_CONFIG.version_lock_file)
loaded_version_lock = VersionLock(RULES_CONFIG.version_lock_file, RULES_CONFIG.deprecated_rules_file, name=name)