Skip to content

Commit 029ba66

Browse files
DavidJBiancoclaude
andcommitted
Complete Phase 1.9: Cross-reference validation and CLI logging improvements
- Add ScenarioValidator with cross-reference checking (personas, users, systems) - Add uniqueness validation for usernames, hostnames, and IPs - Integrate validation into CLI with Rich-formatted error messages - Add granular logging levels: default (WARNING), --verbose (INFO), --debug (DEBUG) - Add retail-store-ftp-attack scenario (24hr retail store with FTP RCE attack) - Add test file stubs for Phase 1.10 test coverage Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent 03ec16f commit 029ba66

10 files changed

Lines changed: 3562 additions & 15 deletions

File tree

TODO.md

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -114,17 +114,24 @@
114114

115115
### 1.9 Validation (Schema Only for Phase 1)
116116

117-
- [ ] `validation/schema.py` - Pydantic-based schema validation
118-
- [ ] Clear error messages with field paths
119-
- [ ] Test: Invalid YAML detection
120-
- [ ] Test: Missing required fields
121-
- [ ] Test: Type violations
117+
- [x] `validation/schema.py` - Pydantic-based schema validation
118+
- [x] Clear error messages with field paths
119+
- [x] Test: Invalid YAML detection
120+
- [x] Test: Missing required fields
121+
- [x] Test: Type violations
122+
- [x] Cross-reference validation (personas, systems, users)
123+
- [x] Uniqueness validation (usernames, hostnames, IPs)
124+
- [x] CLI integration with Rich formatting
125+
- [x] 14 test cases with 100% coverage
122126

123127
### 1.10 Phase 1 Testing & Documentation
124128

125-
- [ ] Unit tests for all core modules (target: 90%+ coverage for Phase 1 code)
126-
- [ ] Integration test: Complete flow with minimal scenario
127-
- [ ] Create test fixture: `fixtures/scenarios/minimal.yaml` (1 user, 1 system, 1 hour)
129+
- [ ] Unit tests for CLI module (target: 90%+ coverage) - IN PROGRESS
130+
- [ ] Unit tests for engine module (target: 90%+ coverage)
131+
- [ ] Unit tests for activity module (target: 90%+ coverage)
132+
- [ ] Unit tests for ground_truth module (target: 90%+ coverage)
133+
- [ ] Integration tests: Complete flow with all scenarios
134+
- [x] Test fixture exists: `fixtures/scenarios/minimal.yaml`
128135
- [ ] Create test fixture: `fixtures/scenarios/small-realistic.yaml` (20 users, 10 systems, 8 hours)
129136
- [ ] Manual testing: Generate logs and verify format compliance
130137
- [ ] Update README with Phase 1 status and basic usage

src/log_generator/cli/commands.py

Lines changed: 49 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -43,13 +43,20 @@
4343
EXIT_SIGINT = 130
4444

4545

46-
def setup_logging(verbose: bool = False) -> None:
46+
def setup_logging(verbose: bool = False, debug: bool = False) -> None:
4747
"""Configure logging with Rich handler.
4848
4949
Args:
50-
verbose: Enable debug logging if True
50+
verbose: Enable INFO level logging if True
51+
debug: Enable DEBUG level logging if True (takes precedence over verbose)
5152
"""
52-
level = logging.DEBUG if verbose else logging.INFO
53+
if debug:
54+
level = logging.DEBUG
55+
elif verbose:
56+
level = logging.INFO
57+
else:
58+
level = logging.WARNING
59+
5360
logging.basicConfig(
5461
level=level,
5562
format="%(message)s",
@@ -140,7 +147,13 @@ def generate(
140147
False,
141148
"--verbose",
142149
"-v",
143-
help="Enable verbose logging"
150+
help="Enable verbose (INFO level) logging"
151+
),
152+
debug: bool = typer.Option(
153+
False,
154+
"--debug",
155+
"-d",
156+
help="Enable debug (DEBUG level) logging"
144157
),
145158
) -> None:
146159
"""Generate synthetic security logs from a scenario file.
@@ -155,7 +168,7 @@ def generate(
155168
- 21: Generation error
156169
- 130: Interrupted (Ctrl+C)
157170
"""
158-
setup_logging(verbose)
171+
setup_logging(verbose, debug)
159172
logger = logging.getLogger(__name__)
160173

161174
console.print("[bold blue]EvidenceForge Log Generator[/bold blue]")
@@ -173,6 +186,35 @@ def generate(
173186
if scenario.storyline:
174187
console.print(f" Storyline events: {len(scenario.storyline)}")
175188

189+
# Cross-reference validation (Phase 1.9)
190+
from log_generator.validation import ScenarioValidator
191+
192+
console.print("\n[bold]Validating cross-references...[/bold]")
193+
validator = ScenarioValidator(scenario)
194+
issues = validator.validate()
195+
196+
if issues:
197+
console.print(f"\n[yellow]Found {len(issues)} validation issue(s):[/yellow]")
198+
for issue in issues:
199+
color = "red" if issue.severity == "error" else "yellow"
200+
icon = "✗" if issue.severity == "error" else "!"
201+
console.print(f" [{color}]{icon} {issue.field_path}[/{color}]")
202+
console.print(f" {issue.message}", style=color)
203+
if issue.suggestion:
204+
console.print(f" 💡 {issue.suggestion}", style="dim")
205+
206+
if validator.has_errors():
207+
console.print("\n[bold red]Validation failed with errors. Cannot proceed with generation.[/bold red]")
208+
raise typer.Exit(EXIT_SCHEMA_VALIDATION)
209+
else:
210+
console.print("\n[yellow]Warnings found but proceeding with generation...[/yellow]")
211+
else:
212+
console.print("[green]✓[/green] All cross-references valid")
213+
214+
except typer.Exit:
215+
# Re-raise typer.Exit to preserve exit codes
216+
raise
217+
176218
except FileNotFoundError:
177219
console.print(
178220
f"[bold red]Error:[/bold red] Scenario file not found: {scenario_file}",
@@ -196,7 +238,7 @@ def generate(
196238
f"[bold red]Error:[/bold red] Failed to load scenario: {e}",
197239
style="red"
198240
)
199-
if verbose:
241+
if verbose or debug:
200242
console.print_exception()
201243
raise typer.Exit(EXIT_INPUT_ERROR)
202244

@@ -306,7 +348,7 @@ def progress_callback(event_type: str, data: dict) -> None:
306348
f"\n[bold red]Error:[/bold red] Generation failed: {e}",
307349
style="red"
308350
)
309-
if verbose:
351+
if verbose or debug:
310352
console.print_exception()
311353
logger.exception("Generation failed")
312354
raise typer.Exit(EXIT_GENERATION_ERROR)
Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
"""Scenario validation for EvidenceForge."""
2+
3+
from .schema import ScenarioValidator, ValidationIssue
4+
5+
__all__ = ["ScenarioValidator", "ValidationIssue"]
Lines changed: 214 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,214 @@
1+
"""Cross-reference and logical validation for scenarios.
2+
3+
This module provides validation beyond Pydantic's schema validation:
4+
- Cross-references between users, systems, personas, groups
5+
- Uniqueness constraints (usernames, hostnames, IPs)
6+
- Logical consistency checks
7+
"""
8+
9+
from dataclasses import dataclass
10+
from typing import Optional
11+
12+
from log_generator.models import Scenario
13+
14+
15+
@dataclass
16+
class ValidationIssue:
17+
"""Represents a validation issue found in a scenario.
18+
19+
Attributes:
20+
severity: "error" (blocks generation) or "warning" (informational)
21+
field_path: Dot-separated path to the problematic field
22+
message: Human-readable description of the issue
23+
suggestion: Optional actionable suggestion to fix the issue
24+
"""
25+
26+
severity: str # "error" | "warning"
27+
field_path: str
28+
message: str
29+
suggestion: Optional[str] = None
30+
31+
32+
class ScenarioValidator:
33+
"""Validates cross-references and logical consistency in scenarios."""
34+
35+
def __init__(self, scenario: Scenario):
36+
"""Initialize validator with a scenario.
37+
38+
Args:
39+
scenario: The scenario to validate
40+
"""
41+
self.scenario = scenario
42+
self.issues: list[ValidationIssue] = []
43+
44+
# Build lookup sets for fast reference checking
45+
self._build_lookups()
46+
47+
def _build_lookups(self) -> None:
48+
"""Build lookup dictionaries for users, systems, personas, groups."""
49+
self.usernames = {user.username for user in self.scenario.environment.users}
50+
self.hostnames = {
51+
system.hostname for system in self.scenario.environment.systems
52+
}
53+
self.ips = {system.ip for system in self.scenario.environment.systems}
54+
self.persona_names = {persona.name for persona in self.scenario.personas}
55+
self.group_names = (
56+
{group.name for group in self.scenario.environment.groups}
57+
if self.scenario.environment.groups
58+
else set()
59+
)
60+
61+
def validate(self) -> list[ValidationIssue]:
62+
"""Run all validation checks and return issues found.
63+
64+
Returns:
65+
List of validation issues (errors and warnings)
66+
"""
67+
self._validate_user_persona_references()
68+
self._validate_system_user_references()
69+
self._validate_user_primary_system_references()
70+
self._validate_group_member_references()
71+
self._validate_storyline_references()
72+
self._validate_uniqueness()
73+
return self.issues
74+
75+
def has_errors(self) -> bool:
76+
"""Check if any error-level issues were found.
77+
78+
Returns:
79+
True if any errors found, False otherwise
80+
"""
81+
return any(issue.severity == "error" for issue in self.issues)
82+
83+
def _validate_user_persona_references(self) -> None:
84+
"""Check that user persona references exist in personas list."""
85+
for idx, user in enumerate(self.scenario.environment.users):
86+
if user.persona and user.persona not in self.persona_names:
87+
available = (
88+
", ".join(sorted(self.persona_names))
89+
if self.persona_names
90+
else "none defined"
91+
)
92+
self.issues.append(
93+
ValidationIssue(
94+
severity="error",
95+
field_path=f"environment.users.{idx}.persona",
96+
message=f"User '{user.username}' references undefined persona '{user.persona}'",
97+
suggestion=f"Available personas: {available}",
98+
)
99+
)
100+
101+
def _validate_system_user_references(self) -> None:
102+
"""Check that system assigned_user references exist in users list."""
103+
for idx, system in enumerate(self.scenario.environment.systems):
104+
if system.assigned_user and system.assigned_user not in self.usernames:
105+
self.issues.append(
106+
ValidationIssue(
107+
severity="error",
108+
field_path=f"environment.systems.{idx}.assigned_user",
109+
message=f"System '{system.hostname}' references undefined user '{system.assigned_user}'",
110+
suggestion=f"Available users: {', '.join(sorted(self.usernames))}",
111+
)
112+
)
113+
114+
def _validate_user_primary_system_references(self) -> None:
115+
"""Check that user primary_system references exist in systems list."""
116+
for idx, user in enumerate(self.scenario.environment.users):
117+
if user.primary_system and user.primary_system not in self.hostnames:
118+
self.issues.append(
119+
ValidationIssue(
120+
severity="error",
121+
field_path=f"environment.users.{idx}.primary_system",
122+
message=f"User '{user.username}' references undefined system '{user.primary_system}'",
123+
suggestion=f"Available systems: {', '.join(sorted(self.hostnames))}",
124+
)
125+
)
126+
127+
def _validate_group_member_references(self) -> None:
128+
"""Check that group members exist in users list."""
129+
if not self.scenario.environment.groups:
130+
return
131+
132+
for idx, group in enumerate(self.scenario.environment.groups):
133+
for member_idx, member in enumerate(group.members):
134+
if member not in self.usernames:
135+
self.issues.append(
136+
ValidationIssue(
137+
severity="error",
138+
field_path=f"environment.groups.{idx}.members.{member_idx}",
139+
message=f"Group '{group.name}' references undefined member '{member}'",
140+
suggestion=f"Available users: {', '.join(sorted(self.usernames))}",
141+
)
142+
)
143+
144+
def _validate_storyline_references(self) -> None:
145+
"""Check that storyline actor/system references are valid."""
146+
if not self.scenario.storyline:
147+
return
148+
149+
for idx, event in enumerate(self.scenario.storyline):
150+
# Validate actor (must be user or "attacker")
151+
if event.actor not in self.usernames and event.actor != "attacker":
152+
self.issues.append(
153+
ValidationIssue(
154+
severity="error",
155+
field_path=f"storyline.{idx}.actor",
156+
message=f"Storyline event references undefined actor '{event.actor}'",
157+
suggestion=f"Available users: {', '.join(sorted(self.usernames))}, or use 'attacker'",
158+
)
159+
)
160+
161+
# Validate system
162+
if event.system not in self.hostnames:
163+
self.issues.append(
164+
ValidationIssue(
165+
severity="error",
166+
field_path=f"storyline.{idx}.system",
167+
message=f"Storyline event references undefined system '{event.system}'",
168+
suggestion=f"Available systems: {', '.join(sorted(self.hostnames))}",
169+
)
170+
)
171+
172+
def _validate_uniqueness(self) -> None:
173+
"""Check for duplicate usernames, hostnames, and IPs."""
174+
# Check duplicate usernames
175+
seen_usernames = set()
176+
for idx, user in enumerate(self.scenario.environment.users):
177+
if user.username in seen_usernames:
178+
self.issues.append(
179+
ValidationIssue(
180+
severity="error",
181+
field_path=f"environment.users.{idx}.username",
182+
message=f"Duplicate username '{user.username}' found",
183+
suggestion="Usernames must be unique across all users",
184+
)
185+
)
186+
seen_usernames.add(user.username)
187+
188+
# Check duplicate hostnames
189+
seen_hostnames = set()
190+
for idx, system in enumerate(self.scenario.environment.systems):
191+
if system.hostname in seen_hostnames:
192+
self.issues.append(
193+
ValidationIssue(
194+
severity="error",
195+
field_path=f"environment.systems.{idx}.hostname",
196+
message=f"Duplicate hostname '{system.hostname}' found",
197+
suggestion="Hostnames must be unique across all systems",
198+
)
199+
)
200+
seen_hostnames.add(system.hostname)
201+
202+
# Check duplicate IPs
203+
seen_ips = set()
204+
for idx, system in enumerate(self.scenario.environment.systems):
205+
if system.ip in seen_ips:
206+
self.issues.append(
207+
ValidationIssue(
208+
severity="error",
209+
field_path=f"environment.systems.{idx}.ip",
210+
message=f"Duplicate IP address '{system.ip}' found",
211+
suggestion="IP addresses must be unique across all systems",
212+
)
213+
)
214+
seen_ips.add(system.ip)

0 commit comments

Comments
 (0)