crashlens/policies/ci-sample.yaml at main · Crashlens/crashlens · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
# CrashLens Policy: CI Sample - Lightweight checks for continuous integration
# Quick validation suitable for CI/CD pipelines
# yaml-language-server: $schema=../crashlens/config/policy-schema.json
#
# Usage:
#   crashlens guard logs.jsonl --policy-file policies/ci-sample.yaml --fail-on-violations
#   crashlens guard logs.jsonl --policy-file policies/ci-sample.yaml --severity critical --dry-run
#
# CI/CD Integration:
#   # GitHub Actions
#   - run: crashlens guard logs.jsonl --policy-file policies/ci-sample.yaml --fail-on-violations
#
#   # GitLab CI
#   script:
#     - crashlens guard logs.jsonl --policy-file policies/ci-sample.yaml --fail-on-violations
#
# Related templates:
#   - budget-protection
#   - rate-limit-management

version: 1

rules:
  - id: ci_cost_check
    description: "Enforce cost limits in CI environment"
    match:
      cost: ">0.05"
    action: fail
    severity: critical
    suggestion: "CI cost limit exceeded - review model selection for testing"

  - id: ci_model_allowlist
    description: "Restrict to approved models in CI"
    match:
      model: "not_in:[gpt-4o-mini,gpt-3.5-turbo,claude-3-haiku]"
    action: fail
    severity: critical
    suggestion: "Unauthorized model in CI - use approved models: gpt-4o-mini, gpt-3.5-turbo, claude-3-haiku"

  - id: ci_token_limit
    description: "Prevent large token usage in CI"
    match:
      usage.completion_tokens: ">500"
    action: warn
    severity: medium
    suggestion: "High token count in CI - consider shorter test prompts"

  - id: ci_response_time
    description: "Monitor response times in CI"
    match:
      metadata.duration: ">5000"
    action: warn
    severity: low
    suggestion: "Slow response in CI - may impact build times"

  - id: ci_error_detection
    description: "Catch API errors in CI"
    match:
      level: "ERROR"
    action: fail
    severity: critical
    suggestion: "API error in CI - check model availability and API keys"

global:
  max_violations_per_rule: 5
  enable_cost_estimation: true
  ci_mode: true

cost_thresholds:
  warning_threshold: 0.01
  critical_threshold: 0.05

# CI-specific configuration
ci_config:
  timeout: 30s              # Fast timeout for CI
  max_parallel_requests: 3  # Limited concurrency
  allowed_models:
    - "gpt-4o-mini"
    - "gpt-3.5-turbo"
    - "claude-3-haiku"
  cost_budget: 0.05         # $0.05 per CI run

# Sample test cases for validation
test_cases:
  - name: "valid_ci_request"
    input:
      model: "gpt-4o-mini"
      cost: 0.01
      duration: 1500
      status: "success"
    expected: "pass"

  - name: "expensive_model_in_ci"
    input:
      model: "gpt-4"
      cost: 0.10
    expected: "fail"

  - name: "api_error_in_ci"
    input:
      status: "error"
    expected: "fail"