Skip to content

Commit c49cca5

Browse files
Brian KrafftCopilot
andcommitted
feat(5.3): Extract trigger functions into evolution/triggers.py
- New: evolution/triggers.py (380 lines) — process_analysis, process_tool_degradation, process_metric_check, build_context_from_analysis, load_skill_content, diagnose_skill_health - Modified: evolver.py — 6 methods now delegate to triggers module (-230 lines, 44.7KB from 58KB) - Constants moved: _ANALYSIS_CONTEXT_MAX, thresholds (re-imported in evolver.py) - New: tests/test_evolution_triggers.py — 22 tests - Suite: 1,468 passed, 127 skipped Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
1 parent 9df2f33 commit c49cca5

3 files changed

Lines changed: 807 additions & 323 deletions

File tree

Lines changed: 391 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,391 @@
1+
"""Evolution trigger functions — analysis, tool degradation, metric monitor.
2+
3+
Functions extracted from ``SkillEvolver`` (Epic 5.3):
4+
- ``process_analysis`` — Trigger 1: post-analysis evolution
5+
- ``process_tool_degradation`` — Trigger 2: fix skills for degraded tools
6+
- ``process_metric_check`` — Trigger 3: periodic health-based evolution
7+
- ``build_context_from_analysis`` — Build EvolutionContext from analyzer output
8+
- ``load_skill_content`` — Load SKILL.md content (registry or disk)
9+
- ``diagnose_skill_health`` — Pure metric classifier for Trigger 3
10+
"""
11+
12+
from __future__ import annotations
13+
14+
import logging
15+
from pathlib import Path
16+
from typing import TYPE_CHECKING, List, Optional
17+
18+
from openspace.utils.logging import Logger
19+
20+
from .models import EvolutionContext, EvolutionTrigger
21+
22+
from openspace.skill_engine.types import EvolutionSuggestion, EvolutionType
23+
24+
if TYPE_CHECKING:
25+
from openspace.grounding.core.quality.types import ToolQualityRecord
26+
from openspace.skill_engine.types import (
27+
ExecutionAnalysis,
28+
SkillRecord,
29+
)
30+
31+
logger = Logger.get_logger(__name__)
32+
33+
# ---------------------------------------------------------------------------
34+
# Constants (moved from evolver.py — only used by trigger functions)
35+
# ---------------------------------------------------------------------------
36+
37+
_ANALYSIS_CONTEXT_MAX = 5 # Max recent analyses to include in prompt
38+
_ANALYSIS_NOTE_MAX_CHARS = 500 # Per-analysis note truncation
39+
40+
# Rule-based thresholds for candidate screening (relaxed — LLM confirms)
41+
_FALLBACK_THRESHOLD = 0.4
42+
_LOW_COMPLETION_THRESHOLD = 0.35
43+
_HIGH_APPLIED_FOR_FIX = 0.4
44+
_MODERATE_EFFECTIVE_THRESHOLD = 0.55
45+
_MIN_APPLIED_FOR_DERIVED = 0.25
46+
47+
48+
# ---------------------------------------------------------------------------
49+
# Trigger 1: post-analysis
50+
# ---------------------------------------------------------------------------
51+
52+
async def process_analysis(
53+
evolver,
54+
analysis: "ExecutionAnalysis",
55+
) -> List["SkillRecord"]:
56+
"""Process all evolution suggestions from a completed analysis.
57+
58+
Called immediately after ``ExecutionAnalyzer.analyze_execution()``.
59+
Each suggestion becomes one evolution action, executed in parallel
60+
(throttled by semaphore).
61+
"""
62+
if not analysis.candidate_for_evolution:
63+
return []
64+
65+
contexts: List[EvolutionContext] = []
66+
for suggestion in analysis.evolution_suggestions:
67+
ctx = build_context_from_analysis(evolver, analysis, suggestion)
68+
if ctx is not None:
69+
contexts.append(ctx)
70+
71+
if not contexts:
72+
return []
73+
74+
results = await evolver._execute_contexts(contexts, "analysis")
75+
76+
if results:
77+
names = [r.name for r in results]
78+
logger.info(
79+
"[Trigger:analysis] Evolved %d skill(s): %s from task %s",
80+
len(results), names, analysis.task_id,
81+
)
82+
return results
83+
84+
85+
# ---------------------------------------------------------------------------
86+
# Trigger 2: tool degradation
87+
# ---------------------------------------------------------------------------
88+
89+
async def process_tool_degradation(
90+
evolver,
91+
problematic_tools: List["ToolQualityRecord"],
92+
) -> List["SkillRecord"]:
93+
"""Fix skills that depend on degraded tools.
94+
95+
Two-phase: rule-based candidate screening → LLM confirmation.
96+
97+
Anti-loop (state-driven):
98+
``evolver._addressed_degradations[tool_key]`` records skill names
99+
already evolved for that tool's degradation. Recovered tools are
100+
pruned so future re-degradation gets a fresh pass.
101+
"""
102+
if not problematic_tools:
103+
return []
104+
105+
# Prune recovered tools
106+
current_tool_keys = {t.tool_key for t in problematic_tools}
107+
recovered = [k for k in evolver._addressed_degradations if k not in current_tool_keys]
108+
for k in recovered:
109+
logger.debug("[Trigger:tool_degradation] Tool '%s' recovered, clearing addressed set", k)
110+
del evolver._addressed_degradations[k]
111+
112+
# Phase 1: screen & confirm candidates
113+
confirmed_contexts: List[EvolutionContext] = []
114+
seen_skills: set = set()
115+
116+
for tool_rec in problematic_tools:
117+
addressed = evolver._addressed_degradations.get(tool_rec.tool_key, set())
118+
119+
skill_ids = evolver._store.find_skills_by_tool(tool_rec.tool_key)
120+
for skill_id in skill_ids:
121+
skill_record = evolver._store.load_record(skill_id)
122+
if not skill_record or not skill_record.is_active:
123+
continue
124+
125+
if skill_record.skill_id in seen_skills:
126+
continue
127+
seen_skills.add(skill_record.skill_id)
128+
129+
if skill_record.skill_id in addressed:
130+
logger.debug(
131+
"[Trigger:tool_degradation] Skipping '%s' "
132+
"(already addressed for tool '%s')",
133+
skill_record.skill_id, tool_rec.tool_key,
134+
)
135+
continue
136+
137+
recent = evolver._store.load_analyses(
138+
skill_id=skill_record.skill_id, limit=_ANALYSIS_CONTEXT_MAX,
139+
)
140+
content = load_skill_content(evolver, skill_record)
141+
if not content:
142+
continue
143+
144+
issue_summary = (
145+
f"Tool `{tool_rec.tool_key}` degraded — "
146+
f"recent success rate: {tool_rec.recent_success_rate:.0%}, "
147+
f"total calls: {tool_rec.total_calls}, "
148+
f"LLM flagged: {tool_rec.llm_flagged_count} time(s)."
149+
)
150+
151+
direction = (
152+
f"Tool `{tool_rec.tool_key}` has degraded "
153+
f"(success_rate={tool_rec.recent_success_rate:.0%}). "
154+
f"Update skill instructions to handle this tool's "
155+
f"failures gracefully or suggest alternatives."
156+
)
157+
158+
confirmed = await evolver._llm_confirm_evolution(
159+
skill_record=skill_record,
160+
skill_content=content,
161+
proposed_type=EvolutionType.FIX,
162+
proposed_direction=direction,
163+
trigger_context=f"Tool degradation: {issue_summary}",
164+
recent_analyses=recent,
165+
)
166+
if not confirmed:
167+
logger.debug(
168+
"[Trigger:tool_degradation] LLM rejected evolution "
169+
"for skill '%s' (tool=%s)",
170+
skill_record.skill_id, tool_rec.tool_key,
171+
)
172+
evolver._addressed_degradations.setdefault(
173+
tool_rec.tool_key, set(),
174+
).add(skill_record.skill_id)
175+
continue
176+
177+
skill_dir = Path(skill_record.path).parent if skill_record.path else None
178+
confirmed_contexts.append(
179+
EvolutionContext(
180+
trigger=EvolutionTrigger.TOOL_DEGRADATION,
181+
suggestion=EvolutionSuggestion(
182+
evolution_type=EvolutionType.FIX,
183+
target_skill_ids=[skill_record.skill_id],
184+
direction=direction,
185+
),
186+
skill_records=[skill_record],
187+
skill_contents=[content],
188+
skill_dirs=[skill_dir] if skill_dir else [],
189+
recent_analyses=recent,
190+
tool_issue_summary=issue_summary,
191+
available_tools=evolver._available_tools,
192+
)
193+
)
194+
195+
evolver._addressed_degradations.setdefault(
196+
tool_rec.tool_key, set(),
197+
).add(skill_record.skill_id)
198+
199+
if not confirmed_contexts:
200+
return []
201+
202+
return await evolver._execute_contexts(confirmed_contexts, "tool_degradation")
203+
204+
205+
# ---------------------------------------------------------------------------
206+
# Trigger 3: metric monitor
207+
# ---------------------------------------------------------------------------
208+
209+
async def process_metric_check(
210+
evolver,
211+
min_selections: int = 5,
212+
) -> List["SkillRecord"]:
213+
"""Scan active skills and evolve those with poor health metrics.
214+
215+
Two-phase: rule-based candidate screening (relaxed thresholds) →
216+
LLM confirmation. Only considers skills with enough data.
217+
218+
Anti-loop (data-driven): newly-evolved skills start with
219+
``total_selections=0``, needing ``min_selections`` fresh executions.
220+
"""
221+
confirmed_contexts: List[EvolutionContext] = []
222+
all_active = evolver._store.load_active()
223+
224+
for skill_id, record in all_active.items():
225+
if record.total_selections < min_selections:
226+
continue
227+
228+
evo_type, direction = diagnose_skill_health(record)
229+
if evo_type is None:
230+
continue
231+
232+
content = load_skill_content(evolver, record)
233+
if not content:
234+
continue
235+
236+
recent = evolver._store.load_analyses(
237+
skill_id=record.skill_id, limit=_ANALYSIS_CONTEXT_MAX,
238+
)
239+
metric_summary = (
240+
f"selections={record.total_selections}, "
241+
f"applied_rate={record.applied_rate:.0%}, "
242+
f"completion_rate={record.completion_rate:.0%}, "
243+
f"effective_rate={record.effective_rate:.0%}, "
244+
f"fallback_rate={record.fallback_rate:.0%}"
245+
)
246+
247+
confirmed = await evolver._llm_confirm_evolution(
248+
skill_record=record,
249+
skill_content=content,
250+
proposed_type=evo_type,
251+
proposed_direction=direction,
252+
trigger_context=f"Metric check: {metric_summary}",
253+
recent_analyses=recent,
254+
)
255+
if not confirmed:
256+
logger.debug(
257+
"[Trigger:metric_monitor] LLM rejected evolution for skill '%s' (%s)",
258+
record.name, evo_type.value,
259+
)
260+
continue
261+
262+
skill_dir = Path(record.path).parent if record.path else None
263+
confirmed_contexts.append(
264+
EvolutionContext(
265+
trigger=EvolutionTrigger.METRIC_MONITOR,
266+
suggestion=EvolutionSuggestion(
267+
evolution_type=evo_type,
268+
target_skill_ids=[record.skill_id],
269+
direction=direction,
270+
),
271+
skill_records=[record],
272+
skill_contents=[content],
273+
skill_dirs=[skill_dir] if skill_dir else [],
274+
recent_analyses=recent,
275+
metric_summary=metric_summary,
276+
available_tools=evolver._available_tools,
277+
)
278+
)
279+
280+
if not confirmed_contexts:
281+
return []
282+
283+
return await evolver._execute_contexts(confirmed_contexts, "metric_monitor")
284+
285+
286+
# ---------------------------------------------------------------------------
287+
# Helpers
288+
# ---------------------------------------------------------------------------
289+
290+
def build_context_from_analysis(
291+
evolver,
292+
analysis: "ExecutionAnalysis",
293+
suggestion: "EvolutionSuggestion",
294+
) -> Optional[EvolutionContext]:
295+
"""Build EvolutionContext from a single analysis suggestion.
296+
297+
Loads all target skills referenced by ``suggestion.target_skill_ids``.
298+
For FIX: exactly 1 parent required.
299+
For DERIVED: 1+ parents (multi-parent = merge).
300+
For CAPTURED: parents list is empty.
301+
"""
302+
records: List["SkillRecord"] = []
303+
contents: List[str] = []
304+
dirs: List[Path] = []
305+
306+
if suggestion.evolution_type in (EvolutionType.FIX, EvolutionType.DERIVED):
307+
if not suggestion.target_skill_ids:
308+
logger.warning("FIX/DERIVED suggestion missing target_skill_ids")
309+
return None
310+
311+
for target_id in suggestion.target_skill_ids:
312+
rec = evolver._store.load_record(target_id)
313+
if not rec:
314+
logger.warning("Target skill not found: %s", target_id)
315+
return None
316+
content = load_skill_content(evolver, rec)
317+
if not content:
318+
logger.warning("Cannot load content for skill: %s", target_id)
319+
return None
320+
skill_dir = Path(rec.path).parent if rec.path else None
321+
322+
records.append(rec)
323+
contents.append(content)
324+
if skill_dir:
325+
dirs.append(skill_dir)
326+
327+
if suggestion.evolution_type == EvolutionType.FIX and len(records) != 1:
328+
logger.warning(
329+
"FIX requires exactly 1 target, got %d: %s",
330+
len(records), suggestion.target_skill_ids,
331+
)
332+
return None
333+
334+
return EvolutionContext(
335+
trigger=EvolutionTrigger.ANALYSIS,
336+
suggestion=suggestion,
337+
skill_records=records,
338+
skill_contents=contents,
339+
skill_dirs=dirs,
340+
source_task_id=analysis.task_id,
341+
recent_analyses=[analysis],
342+
available_tools=evolver._available_tools,
343+
)
344+
345+
346+
def load_skill_content(evolver, record: "SkillRecord") -> str:
347+
"""Load SKILL.md content from disk via registry or direct read."""
348+
content = evolver._registry.load_skill_content(record.skill_id)
349+
if content:
350+
return content
351+
if record.path:
352+
p = Path(record.path)
353+
if p.exists():
354+
try:
355+
return p.read_text(encoding="utf-8")
356+
except Exception:
357+
pass
358+
return ""
359+
360+
361+
def diagnose_skill_health(
362+
record: "SkillRecord",
363+
) -> tuple:
364+
"""Diagnose what type of evolution a skill needs based on metrics.
365+
366+
Returns ``(EvolutionType, direction_str)`` or ``(None, "")`` if healthy.
367+
Thresholds are intentionally relaxed — LLM confirmation filters
368+
false positives.
369+
"""
370+
if record.fallback_rate > _FALLBACK_THRESHOLD:
371+
return EvolutionType.FIX, (
372+
f"High fallback rate ({record.fallback_rate:.0%}): "
373+
f"skill is frequently selected but not applied, "
374+
f"suggesting instructions are unclear or outdated."
375+
)
376+
377+
if record.applied_rate > _HIGH_APPLIED_FOR_FIX and record.completion_rate < _LOW_COMPLETION_THRESHOLD:
378+
return EvolutionType.FIX, (
379+
f"Low completion rate ({record.completion_rate:.0%}) despite "
380+
f"high applied rate ({record.applied_rate:.0%}): "
381+
f"skill instructions may be incorrect or incomplete."
382+
)
383+
384+
if record.effective_rate < _MODERATE_EFFECTIVE_THRESHOLD and record.applied_rate > _MIN_APPLIED_FOR_DERIVED:
385+
return EvolutionType.DERIVED, (
386+
f"Moderate effectiveness ({record.effective_rate:.0%}): "
387+
f"skill works sometimes but could be enhanced with "
388+
f"better error handling or alternative approaches."
389+
)
390+
391+
return None, ""

0 commit comments

Comments
 (0)