Skip to content

Commit fc743e1

Browse files
DeepfreezechillBrian KrafftCopilot
authored
feat(evolution): extract trigger functions (Epic 5.3) (#41)
* feat(5.3): Extract trigger functions into evolution/triggers.py - New: evolution/triggers.py (380 lines) — process_analysis, process_tool_degradation, process_metric_check, build_context_from_analysis, load_skill_content, diagnose_skill_health - Modified: evolver.py — 6 methods now delegate to triggers module (-230 lines, 44.7KB from 58KB) - Constants moved: _ANALYSIS_CONTEXT_MAX, thresholds (re-imported in evolver.py) - New: tests/test_evolution_triggers.py — 22 tests - Suite: 1,468 passed, 127 skipped Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * fix(5.3): preserve MRO for helper calls, add happy-path tests Review fixes from /8eyes + /collab gate: - Route helper calls through evolver._method() to preserve MRO (GPT-5.4 MEDIUM) - Add 2 happy-path tests: confirm->execute for triggers 2 & 3 (Opus M-2) - Remove unused import logging (Opus L-1) - Move _ANALYSIS_NOTE_MAX_CHARS to evolver.py where used (Opus L-2) - Restore diagnose_skill_health return type annotation (Opus L-3) 24 trigger tests pass; 1,470 total pass, 127 skipped. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Brian Krafft <bkrafft@microsoft.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
1 parent 9df2f33 commit fc743e1

3 files changed

Lines changed: 849 additions & 323 deletions

File tree

Lines changed: 389 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,389 @@
1+
"""Evolution trigger functions — analysis, tool degradation, metric monitor.
2+
3+
Functions extracted from ``SkillEvolver`` (Epic 5.3):
4+
- ``process_analysis`` — Trigger 1: post-analysis evolution
5+
- ``process_tool_degradation`` — Trigger 2: fix skills for degraded tools
6+
- ``process_metric_check`` — Trigger 3: periodic health-based evolution
7+
- ``build_context_from_analysis`` — Build EvolutionContext from analyzer output
8+
- ``load_skill_content`` — Load SKILL.md content (registry or disk)
9+
- ``diagnose_skill_health`` — Pure metric classifier for Trigger 3
10+
"""
11+
12+
from __future__ import annotations
13+
14+
from pathlib import Path
15+
from typing import TYPE_CHECKING, List, Optional
16+
17+
from openspace.utils.logging import Logger
18+
19+
from .models import EvolutionContext, EvolutionTrigger
20+
21+
from openspace.skill_engine.types import EvolutionSuggestion, EvolutionType
22+
23+
if TYPE_CHECKING:
24+
from openspace.grounding.core.quality.types import ToolQualityRecord
25+
from openspace.skill_engine.types import (
26+
ExecutionAnalysis,
27+
SkillRecord,
28+
)
29+
30+
logger = Logger.get_logger(__name__)
31+
32+
# ---------------------------------------------------------------------------
33+
# Constants (moved from evolver.py — only used by trigger functions)
34+
# ---------------------------------------------------------------------------
35+
36+
_ANALYSIS_CONTEXT_MAX = 5 # Max recent analyses to include in prompt
37+
38+
# Rule-based thresholds for candidate screening (relaxed — LLM confirms)
39+
_FALLBACK_THRESHOLD = 0.4
40+
_LOW_COMPLETION_THRESHOLD = 0.35
41+
_HIGH_APPLIED_FOR_FIX = 0.4
42+
_MODERATE_EFFECTIVE_THRESHOLD = 0.55
43+
_MIN_APPLIED_FOR_DERIVED = 0.25
44+
45+
46+
# ---------------------------------------------------------------------------
47+
# Trigger 1: post-analysis
48+
# ---------------------------------------------------------------------------
49+
50+
async def process_analysis(
51+
evolver,
52+
analysis: "ExecutionAnalysis",
53+
) -> List["SkillRecord"]:
54+
"""Process all evolution suggestions from a completed analysis.
55+
56+
Called immediately after ``ExecutionAnalyzer.analyze_execution()``.
57+
Each suggestion becomes one evolution action, executed in parallel
58+
(throttled by semaphore).
59+
"""
60+
if not analysis.candidate_for_evolution:
61+
return []
62+
63+
contexts: List[EvolutionContext] = []
64+
for suggestion in analysis.evolution_suggestions:
65+
ctx = evolver._build_context_from_analysis(analysis, suggestion)
66+
if ctx is not None:
67+
contexts.append(ctx)
68+
69+
if not contexts:
70+
return []
71+
72+
results = await evolver._execute_contexts(contexts, "analysis")
73+
74+
if results:
75+
names = [r.name for r in results]
76+
logger.info(
77+
"[Trigger:analysis] Evolved %d skill(s): %s from task %s",
78+
len(results), names, analysis.task_id,
79+
)
80+
return results
81+
82+
83+
# ---------------------------------------------------------------------------
84+
# Trigger 2: tool degradation
85+
# ---------------------------------------------------------------------------
86+
87+
async def process_tool_degradation(
88+
evolver,
89+
problematic_tools: List["ToolQualityRecord"],
90+
) -> List["SkillRecord"]:
91+
"""Fix skills that depend on degraded tools.
92+
93+
Two-phase: rule-based candidate screening → LLM confirmation.
94+
95+
Anti-loop (state-driven):
96+
``evolver._addressed_degradations[tool_key]`` records skill names
97+
already evolved for that tool's degradation. Recovered tools are
98+
pruned so future re-degradation gets a fresh pass.
99+
"""
100+
if not problematic_tools:
101+
return []
102+
103+
# Prune recovered tools
104+
current_tool_keys = {t.tool_key for t in problematic_tools}
105+
recovered = [k for k in evolver._addressed_degradations if k not in current_tool_keys]
106+
for k in recovered:
107+
logger.debug("[Trigger:tool_degradation] Tool '%s' recovered, clearing addressed set", k)
108+
del evolver._addressed_degradations[k]
109+
110+
# Phase 1: screen & confirm candidates
111+
confirmed_contexts: List[EvolutionContext] = []
112+
seen_skills: set = set()
113+
114+
for tool_rec in problematic_tools:
115+
addressed = evolver._addressed_degradations.get(tool_rec.tool_key, set())
116+
117+
skill_ids = evolver._store.find_skills_by_tool(tool_rec.tool_key)
118+
for skill_id in skill_ids:
119+
skill_record = evolver._store.load_record(skill_id)
120+
if not skill_record or not skill_record.is_active:
121+
continue
122+
123+
if skill_record.skill_id in seen_skills:
124+
continue
125+
seen_skills.add(skill_record.skill_id)
126+
127+
if skill_record.skill_id in addressed:
128+
logger.debug(
129+
"[Trigger:tool_degradation] Skipping '%s' "
130+
"(already addressed for tool '%s')",
131+
skill_record.skill_id, tool_rec.tool_key,
132+
)
133+
continue
134+
135+
recent = evolver._store.load_analyses(
136+
skill_id=skill_record.skill_id, limit=_ANALYSIS_CONTEXT_MAX,
137+
)
138+
content = evolver._load_skill_content(skill_record)
139+
if not content:
140+
continue
141+
142+
issue_summary = (
143+
f"Tool `{tool_rec.tool_key}` degraded — "
144+
f"recent success rate: {tool_rec.recent_success_rate:.0%}, "
145+
f"total calls: {tool_rec.total_calls}, "
146+
f"LLM flagged: {tool_rec.llm_flagged_count} time(s)."
147+
)
148+
149+
direction = (
150+
f"Tool `{tool_rec.tool_key}` has degraded "
151+
f"(success_rate={tool_rec.recent_success_rate:.0%}). "
152+
f"Update skill instructions to handle this tool's "
153+
f"failures gracefully or suggest alternatives."
154+
)
155+
156+
confirmed = await evolver._llm_confirm_evolution(
157+
skill_record=skill_record,
158+
skill_content=content,
159+
proposed_type=EvolutionType.FIX,
160+
proposed_direction=direction,
161+
trigger_context=f"Tool degradation: {issue_summary}",
162+
recent_analyses=recent,
163+
)
164+
if not confirmed:
165+
logger.debug(
166+
"[Trigger:tool_degradation] LLM rejected evolution "
167+
"for skill '%s' (tool=%s)",
168+
skill_record.skill_id, tool_rec.tool_key,
169+
)
170+
evolver._addressed_degradations.setdefault(
171+
tool_rec.tool_key, set(),
172+
).add(skill_record.skill_id)
173+
continue
174+
175+
skill_dir = Path(skill_record.path).parent if skill_record.path else None
176+
confirmed_contexts.append(
177+
EvolutionContext(
178+
trigger=EvolutionTrigger.TOOL_DEGRADATION,
179+
suggestion=EvolutionSuggestion(
180+
evolution_type=EvolutionType.FIX,
181+
target_skill_ids=[skill_record.skill_id],
182+
direction=direction,
183+
),
184+
skill_records=[skill_record],
185+
skill_contents=[content],
186+
skill_dirs=[skill_dir] if skill_dir else [],
187+
recent_analyses=recent,
188+
tool_issue_summary=issue_summary,
189+
available_tools=evolver._available_tools,
190+
)
191+
)
192+
193+
evolver._addressed_degradations.setdefault(
194+
tool_rec.tool_key, set(),
195+
).add(skill_record.skill_id)
196+
197+
if not confirmed_contexts:
198+
return []
199+
200+
return await evolver._execute_contexts(confirmed_contexts, "tool_degradation")
201+
202+
203+
# ---------------------------------------------------------------------------
204+
# Trigger 3: metric monitor
205+
# ---------------------------------------------------------------------------
206+
207+
async def process_metric_check(
208+
evolver,
209+
min_selections: int = 5,
210+
) -> List["SkillRecord"]:
211+
"""Scan active skills and evolve those with poor health metrics.
212+
213+
Two-phase: rule-based candidate screening (relaxed thresholds) →
214+
LLM confirmation. Only considers skills with enough data.
215+
216+
Anti-loop (data-driven): newly-evolved skills start with
217+
``total_selections=0``, needing ``min_selections`` fresh executions.
218+
"""
219+
confirmed_contexts: List[EvolutionContext] = []
220+
all_active = evolver._store.load_active()
221+
222+
for skill_id, record in all_active.items():
223+
if record.total_selections < min_selections:
224+
continue
225+
226+
evo_type, direction = evolver._diagnose_skill_health(record)
227+
if evo_type is None:
228+
continue
229+
230+
content = evolver._load_skill_content(record)
231+
if not content:
232+
continue
233+
234+
recent = evolver._store.load_analyses(
235+
skill_id=record.skill_id, limit=_ANALYSIS_CONTEXT_MAX,
236+
)
237+
metric_summary = (
238+
f"selections={record.total_selections}, "
239+
f"applied_rate={record.applied_rate:.0%}, "
240+
f"completion_rate={record.completion_rate:.0%}, "
241+
f"effective_rate={record.effective_rate:.0%}, "
242+
f"fallback_rate={record.fallback_rate:.0%}"
243+
)
244+
245+
confirmed = await evolver._llm_confirm_evolution(
246+
skill_record=record,
247+
skill_content=content,
248+
proposed_type=evo_type,
249+
proposed_direction=direction,
250+
trigger_context=f"Metric check: {metric_summary}",
251+
recent_analyses=recent,
252+
)
253+
if not confirmed:
254+
logger.debug(
255+
"[Trigger:metric_monitor] LLM rejected evolution for skill '%s' (%s)",
256+
record.name, evo_type.value,
257+
)
258+
continue
259+
260+
skill_dir = Path(record.path).parent if record.path else None
261+
confirmed_contexts.append(
262+
EvolutionContext(
263+
trigger=EvolutionTrigger.METRIC_MONITOR,
264+
suggestion=EvolutionSuggestion(
265+
evolution_type=evo_type,
266+
target_skill_ids=[record.skill_id],
267+
direction=direction,
268+
),
269+
skill_records=[record],
270+
skill_contents=[content],
271+
skill_dirs=[skill_dir] if skill_dir else [],
272+
recent_analyses=recent,
273+
metric_summary=metric_summary,
274+
available_tools=evolver._available_tools,
275+
)
276+
)
277+
278+
if not confirmed_contexts:
279+
return []
280+
281+
return await evolver._execute_contexts(confirmed_contexts, "metric_monitor")
282+
283+
284+
# ---------------------------------------------------------------------------
285+
# Helpers
286+
# ---------------------------------------------------------------------------
287+
288+
def build_context_from_analysis(
289+
evolver,
290+
analysis: "ExecutionAnalysis",
291+
suggestion: "EvolutionSuggestion",
292+
) -> Optional[EvolutionContext]:
293+
"""Build EvolutionContext from a single analysis suggestion.
294+
295+
Loads all target skills referenced by ``suggestion.target_skill_ids``.
296+
For FIX: exactly 1 parent required.
297+
For DERIVED: 1+ parents (multi-parent = merge).
298+
For CAPTURED: parents list is empty.
299+
"""
300+
records: List["SkillRecord"] = []
301+
contents: List[str] = []
302+
dirs: List[Path] = []
303+
304+
if suggestion.evolution_type in (EvolutionType.FIX, EvolutionType.DERIVED):
305+
if not suggestion.target_skill_ids:
306+
logger.warning("FIX/DERIVED suggestion missing target_skill_ids")
307+
return None
308+
309+
for target_id in suggestion.target_skill_ids:
310+
rec = evolver._store.load_record(target_id)
311+
if not rec:
312+
logger.warning("Target skill not found: %s", target_id)
313+
return None
314+
content = evolver._load_skill_content(rec)
315+
if not content:
316+
logger.warning("Cannot load content for skill: %s", target_id)
317+
return None
318+
skill_dir = Path(rec.path).parent if rec.path else None
319+
320+
records.append(rec)
321+
contents.append(content)
322+
if skill_dir:
323+
dirs.append(skill_dir)
324+
325+
if suggestion.evolution_type == EvolutionType.FIX and len(records) != 1:
326+
logger.warning(
327+
"FIX requires exactly 1 target, got %d: %s",
328+
len(records), suggestion.target_skill_ids,
329+
)
330+
return None
331+
332+
return EvolutionContext(
333+
trigger=EvolutionTrigger.ANALYSIS,
334+
suggestion=suggestion,
335+
skill_records=records,
336+
skill_contents=contents,
337+
skill_dirs=dirs,
338+
source_task_id=analysis.task_id,
339+
recent_analyses=[analysis],
340+
available_tools=evolver._available_tools,
341+
)
342+
343+
344+
def load_skill_content(evolver, record: "SkillRecord") -> str:
345+
"""Load SKILL.md content from disk via registry or direct read."""
346+
content = evolver._registry.load_skill_content(record.skill_id)
347+
if content:
348+
return content
349+
if record.path:
350+
p = Path(record.path)
351+
if p.exists():
352+
try:
353+
return p.read_text(encoding="utf-8")
354+
except Exception:
355+
pass
356+
return ""
357+
358+
359+
def diagnose_skill_health(
360+
record: "SkillRecord",
361+
) -> tuple[Optional[EvolutionType], str]:
362+
"""Diagnose what type of evolution a skill needs based on metrics.
363+
364+
Returns ``(EvolutionType, direction_str)`` or ``(None, "")`` if healthy.
365+
Thresholds are intentionally relaxed — LLM confirmation filters
366+
false positives.
367+
"""
368+
if record.fallback_rate > _FALLBACK_THRESHOLD:
369+
return EvolutionType.FIX, (
370+
f"High fallback rate ({record.fallback_rate:.0%}): "
371+
f"skill is frequently selected but not applied, "
372+
f"suggesting instructions are unclear or outdated."
373+
)
374+
375+
if record.applied_rate > _HIGH_APPLIED_FOR_FIX and record.completion_rate < _LOW_COMPLETION_THRESHOLD:
376+
return EvolutionType.FIX, (
377+
f"Low completion rate ({record.completion_rate:.0%}) despite "
378+
f"high applied rate ({record.applied_rate:.0%}): "
379+
f"skill instructions may be incorrect or incomplete."
380+
)
381+
382+
if record.effective_rate < _MODERATE_EFFECTIVE_THRESHOLD and record.applied_rate > _MIN_APPLIED_FOR_DERIVED:
383+
return EvolutionType.DERIVED, (
384+
f"Moderate effectiveness ({record.effective_rate:.0%}): "
385+
f"skill works sometimes but could be enhanced with "
386+
f"better error handling or alternative approaches."
387+
)
388+
389+
return None, ""

0 commit comments

Comments
 (0)