diff --git a/openspace/skill_engine/evolution/loop.py b/openspace/skill_engine/evolution/loop.py new file mode 100644 index 00000000..46b8d533 --- /dev/null +++ b/openspace/skill_engine/evolution/loop.py @@ -0,0 +1,386 @@ +"""Evolution execution loop and apply-retry cycle. + +Provides the core execution engine used by all three evolution strategies: + +- ``run_evolution_loop`` — token-driven LLM agent loop with tool support +- ``parse_evolution_output`` — extract edit content or failure reason +- ``apply_with_retry`` — apply edit with retry on validation failure + +All functions accept an ``evolver`` parameter (the ``SkillEvolver`` instance) +and delegate back through ``evolver._method()`` to preserve method resolution +order for subclass / hook / telemetry compatibility. +""" + +from __future__ import annotations + +import shutil +from pathlib import Path +from typing import TYPE_CHECKING, Any, Dict, List, Optional + +from openspace.prompts import SkillEnginePrompts +from openspace.utils.logging import Logger + +from ..skill_utils import ( + extract_change_summary as _extract_change_summary, + strip_markdown_fences as _strip_markdown_fences, + truncate as _truncate, + validate_skill_dir as _validate_skill_dir, +) +from .confirmation import _RECORDING_MAX_CHARS, _SKILL_CONTENT_MAX_CHARS +from .models import EvolutionContext + +if TYPE_CHECKING: + from ..patch import SkillEditResult + +logger = Logger.get_logger("openspace.skill_engine.evolver") + +EVOLUTION_COMPLETE = SkillEnginePrompts.EVOLUTION_COMPLETE +EVOLUTION_FAILED = SkillEnginePrompts.EVOLUTION_FAILED + +# Agent loop / retry constants +_MAX_EVOLUTION_ITERATIONS = 5 # Max tool-calling rounds for evolution agent +_MAX_EVOLUTION_ATTEMPTS = 3 # Max apply-retry attempts per evolution + + +async def run_evolution_loop( + evolver, + prompt: str, + ctx: EvolutionContext, +) -> Optional[str]: + """Run evolution as a token-driven agent loop. + + Modeled after ``GroundingAgent.process()`` — the loop continues + until the LLM outputs an explicit completion/failure token, NOT + based on whether tools were called. + + Termination signals (checked every iteration, regardless of tool use): + - ``EVOLUTION_COMPLETE`` in assistant content → success, return edit. + - ``EVOLUTION_FAILED`` in assistant content → failure, return None. + + Tool availability: + - Iterations 1 … N-1: tools enabled (LLM may gather information). + - Iteration N (final): tools disabled, LLM must output a decision. + + Each non-final iteration without a token gets a nudge message + telling the LLM which iteration it is on and how many remain. + + Conversations are recorded to ``conversations.jsonl`` via + ``RecordingManager`` (agent_name="SkillEvolver") so the full + evolution dialogue is preserved for debugging and replay. + """ + from openspace.recording import RecordingManager + + model = evolver._model or evolver._llm_client.model + + # Merge tools from context and instance-level + evolution_tools: List = list(ctx.available_tools or []) + if not evolution_tools: + evolution_tools = list(evolver._available_tools) + + messages: List[Dict[str, Any]] = [ + {"role": "user", "content": prompt}, + ] + + # Record initial conversation setup (truncated for data minimization) + recorded_setup = [ + {"role": m["role"], "content": _truncate(m["content"], _RECORDING_MAX_CHARS)} + for m in messages + ] + await RecordingManager.record_conversation_setup( + setup_messages=recorded_setup, + tools=evolution_tools if evolution_tools else None, + agent_name="SkillEvolver", + extra={ + "evolution_type": ctx.suggestion.evolution_type.value, + "trigger": ctx.trigger.value, + "target_skills": ctx.suggestion.target_skill_ids, + }, + ) + + for iteration in range(_MAX_EVOLUTION_ITERATIONS): + is_last = iteration == _MAX_EVOLUTION_ITERATIONS - 1 + + # Snapshot message count before any additions + LLM call + msg_count_before = len(messages) + + # Final round: disable tools and force a decision + if is_last: + messages.append( + { + "role": "system", + "content": ( + f"This is your FINAL round (iteration " + f"{iteration + 1}/{_MAX_EVOLUTION_ITERATIONS}) — " + f"no more tool calls allowed. " + f"You MUST output the skill edit content now based on " + f"all information gathered so far. Follow the output " + f"format specified in the original instructions. " + f"End with {EVOLUTION_COMPLETE} if the edit is satisfactory, " + f"or {EVOLUTION_FAILED} with a reason if you cannot produce one." + ), + } + ) + + try: + result = await evolver._llm_client.complete( + messages=messages, + tools=evolution_tools if (evolution_tools and not is_last) else None, + execute_tools=True, + model=model, + ) + except Exception as e: + logger.error(f"Evolution LLM call failed (iter {iteration + 1}): {e}") + return None + + content = result["message"].get("content", "") + updated_messages = result["messages"] + has_tool_calls = result.get("has_tool_calls", False) + + # Record iteration delta (truncated for data minimization) + delta = updated_messages[msg_count_before:] + recorded_delta = [ + {"role": m["role"], "content": _truncate(m.get("content", ""), _RECORDING_MAX_CHARS)} + for m in delta + ] + await RecordingManager.record_iteration_context( + iteration=iteration + 1, + delta_messages=recorded_delta, + response_metadata={ + "has_tool_calls": has_tool_calls, + "tool_calls_count": len(result.get("tool_results", [])), + "has_completion_token": bool( + content and (EVOLUTION_COMPLETE in content or EVOLUTION_FAILED in content) + ), + }, + agent_name="SkillEvolver", + ) + + messages = updated_messages + + # ── Token check (every iteration, regardless of tool calls) ── + if content and (EVOLUTION_COMPLETE in content or EVOLUTION_FAILED in content): + edit_content, failure_reason = evolver._parse_evolution_output(content) + if failure_reason is not None: + targets = "+".join(ctx.suggestion.target_skill_ids) or "(new)" + logger.warning( + f"Evolution LLM signalled failure " + f"[{ctx.suggestion.evolution_type.value}] " + f"target={targets}: {failure_reason}" + ) + return None + return edit_content + + # No token found + if is_last: + # Final round exhausted without a decision + logger.warning( + f"Evolution agent finished {_MAX_EVOLUTION_ITERATIONS} iterations " + f"without signalling {EVOLUTION_COMPLETE} or {EVOLUTION_FAILED}" + ) + return None + + if has_tool_calls: + logger.debug(f"Evolution agent used tools (iter {iteration + 1}/{_MAX_EVOLUTION_ITERATIONS})") + else: + # No tools, no token — nudge the LLM + logger.debug( + f"Evolution agent produced content without token or tools " + f"(iter {iteration + 1}/{_MAX_EVOLUTION_ITERATIONS})" + ) + + # Iteration guidance + remaining = _MAX_EVOLUTION_ITERATIONS - iteration - 1 + messages.append( + { + "role": "system", + "content": ( + f"Iteration {iteration + 1}/{_MAX_EVOLUTION_ITERATIONS} complete " + f"({remaining} remaining). " + f"If your edit is ready, output it and include {EVOLUTION_COMPLETE} " + f"at the end. " + f"If you cannot complete this evolution, output {EVOLUTION_FAILED} " + f"with a reason. " + f"Otherwise, continue gathering information with tools." + ), + } + ) + + # Should never reach here (is_last handles the final iteration) + return None + + +def parse_evolution_output(content: str) -> tuple[Optional[str], Optional[str]]: + """Extract edit content or failure reason from LLM output. + + MUST only be called when ``EVOLUTION_COMPLETE`` or + ``EVOLUTION_FAILED`` is present in *content*. + + Returns ``(clean_content, failure_reason)``: + - ``(content, None)`` — ``EVOLUTION_COMPLETE`` found. + - ``(None, reason)`` — ``EVOLUTION_FAILED`` found. + """ + stripped = content.strip() + + # Failure takes priority (if both tokens appear, treat as failure) + if EVOLUTION_FAILED in stripped: + idx = stripped.index(EVOLUTION_FAILED) + reason_part = stripped[idx + len(EVOLUTION_FAILED) :].strip() + if reason_part.lower().startswith("reason:"): + reason_part = reason_part[len("reason:") :].strip() + reason = reason_part[:500] if reason_part else "LLM declined to produce edit (no reason given)" + return None, reason + + if EVOLUTION_COMPLETE in stripped: + clean = stripped.replace(EVOLUTION_COMPLETE, "").strip() + clean = _strip_markdown_fences(clean) + return clean, None + + # Caller guarantees a token is present; defensive fallback + return None, "No completion token found (unexpected)" + + +async def apply_with_retry( + evolver, + *, + apply_fn, + initial_content: str, + skill_dir: Path, + ctx: EvolutionContext, + prompt: str, + cleanup_on_retry: Optional[Path] = None, +) -> "Optional[SkillEditResult]": + """Apply an edit with retry on failure. + + If the first attempt fails (patch parse error, path mismatch, etc.), + feeds the error back to the LLM and asks for a corrected version. + + After successful application, runs structural validation. + + Retry conversations are recorded to ``conversations.jsonl`` under + agent_name="SkillEvolver.retry" so failed apply attempts and LLM + corrections are preserved for debugging. + + Args: + evolver: The SkillEvolver instance. + apply_fn: Callable that takes content str and returns SkillEditResult. + initial_content: First LLM-generated content to try. + skill_dir: Skill directory for validation. + ctx: Evolution context (for retry LLM calls). + prompt: Original prompt (for retry context). + cleanup_on_retry: Directory to remove before retrying (for derive/create). + """ + from openspace.recording import RecordingManager + + current_content = initial_content + msg_history: List[Dict[str, Any]] = [ + {"role": "user", "content": prompt}, + {"role": "assistant", "content": initial_content}, + ] + + # Track whether we've recorded the retry setup (only on first retry) + retry_setup_recorded = False + + for attempt in range(_MAX_EVOLUTION_ATTEMPTS): + # Clean up previous failed attempt (for derive/create) + if attempt > 0 and cleanup_on_retry and cleanup_on_retry.exists(): + shutil.rmtree(cleanup_on_retry, ignore_errors=True) + + # Apply the edit + edit_result = apply_fn(current_content) + + if edit_result.ok: + # Validate the result + validation_error = _validate_skill_dir(skill_dir) + if validation_error is None: + if attempt > 0: + logger.info(f"Apply-retry succeeded on attempt {attempt + 1}/{_MAX_EVOLUTION_ATTEMPTS}") + return edit_result + else: + # Validation failed — treat as error for retry + error_msg = f"Validation failed: {validation_error}" + logger.warning( + f"Apply succeeded but validation failed " + f"(attempt {attempt + 1}/{_MAX_EVOLUTION_ATTEMPTS}): " + f"{validation_error}" + ) + else: + error_msg = edit_result.error or "Unknown apply error" + logger.warning(f"Apply failed (attempt {attempt + 1}/{_MAX_EVOLUTION_ATTEMPTS}): {error_msg}") + + # Last attempt? Give up. + if attempt >= _MAX_EVOLUTION_ATTEMPTS - 1: + logger.error(f"Apply-retry exhausted after {_MAX_EVOLUTION_ATTEMPTS} attempts. Last error: {error_msg}") + # Clean up any partially created directory + if cleanup_on_retry and cleanup_on_retry.exists(): + shutil.rmtree(cleanup_on_retry, ignore_errors=True) + return None + + # Record retry setup on first retry attempt + if not retry_setup_recorded: + recorded_retry = [ + {"role": m["role"], "content": _truncate(m.get("content", ""), _RECORDING_MAX_CHARS)} + for m in msg_history + ] + await RecordingManager.record_conversation_setup( + setup_messages=recorded_retry, + agent_name="SkillEvolver.retry", + extra={ + "evolution_type": ctx.suggestion.evolution_type.value, + "target_skills": ctx.suggestion.target_skill_ids, + "first_error": error_msg[:300], + }, + ) + retry_setup_recorded = True + + # Feed error back to LLM for retry, including current file + # content so the LLM doesn't hallucinate what's on disk. + current_on_disk = evolver._format_skill_dir_content(skill_dir) if skill_dir.is_dir() else "" + retry_prompt = f"The previous edit was not successful. This was the error:\n\n{error_msg}\n\n" + if current_on_disk: + retry_prompt += ( + f"Here is the CURRENT content of the skill files on disk " + f"(use this as the ground truth for any SEARCH/REPLACE or " + f"context anchors):\n\n{_truncate(current_on_disk, _SKILL_CONTENT_MAX_CHARS)}\n\n" + ) + retry_prompt += "Please fix the issue and generate the edit again. Follow the same output format as before." + msg_history.append({"role": "user", "content": retry_prompt}) + + # Call LLM for corrected version (no tools — just fix the edit) + model = evolver._model or evolver._llm_client.model + try: + result = await evolver._llm_client.complete( + messages=msg_history, + model=model, + ) + new_content = result["message"].get("content", "") + if not new_content: + logger.warning("Retry LLM returned empty content") + continue + + new_content = _strip_markdown_fences(new_content) + # Strip evolution tokens that the LLM may include in retry responses + new_content = new_content.replace(EVOLUTION_COMPLETE, "").replace(EVOLUTION_FAILED, "").strip() + new_content, _ = _extract_change_summary(new_content) + msg_history.append({"role": "assistant", "content": new_content}) + current_content = new_content + + # Record retry iteration + await RecordingManager.record_iteration_context( + iteration=attempt + 1, + delta_messages=[ + {"role": "user", "content": _truncate(retry_prompt, _RECORDING_MAX_CHARS)}, + {"role": "assistant", "content": _truncate(new_content, _RECORDING_MAX_CHARS)}, + ], + response_metadata={ + "has_tool_calls": False, + "attempt": attempt + 1, + "error": error_msg[:300], + }, + agent_name="SkillEvolver.retry", + ) + + except Exception as e: + logger.error(f"Retry LLM call failed: {e}") + continue + + return None diff --git a/openspace/skill_engine/evolution/strategies.py b/openspace/skill_engine/evolution/strategies.py new file mode 100644 index 00000000..dd927241 --- /dev/null +++ b/openspace/skill_engine/evolution/strategies.py @@ -0,0 +1,380 @@ +"""Evolution strategies — FIX, DERIVED, and CAPTURED. + +Each strategy takes an ``evolver`` (``SkillEvolver`` instance) and an +``EvolutionContext``, runs an LLM agent loop, applies the edit with +retry, persists the new ``SkillRecord``, and registers the result. + +All internal calls go through ``evolver._method()`` to preserve method +resolution order for subclass / hook / telemetry compatibility. +""" + +from __future__ import annotations + +import uuid +from pathlib import Path +from typing import Optional + +from openspace.prompts import SkillEnginePrompts +from openspace.utils.logging import Logger + +from ..patch import ( + SKILL_FILENAME, + PatchType, + create_skill, + derive_skill, + fix_skill, +) +from ..registry import write_skill_id +from ..skill_utils import ( + extract_change_summary as _extract_change_summary, + get_frontmatter_field as _extract_frontmatter_field, + set_frontmatter_field as _set_frontmatter_field, + truncate as _truncate, +) +from ..types import ( + SkillCategory, + SkillLineage, + SkillOrigin, + SkillRecord, +) +from .confirmation import _SKILL_CONTENT_MAX_CHARS +from .models import EvolutionContext, _sanitize_skill_name +from .triggers import _ANALYSIS_CONTEXT_MAX + +logger = Logger.get_logger("openspace.skill_engine.evolver") + + +async def evolve_fix(evolver, ctx: EvolutionContext) -> Optional[SkillRecord]: + """In-place fix: same name, same directory, new version record. + + Uses agent loop for information gathering + apply-retry cycle. + """ + if not ctx.skill_records or not ctx.skill_contents or not ctx.skill_dirs: + logger.warning("FIX requires exactly 1 parent (skill_records/contents/dirs)") + return None + + parent = ctx.skill_records[0] + parent_content = ctx.skill_contents[0] + parent_dir = ctx.skill_dirs[0] + + # Build prompt with full directory content for multi-file skills + dir_content = evolver._format_skill_dir_content(parent_dir) + prompt = SkillEnginePrompts.evolution_fix( + current_content=_truncate(dir_content or parent_content, _SKILL_CONTENT_MAX_CHARS), + direction=ctx.suggestion.direction, + failure_context=evolver._format_analysis_context(ctx.recent_analyses), + tool_issue_summary=ctx.tool_issue_summary, + metric_summary=ctx.metric_summary, + ) + + # Agent loop: LLM can gather information via tools before generating edits + new_content = await evolver._run_evolution_loop(prompt, ctx) + if not new_content: + return None + + # Extract change_summary from LLM output (first line if prefixed) + new_content, change_summary = _extract_change_summary(new_content) + + # Apply-retry cycle + edit_result = await evolver._apply_with_retry( + apply_fn=lambda content: fix_skill(parent_dir, content, PatchType.AUTO), + initial_content=new_content, + skill_dir=parent_dir, + ctx=ctx, + prompt=prompt, + ) + if edit_result is None or not edit_result.ok: + return None + + # Re-read name/description from the updated SKILL.md on disk — + # the LLM may have refined the description (or even name) during the fix. + updated_skill_md = edit_result.content_snapshot.get(SKILL_FILENAME, "") + fixed_name = _extract_frontmatter_field(updated_skill_md, "name") or parent.name + fixed_desc = _extract_frontmatter_field(updated_skill_md, "description") or parent.description + + new_id = f"{fixed_name}__v{parent.lineage.generation + 1}_{uuid.uuid4().hex[:8]}" + model = evolver._model or evolver._llm_client.model + + new_record = SkillRecord( + skill_id=new_id, + name=fixed_name, + description=fixed_desc, + path=parent.path, + category=parent.category, + tags=list(parent.tags), + visibility=parent.visibility, + creator_id=parent.creator_id, + lineage=SkillLineage( + origin=SkillOrigin.FIXED, + generation=parent.lineage.generation + 1, + parent_skill_ids=[parent.skill_id], + source_task_id=ctx.source_task_id, + change_summary=change_summary or ctx.suggestion.direction, + content_diff=edit_result.content_diff, + content_snapshot=edit_result.content_snapshot, + created_by=model, + ), + tool_dependencies=list(parent.tool_dependencies), + critical_tools=list(parent.critical_tools), + ) + + await evolver._store.evolve_skill(new_record, [parent.skill_id]) + + # Stamp the new skill_id into the sidecar file so next discover() + write_skill_id(parent_dir, new_id) + + from ..registry import SkillMeta + + new_meta = SkillMeta( + skill_id=new_id, + name=fixed_name, + description=fixed_desc, + path=Path(parent.path), + ) + evolver._registry.update_skill(parent.skill_id, new_meta) + + logger.info( + f"FIX: {parent.name} gen{parent.lineage.generation} → gen{new_record.lineage.generation} [{new_id}]" + ) + return new_record + + +async def evolve_derived(evolver, ctx: EvolutionContext) -> Optional[SkillRecord]: + """Create enhanced version in a new directory. + + Supports single-parent (enhance) and multi-parent (merge/fuse). + Uses agent loop for information gathering + apply-retry cycle. + """ + if not ctx.skill_records or not ctx.skill_contents or not ctx.skill_dirs: + logger.warning("DERIVED requires at least one parent skill_record + content + dir") + return None + + first_parent = ctx.skill_records[0] # For fallback defaults only + is_merge = len(ctx.skill_records) > 1 + + # Build prompt — include all parent contents for multi-parent merge + if is_merge: + parent_sections = [] + for i, (rec, sd) in enumerate(zip(ctx.skill_records, ctx.skill_dirs)): + dir_content = evolver._format_skill_dir_content(sd) + label = f"Parent {i + 1}: {rec.name}" + parent_sections.append( + f"## {label}\n{_truncate(dir_content or ctx.skill_contents[i], _SKILL_CONTENT_MAX_CHARS)}" + ) + combined_content = "\n\n---\n\n".join(parent_sections) + else: + dir_content = evolver._format_skill_dir_content(ctx.skill_dirs[0]) + combined_content = _truncate(dir_content or ctx.skill_contents[0], _SKILL_CONTENT_MAX_CHARS) + + prompt = SkillEnginePrompts.evolution_derived( + parent_content=combined_content, + direction=ctx.suggestion.direction, + execution_insights=evolver._format_analysis_context(ctx.recent_analyses), + metric_summary=ctx.metric_summary, + ) + + # Agent loop + new_content = await evolver._run_evolution_loop(prompt, ctx) + if not new_content: + return None + + new_content, change_summary = _extract_change_summary(new_content) + + # Determine new skill name from frontmatter, or generate one + new_name = _extract_frontmatter_field(new_content, "name") + if not new_name or new_name == first_parent.name: + suffix = "-merged" if is_merge else "-enhanced" + new_name = f"{first_parent.name}{suffix}" + new_content = _set_frontmatter_field(new_content, "name", new_name) + + # Cap name length to avoid ever-growing chains like + # "panel-component-enhanced-enhanced-merged_abc123" + new_name = _sanitize_skill_name(new_name) + new_content = _set_frontmatter_field(new_content, "name", new_name) + + # Directory name always matches the skill name + target_dir = ctx.skill_dirs[0].parent / new_name + if target_dir.exists(): + new_name = f"{new_name}-{uuid.uuid4().hex[:6]}" + new_name = _sanitize_skill_name(new_name) + target_dir = ctx.skill_dirs[0].parent / new_name + new_content = _set_frontmatter_field(new_content, "name", new_name) + + # Apply-retry cycle for derive_skill + edit_result = await evolver._apply_with_retry( + apply_fn=lambda content: derive_skill(ctx.skill_dirs, target_dir, content, PatchType.AUTO), + initial_content=new_content, + skill_dir=target_dir, + ctx=ctx, + prompt=prompt, + cleanup_on_retry=target_dir, # Remove failed target dir before retry + ) + if edit_result is None or not edit_result.ok: + return None + + # Extract description from new content + new_desc = _extract_frontmatter_field(new_content, "description") or first_parent.description + + # Collect parent info from ALL parents + parent_ids = [r.skill_id for r in ctx.skill_records] + max_gen = max(r.lineage.generation for r in ctx.skill_records) + all_tool_deps: set = set() + all_critical: set = set() + all_tags: set = set() + for rec in ctx.skill_records: + all_tool_deps.update(rec.tool_dependencies) + all_critical.update(rec.critical_tools) + all_tags.update(rec.tags) + + new_id = f"{new_name}__v0_{uuid.uuid4().hex[:8]}" + model = evolver._model or evolver._llm_client.model + + new_record = SkillRecord( + skill_id=new_id, + name=new_name, + description=new_desc, + path=str(target_dir / SKILL_FILENAME), + category=ctx.suggestion.category or first_parent.category, + tags=sorted(all_tags), + visibility=first_parent.visibility, + creator_id=first_parent.creator_id, + lineage=SkillLineage( + origin=SkillOrigin.DERIVED, + generation=max_gen + 1, + parent_skill_ids=parent_ids, + source_task_id=ctx.source_task_id, + change_summary=change_summary or ctx.suggestion.direction, + content_diff=edit_result.content_diff, + content_snapshot=edit_result.content_snapshot, + created_by=model, + ), + tool_dependencies=sorted(all_tool_deps), + critical_tools=sorted(all_critical), + ) + + await evolver._store.evolve_skill(new_record, parent_ids) + + # Stamp skill_id sidecar so discover() uses this ID on restart + write_skill_id(target_dir, new_id) + + # Register the new skill so it's immediately available for selection + from ..registry import SkillMeta + + new_meta = SkillMeta( + skill_id=new_id, + name=new_name, + description=new_desc, + path=target_dir / SKILL_FILENAME, + ) + evolver._registry.add_skill(new_meta) + + parent_names = " + ".join(r.name for r in ctx.skill_records) + logger.info(f"DERIVED: {parent_names} → {new_name} [{new_id}]") + return new_record + + +async def evolve_captured(evolver, ctx: EvolutionContext) -> Optional[SkillRecord]: + """Capture a novel pattern as a brand-new skill. + + Uses agent loop for information gathering + apply-retry cycle. + """ + # Build prompt and call LLM + # For CAPTURED, we use analyses as context (the tasks where the pattern was observed) + task_descriptions = [] + for a in ctx.recent_analyses[:_ANALYSIS_CONTEXT_MAX]: + if a.execution_note: + task_descriptions.append(f"- task={a.task_id}: {a.execution_note[:200]}") + + prompt = SkillEnginePrompts.evolution_captured( + direction=ctx.suggestion.direction, + category=(ctx.suggestion.category or SkillCategory.WORKFLOW).value, + execution_highlights="\n".join(task_descriptions) if task_descriptions else "(no task context available)", + ) + + # Agent loop + new_content = await evolver._run_evolution_loop(prompt, ctx) + if not new_content: + return None + + new_content, change_summary = _extract_change_summary(new_content) + + # Extract name/description from the generated content + new_name = _extract_frontmatter_field(new_content, "name") + new_desc = _extract_frontmatter_field(new_content, "description") + if not new_name: + logger.warning("CAPTURED: LLM did not produce a valid skill name") + return None + + # Sanitize name (enforce length limit + valid chars) + new_name = _sanitize_skill_name(new_name) + new_content = _set_frontmatter_field(new_content, "name", new_name) + + # Create new skill directory via create_skill (handles multi-file FULL) + skill_dirs = evolver._registry._skill_dirs + if not skill_dirs: + logger.warning("CAPTURED: no skill directories configured") + return None + + # Directory name always matches the skill name + base_dir = skill_dirs[0] # Primary user skill directory + target_dir = base_dir / new_name + if target_dir.exists(): + new_name = f"{new_name}-{uuid.uuid4().hex[:6]}" + new_name = _sanitize_skill_name(new_name) + target_dir = base_dir / new_name + new_content = _set_frontmatter_field(new_content, "name", new_name) + + # Apply-retry cycle for create_skill + edit_result = await evolver._apply_with_retry( + apply_fn=lambda content: create_skill(target_dir, content, PatchType.AUTO), + initial_content=new_content, + skill_dir=target_dir, + ctx=ctx, + prompt=prompt, + cleanup_on_retry=target_dir, + ) + if edit_result is None or not edit_result.ok: + return None + + snapshot = edit_result.content_snapshot + add_all_diff = edit_result.content_diff + + new_id = f"{new_name}__v0_{uuid.uuid4().hex[:8]}" + model = evolver._model or evolver._llm_client.model + + new_record = SkillRecord( + skill_id=new_id, + name=new_name, + description=new_desc or new_name, + path=str(target_dir / SKILL_FILENAME), + category=ctx.suggestion.category or SkillCategory.WORKFLOW, + lineage=SkillLineage( + origin=SkillOrigin.CAPTURED, + generation=0, + parent_skill_ids=[], + source_task_id=ctx.source_task_id, + change_summary=change_summary or ctx.suggestion.direction, + content_diff=add_all_diff, + content_snapshot=snapshot, + created_by=model, + ), + ) + + await evolver._store.save_record(new_record) + + # Stamp skill_id sidecar so discover() uses this ID on restart + write_skill_id(target_dir, new_id) + + # Register the new skill so it's immediately available + from ..registry import SkillMeta + + new_meta = SkillMeta( + skill_id=new_id, + name=new_name, + description=new_desc or new_name, + path=target_dir / SKILL_FILENAME, + ) + evolver._registry.add_skill(new_meta) + + logger.info(f"CAPTURED: {new_name} [{new_id}]") + return new_record diff --git a/openspace/skill_engine/evolver.py b/openspace/skill_engine/evolver.py index 717a57a0..0204a791 100644 --- a/openspace/skill_engine/evolver.py +++ b/openspace/skill_engine/evolver.py @@ -17,21 +17,14 @@ from __future__ import annotations import asyncio -import copy -import json -import re -import shutil -import uuid from pathlib import Path from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set -from openspace.prompts import SkillEnginePrompts from openspace.utils.logging import Logger from .evolution.models import ( EvolutionContext, EvolutionTrigger, - _sanitize_skill_name, ) from .evolution.orchestrator import ( dispatch_evolution as _dispatch_evolution, @@ -40,11 +33,23 @@ schedule_background as _schedule_background_impl, ) from .evolution.confirmation import ( - _RECORDING_MAX_CHARS, - _SKILL_CONTENT_MAX_CHARS, llm_confirm_evolution as _llm_confirm_evolution_impl, parse_confirmation as _parse_confirmation_impl, ) +from .evolution.loop import ( + EVOLUTION_COMPLETE, + EVOLUTION_FAILED, + _MAX_EVOLUTION_ATTEMPTS, + _MAX_EVOLUTION_ITERATIONS, + apply_with_retry as _apply_with_retry_impl, + parse_evolution_output as _parse_evolution_output_impl, + run_evolution_loop as _run_evolution_loop_impl, +) +from .evolution.strategies import ( + evolve_captured as _evolve_captured_impl, + evolve_derived as _evolve_derived_impl, + evolve_fix as _evolve_fix_impl, +) from .evolution.triggers import ( _ANALYSIS_CONTEXT_MAX, build_context_from_analysis as _build_context_from_analysis_impl, @@ -56,40 +61,14 @@ ) from .patch import ( SKILL_FILENAME, - PatchType, SkillEditResult, collect_skill_snapshot, - create_skill, - derive_skill, - fix_skill, -) -from .registry import write_skill_id -from .skill_utils import ( - extract_change_summary as _extract_change_summary, -) -from .skill_utils import ( - get_frontmatter_field as _extract_frontmatter_field, -) -from .skill_utils import ( - set_frontmatter_field as _set_frontmatter_field, -) -from .skill_utils import ( - strip_markdown_fences as _strip_markdown_fences, -) -from .skill_utils import ( - truncate as _truncate, -) -from .skill_utils import ( - validate_skill_dir as _validate_skill_dir, ) from .store import SkillStore from .types import ( EvolutionSuggestion, EvolutionType, ExecutionAnalysis, - SkillCategory, - SkillLineage, - SkillOrigin, SkillRecord, ) @@ -102,13 +81,6 @@ logger = Logger.get_logger(__name__) -EVOLUTION_COMPLETE = SkillEnginePrompts.EVOLUTION_COMPLETE -EVOLUTION_FAILED = SkillEnginePrompts.EVOLUTION_FAILED - -# Agent loop / retry constants -_MAX_EVOLUTION_ITERATIONS = 5 # Max tool-calling rounds for evolution agent -_MAX_EVOLUTION_ATTEMPTS = 3 # Max apply-retry attempts per evolution - class SkillEvolver: """Execute skill evolution actions. @@ -270,533 +242,26 @@ async def _llm_confirm_evolution( _parse_confirmation = staticmethod(_parse_confirmation_impl) async def _evolve_fix(self, ctx: EvolutionContext) -> Optional[SkillRecord]: - """In-place fix: same name, same directory, new version record. - - Uses agent loop for information gathering + apply-retry cycle. - """ - if not ctx.skill_records or not ctx.skill_contents or not ctx.skill_dirs: - logger.warning("FIX requires exactly 1 parent (skill_records/contents/dirs)") - return None - - parent = ctx.skill_records[0] - parent_content = ctx.skill_contents[0] - parent_dir = ctx.skill_dirs[0] - - # Build prompt with full directory content for multi-file skills - dir_content = self._format_skill_dir_content(parent_dir) - prompt = SkillEnginePrompts.evolution_fix( - current_content=_truncate(dir_content or parent_content, _SKILL_CONTENT_MAX_CHARS), - direction=ctx.suggestion.direction, - failure_context=self._format_analysis_context(ctx.recent_analyses), - tool_issue_summary=ctx.tool_issue_summary, - metric_summary=ctx.metric_summary, - ) - - # Agent loop: LLM can gather information via tools before generating edits - new_content = await self._run_evolution_loop(prompt, ctx) - if not new_content: - return None - - # Extract change_summary from LLM output (first line if prefixed) - new_content, change_summary = _extract_change_summary(new_content) - - # Apply-retry cycle - edit_result = await self._apply_with_retry( - apply_fn=lambda content: fix_skill(parent_dir, content, PatchType.AUTO), - initial_content=new_content, - skill_dir=parent_dir, - ctx=ctx, - prompt=prompt, - ) - if edit_result is None or not edit_result.ok: - return None - - # Re-read name/description from the updated SKILL.md on disk — - # the LLM may have refined the description (or even name) during the fix. - updated_skill_md = edit_result.content_snapshot.get(SKILL_FILENAME, "") - fixed_name = _extract_frontmatter_field(updated_skill_md, "name") or parent.name - fixed_desc = _extract_frontmatter_field(updated_skill_md, "description") or parent.description - - new_id = f"{fixed_name}__v{parent.lineage.generation + 1}_{uuid.uuid4().hex[:8]}" - model = self._model or self._llm_client.model - - new_record = SkillRecord( - skill_id=new_id, - name=fixed_name, - description=fixed_desc, - path=parent.path, - category=parent.category, - tags=list(parent.tags), - visibility=parent.visibility, - creator_id=parent.creator_id, - lineage=SkillLineage( - origin=SkillOrigin.FIXED, - generation=parent.lineage.generation + 1, - parent_skill_ids=[parent.skill_id], - source_task_id=ctx.source_task_id, - change_summary=change_summary or ctx.suggestion.direction, - content_diff=edit_result.content_diff, - content_snapshot=edit_result.content_snapshot, - created_by=model, - ), - tool_dependencies=list(parent.tool_dependencies), - critical_tools=list(parent.critical_tools), - ) - - await self._store.evolve_skill(new_record, [parent.skill_id]) - - # Stamp the new skill_id into the sidecar file so next discover() - write_skill_id(parent_dir, new_id) - - from .registry import SkillMeta - - new_meta = SkillMeta( - skill_id=new_id, - name=fixed_name, - description=fixed_desc, - path=Path(parent.path), - ) - self._registry.update_skill(parent.skill_id, new_meta) - - logger.info( - f"FIX: {parent.name} gen{parent.lineage.generation} → gen{new_record.lineage.generation} [{new_id}]" - ) - return new_record + """In-place fix — delegates to ``evolution.strategies.evolve_fix``.""" + return await _evolve_fix_impl(self, ctx) async def _evolve_derived(self, ctx: EvolutionContext) -> Optional[SkillRecord]: - """Create enhanced version in a new directory. - - Supports single-parent (enhance) and multi-parent (merge/fuse). - Uses agent loop for information gathering + apply-retry cycle. - """ - if not ctx.skill_records or not ctx.skill_contents or not ctx.skill_dirs: - logger.warning("DERIVED requires at least one parent skill_record + content + dir") - return None - - first_parent = ctx.skill_records[0] # For fallback defaults only - is_merge = len(ctx.skill_records) > 1 - - # Build prompt — include all parent contents for multi-parent merge - if is_merge: - parent_sections = [] - for i, (rec, sd) in enumerate(zip(ctx.skill_records, ctx.skill_dirs)): - dir_content = self._format_skill_dir_content(sd) - label = f"Parent {i + 1}: {rec.name}" - parent_sections.append( - f"## {label}\n{_truncate(dir_content or ctx.skill_contents[i], _SKILL_CONTENT_MAX_CHARS)}" - ) - combined_content = "\n\n---\n\n".join(parent_sections) - else: - dir_content = self._format_skill_dir_content(ctx.skill_dirs[0]) - combined_content = _truncate(dir_content or ctx.skill_contents[0], _SKILL_CONTENT_MAX_CHARS) - - prompt = SkillEnginePrompts.evolution_derived( - parent_content=combined_content, - direction=ctx.suggestion.direction, - execution_insights=self._format_analysis_context(ctx.recent_analyses), - metric_summary=ctx.metric_summary, - ) - - # Agent loop - new_content = await self._run_evolution_loop(prompt, ctx) - if not new_content: - return None - - new_content, change_summary = _extract_change_summary(new_content) - - # Determine new skill name from frontmatter, or generate one - new_name = _extract_frontmatter_field(new_content, "name") - if not new_name or new_name == first_parent.name: - suffix = "-merged" if is_merge else "-enhanced" - new_name = f"{first_parent.name}{suffix}" - new_content = _set_frontmatter_field(new_content, "name", new_name) - - # Cap name length to avoid ever-growing chains like - # "panel-component-enhanced-enhanced-merged_abc123" - new_name = _sanitize_skill_name(new_name) - new_content = _set_frontmatter_field(new_content, "name", new_name) - - # Directory name always matches the skill name - target_dir = ctx.skill_dirs[0].parent / new_name - if target_dir.exists(): - new_name = f"{new_name}-{uuid.uuid4().hex[:6]}" - new_name = _sanitize_skill_name(new_name) - target_dir = ctx.skill_dirs[0].parent / new_name - new_content = _set_frontmatter_field(new_content, "name", new_name) - - # Apply-retry cycle for derive_skill - edit_result = await self._apply_with_retry( - apply_fn=lambda content: derive_skill(ctx.skill_dirs, target_dir, content, PatchType.AUTO), - initial_content=new_content, - skill_dir=target_dir, - ctx=ctx, - prompt=prompt, - cleanup_on_retry=target_dir, # Remove failed target dir before retry - ) - if edit_result is None or not edit_result.ok: - return None - - # Extract description from new content - new_desc = _extract_frontmatter_field(new_content, "description") or first_parent.description - - # Collect parent info from ALL parents - parent_ids = [r.skill_id for r in ctx.skill_records] - max_gen = max(r.lineage.generation for r in ctx.skill_records) - all_tool_deps: set = set() - all_critical: set = set() - all_tags: set = set() - for rec in ctx.skill_records: - all_tool_deps.update(rec.tool_dependencies) - all_critical.update(rec.critical_tools) - all_tags.update(rec.tags) - - new_id = f"{new_name}__v0_{uuid.uuid4().hex[:8]}" - model = self._model or self._llm_client.model - - new_record = SkillRecord( - skill_id=new_id, - name=new_name, - description=new_desc, - path=str(target_dir / SKILL_FILENAME), - category=ctx.suggestion.category or first_parent.category, - tags=sorted(all_tags), - visibility=first_parent.visibility, - creator_id=first_parent.creator_id, - lineage=SkillLineage( - origin=SkillOrigin.DERIVED, - generation=max_gen + 1, - parent_skill_ids=parent_ids, - source_task_id=ctx.source_task_id, - change_summary=change_summary or ctx.suggestion.direction, - content_diff=edit_result.content_diff, - content_snapshot=edit_result.content_snapshot, - created_by=model, - ), - tool_dependencies=sorted(all_tool_deps), - critical_tools=sorted(all_critical), - ) - - await self._store.evolve_skill(new_record, parent_ids) - - # Stamp skill_id sidecar so discover() uses this ID on restart - write_skill_id(target_dir, new_id) - - # Register the new skill so it's immediately available for selection - from .registry import SkillMeta - - new_meta = SkillMeta( - skill_id=new_id, - name=new_name, - description=new_desc, - path=target_dir / SKILL_FILENAME, - ) - self._registry.add_skill(new_meta) - - parent_names = " + ".join(r.name for r in ctx.skill_records) - logger.info(f"DERIVED: {parent_names} → {new_name} [{new_id}]") - return new_record + """Create enhanced version — delegates to ``evolution.strategies.evolve_derived``.""" + return await _evolve_derived_impl(self, ctx) async def _evolve_captured(self, ctx: EvolutionContext) -> Optional[SkillRecord]: - """Capture a novel pattern as a brand-new skill. - - Uses agent loop for information gathering + apply-retry cycle. - """ - # Build prompt and call LLM - # For CAPTURED, we use analyses as context (the tasks where the pattern was observed) - task_descriptions = [] - for a in ctx.recent_analyses[:_ANALYSIS_CONTEXT_MAX]: - if a.execution_note: - task_descriptions.append(f"- task={a.task_id}: {a.execution_note[:200]}") - - prompt = SkillEnginePrompts.evolution_captured( - direction=ctx.suggestion.direction, - category=(ctx.suggestion.category or SkillCategory.WORKFLOW).value, - execution_highlights="\n".join(task_descriptions) if task_descriptions else "(no task context available)", - ) - - # Agent loop - new_content = await self._run_evolution_loop(prompt, ctx) - if not new_content: - return None - - new_content, change_summary = _extract_change_summary(new_content) - - # Extract name/description from the generated content - new_name = _extract_frontmatter_field(new_content, "name") - new_desc = _extract_frontmatter_field(new_content, "description") - if not new_name: - logger.warning("CAPTURED: LLM did not produce a valid skill name") - return None - - # Sanitize name (enforce length limit + valid chars) - new_name = _sanitize_skill_name(new_name) - new_content = _set_frontmatter_field(new_content, "name", new_name) - - # Create new skill directory via create_skill (handles multi-file FULL) - skill_dirs = self._registry._skill_dirs - if not skill_dirs: - logger.warning("CAPTURED: no skill directories configured") - return None - - # Directory name always matches the skill name - base_dir = skill_dirs[0] # Primary user skill directory - target_dir = base_dir / new_name - if target_dir.exists(): - new_name = f"{new_name}-{uuid.uuid4().hex[:6]}" - new_name = _sanitize_skill_name(new_name) - target_dir = base_dir / new_name - new_content = _set_frontmatter_field(new_content, "name", new_name) - - # Apply-retry cycle for create_skill - edit_result = await self._apply_with_retry( - apply_fn=lambda content: create_skill(target_dir, content, PatchType.AUTO), - initial_content=new_content, - skill_dir=target_dir, - ctx=ctx, - prompt=prompt, - cleanup_on_retry=target_dir, - ) - if edit_result is None or not edit_result.ok: - return None - - snapshot = edit_result.content_snapshot - add_all_diff = edit_result.content_diff - - new_id = f"{new_name}__v0_{uuid.uuid4().hex[:8]}" - model = self._model or self._llm_client.model - - new_record = SkillRecord( - skill_id=new_id, - name=new_name, - description=new_desc or new_name, - path=str(target_dir / SKILL_FILENAME), - category=ctx.suggestion.category or SkillCategory.WORKFLOW, - lineage=SkillLineage( - origin=SkillOrigin.CAPTURED, - generation=0, - parent_skill_ids=[], - source_task_id=ctx.source_task_id, - change_summary=change_summary or ctx.suggestion.direction, - content_diff=add_all_diff, - content_snapshot=snapshot, - created_by=model, - ), - ) - - await self._store.save_record(new_record) - - # Stamp skill_id sidecar so discover() uses this ID on restart - write_skill_id(target_dir, new_id) - - # Register the new skill so it's immediately available - from .registry import SkillMeta - - new_meta = SkillMeta( - skill_id=new_id, - name=new_name, - description=new_desc or new_name, - path=target_dir / SKILL_FILENAME, - ) - self._registry.add_skill(new_meta) - - logger.info(f"CAPTURED: {new_name} [{new_id}]") - return new_record + """Capture novel pattern — delegates to ``evolution.strategies.evolve_captured``.""" + return await _evolve_captured_impl(self, ctx) async def _run_evolution_loop( self, prompt: str, ctx: EvolutionContext, ) -> Optional[str]: - """Run evolution as a token-driven agent loop. + """Token-driven LLM agent loop — delegates to ``evolution.loop.run_evolution_loop``.""" + return await _run_evolution_loop_impl(self, prompt, ctx) - Modeled after ``GroundingAgent.process()`` — the loop continues - until the LLM outputs an explicit completion/failure token, NOT - based on whether tools were called. - - Termination signals (checked every iteration, regardless of tool use): - - ``EVOLUTION_COMPLETE`` in assistant content → success, return edit. - - ``EVOLUTION_FAILED`` in assistant content → failure, return None. - - Tool availability: - - Iterations 1 … N-1: tools enabled (LLM may gather information). - - Iteration N (final): tools disabled, LLM must output a decision. - - Each non-final iteration without a token gets a nudge message - telling the LLM which iteration it is on and how many remain. - - Conversations are recorded to ``conversations.jsonl`` via - ``RecordingManager`` (agent_name="SkillEvolver") so the full - evolution dialogue is preserved for debugging and replay. - """ - from openspace.recording import RecordingManager - - model = self._model or self._llm_client.model - - # Merge tools from context and instance-level - evolution_tools: List["BaseTool"] = list(ctx.available_tools or []) - if not evolution_tools: - evolution_tools = list(self._available_tools) - - messages: List[Dict[str, Any]] = [ - {"role": "user", "content": prompt}, - ] - - # Record initial conversation setup (truncated for data minimization) - recorded_setup = [ - {"role": m["role"], "content": _truncate(m["content"], _RECORDING_MAX_CHARS)} - for m in messages - ] - await RecordingManager.record_conversation_setup( - setup_messages=recorded_setup, - tools=evolution_tools if evolution_tools else None, - agent_name="SkillEvolver", - extra={ - "evolution_type": ctx.suggestion.evolution_type.value, - "trigger": ctx.trigger.value, - "target_skills": ctx.suggestion.target_skill_ids, - }, - ) - - for iteration in range(_MAX_EVOLUTION_ITERATIONS): - is_last = iteration == _MAX_EVOLUTION_ITERATIONS - 1 - - # Snapshot message count before any additions + LLM call - msg_count_before = len(messages) - - # Final round: disable tools and force a decision - if is_last: - messages.append( - { - "role": "system", - "content": ( - f"This is your FINAL round (iteration " - f"{iteration + 1}/{_MAX_EVOLUTION_ITERATIONS}) — " - f"no more tool calls allowed. " - f"You MUST output the skill edit content now based on " - f"all information gathered so far. Follow the output " - f"format specified in the original instructions. " - f"End with {EVOLUTION_COMPLETE} if the edit is satisfactory, " - f"or {EVOLUTION_FAILED} with a reason if you cannot produce one." - ), - } - ) - - try: - result = await self._llm_client.complete( - messages=messages, - tools=evolution_tools if (evolution_tools and not is_last) else None, - execute_tools=True, - model=model, - ) - except Exception as e: - logger.error(f"Evolution LLM call failed (iter {iteration + 1}): {e}") - return None - - content = result["message"].get("content", "") - updated_messages = result["messages"] - has_tool_calls = result.get("has_tool_calls", False) - - # Record iteration delta (truncated for data minimization) - delta = updated_messages[msg_count_before:] - recorded_delta = [ - {"role": m["role"], "content": _truncate(m.get("content", ""), _RECORDING_MAX_CHARS)} - for m in delta - ] - await RecordingManager.record_iteration_context( - iteration=iteration + 1, - delta_messages=recorded_delta, - response_metadata={ - "has_tool_calls": has_tool_calls, - "tool_calls_count": len(result.get("tool_results", [])), - "has_completion_token": bool( - content and (EVOLUTION_COMPLETE in content or EVOLUTION_FAILED in content) - ), - }, - agent_name="SkillEvolver", - ) - - messages = updated_messages - - # ── Token check (every iteration, regardless of tool calls) ── - if content and (EVOLUTION_COMPLETE in content or EVOLUTION_FAILED in content): - edit_content, failure_reason = self._parse_evolution_output(content) - if failure_reason is not None: - targets = "+".join(ctx.suggestion.target_skill_ids) or "(new)" - logger.warning( - f"Evolution LLM signalled failure " - f"[{ctx.suggestion.evolution_type.value}] " - f"target={targets}: {failure_reason}" - ) - return None - return edit_content - - # No token found - if is_last: - # Final round exhausted without a decision - logger.warning( - f"Evolution agent finished {_MAX_EVOLUTION_ITERATIONS} iterations " - f"without signalling {EVOLUTION_COMPLETE} or {EVOLUTION_FAILED}" - ) - return None - - if has_tool_calls: - logger.debug(f"Evolution agent used tools (iter {iteration + 1}/{_MAX_EVOLUTION_ITERATIONS})") - else: - # No tools, no token — nudge the LLM - logger.debug( - f"Evolution agent produced content without token or tools " - f"(iter {iteration + 1}/{_MAX_EVOLUTION_ITERATIONS})" - ) - - # Iteration guidance - remaining = _MAX_EVOLUTION_ITERATIONS - iteration - 1 - messages.append( - { - "role": "system", - "content": ( - f"Iteration {iteration + 1}/{_MAX_EVOLUTION_ITERATIONS} complete " - f"({remaining} remaining). " - f"If your edit is ready, output it and include {EVOLUTION_COMPLETE} " - f"at the end. " - f"If you cannot complete this evolution, output {EVOLUTION_FAILED} " - f"with a reason. " - f"Otherwise, continue gathering information with tools." - ), - } - ) - - # Should never reach here (is_last handles the final iteration) - return None - - @staticmethod - def _parse_evolution_output(content: str) -> tuple[Optional[str], Optional[str]]: - """Extract edit content or failure reason from LLM output. - - MUST only be called when ``EVOLUTION_COMPLETE`` or - ``EVOLUTION_FAILED`` is present in *content*. - - Returns ``(clean_content, failure_reason)``: - - ``(content, None)`` — ``EVOLUTION_COMPLETE`` found. - - ``(None, reason)`` — ``EVOLUTION_FAILED`` found. - """ - stripped = content.strip() - - # Failure takes priority (if both tokens appear, treat as failure) - if EVOLUTION_FAILED in stripped: - idx = stripped.index(EVOLUTION_FAILED) - reason_part = stripped[idx + len(EVOLUTION_FAILED) :].strip() - if reason_part.lower().startswith("reason:"): - reason_part = reason_part[len("reason:") :].strip() - reason = reason_part[:500] if reason_part else "LLM declined to produce edit (no reason given)" - return None, reason - - if EVOLUTION_COMPLETE in stripped: - clean = stripped.replace(EVOLUTION_COMPLETE, "").strip() - clean = _strip_markdown_fences(clean) - return clean, None - - # Caller guarantees a token is present; defensive fallback - return None, "No completion token found (unexpected)" + _parse_evolution_output = staticmethod(_parse_evolution_output_impl) async def _apply_with_retry( self, @@ -808,140 +273,17 @@ async def _apply_with_retry( prompt: str, cleanup_on_retry: Optional[Path] = None, ) -> Optional[SkillEditResult]: - """Apply an edit with retry on failure. - - If the first attempt fails (patch parse error, path mismatch, etc.), - feeds the error back to the LLM and asks for a corrected version. - - After successful application, runs structural validation. - - Retry conversations are recorded to ``conversations.jsonl`` under - agent_name="SkillEvolver.retry" so failed apply attempts and LLM - corrections are preserved for debugging. - - Args: - apply_fn: Callable that takes content str and returns SkillEditResult. - initial_content: First LLM-generated content to try. - skill_dir: Skill directory for validation. - ctx: Evolution context (for retry LLM calls). - prompt: Original prompt (for retry context). - cleanup_on_retry: Directory to remove before retrying (for derive/create). - """ - from openspace.recording import RecordingManager - - current_content = initial_content - msg_history: List[Dict[str, Any]] = [ - {"role": "user", "content": prompt}, - {"role": "assistant", "content": initial_content}, - ] - - # Track whether we've recorded the retry setup (only on first retry) - retry_setup_recorded = False - - for attempt in range(_MAX_EVOLUTION_ATTEMPTS): - # Clean up previous failed attempt (for derive/create) - if attempt > 0 and cleanup_on_retry and cleanup_on_retry.exists(): - shutil.rmtree(cleanup_on_retry, ignore_errors=True) - - # Apply the edit - edit_result = apply_fn(current_content) - - if edit_result.ok: - # Validate the result - validation_error = _validate_skill_dir(skill_dir) - if validation_error is None: - if attempt > 0: - logger.info(f"Apply-retry succeeded on attempt {attempt + 1}/{_MAX_EVOLUTION_ATTEMPTS}") - return edit_result - else: - # Validation failed — treat as error for retry - error_msg = f"Validation failed: {validation_error}" - logger.warning( - f"Apply succeeded but validation failed " - f"(attempt {attempt + 1}/{_MAX_EVOLUTION_ATTEMPTS}): " - f"{validation_error}" - ) - else: - error_msg = edit_result.error or "Unknown apply error" - logger.warning(f"Apply failed (attempt {attempt + 1}/{_MAX_EVOLUTION_ATTEMPTS}): {error_msg}") - - # Last attempt? Give up. - if attempt >= _MAX_EVOLUTION_ATTEMPTS - 1: - logger.error(f"Apply-retry exhausted after {_MAX_EVOLUTION_ATTEMPTS} attempts. Last error: {error_msg}") - # Clean up any partially created directory - if cleanup_on_retry and cleanup_on_retry.exists(): - shutil.rmtree(cleanup_on_retry, ignore_errors=True) - return None - - # Record retry setup on first retry attempt - if not retry_setup_recorded: - recorded_retry = [ - {"role": m["role"], "content": _truncate(m.get("content", ""), _RECORDING_MAX_CHARS)} - for m in msg_history - ] - await RecordingManager.record_conversation_setup( - setup_messages=recorded_retry, - agent_name="SkillEvolver.retry", - extra={ - "evolution_type": ctx.suggestion.evolution_type.value, - "target_skills": ctx.suggestion.target_skill_ids, - "first_error": error_msg[:300], - }, - ) - retry_setup_recorded = True - - # Feed error back to LLM for retry, including current file - # content so the LLM doesn't hallucinate what's on disk. - current_on_disk = self._format_skill_dir_content(skill_dir) if skill_dir.is_dir() else "" - retry_prompt = f"The previous edit was not successful. This was the error:\n\n{error_msg}\n\n" - if current_on_disk: - retry_prompt += ( - f"Here is the CURRENT content of the skill files on disk " - f"(use this as the ground truth for any SEARCH/REPLACE or " - f"context anchors):\n\n{_truncate(current_on_disk, _SKILL_CONTENT_MAX_CHARS)}\n\n" - ) - retry_prompt += "Please fix the issue and generate the edit again. Follow the same output format as before." - msg_history.append({"role": "user", "content": retry_prompt}) - - # Call LLM for corrected version (no tools — just fix the edit) - model = self._model or self._llm_client.model - try: - result = await self._llm_client.complete( - messages=msg_history, - model=model, - ) - new_content = result["message"].get("content", "") - if not new_content: - logger.warning("Retry LLM returned empty content") - continue - - new_content = _strip_markdown_fences(new_content) - # Strip evolution tokens that the LLM may include in retry responses - new_content = new_content.replace(EVOLUTION_COMPLETE, "").replace(EVOLUTION_FAILED, "").strip() - new_content, _ = _extract_change_summary(new_content) - msg_history.append({"role": "assistant", "content": new_content}) - current_content = new_content - - # Record retry iteration - await RecordingManager.record_iteration_context( - iteration=attempt + 1, - delta_messages=[ - {"role": "user", "content": _truncate(retry_prompt, _RECORDING_MAX_CHARS)}, - {"role": "assistant", "content": _truncate(new_content, _RECORDING_MAX_CHARS)}, - ], - response_metadata={ - "has_tool_calls": False, - "attempt": attempt + 1, - "error": error_msg[:300], - }, - agent_name="SkillEvolver.retry", - ) - - except Exception as e: - logger.error(f"Retry LLM call failed: {e}") - continue + """Apply edit with retry — delegates to ``evolution.loop.apply_with_retry``.""" + return await _apply_with_retry_impl( + self, + apply_fn=apply_fn, + initial_content=initial_content, + skill_dir=skill_dir, + ctx=ctx, + prompt=prompt, + cleanup_on_retry=cleanup_on_retry, + ) - return None def _build_context_from_analysis( self, diff --git a/tests/test_evolution_loop.py b/tests/test_evolution_loop.py new file mode 100644 index 00000000..cd4d0bfb --- /dev/null +++ b/tests/test_evolution_loop.py @@ -0,0 +1,470 @@ +"""Tests for openspace.skill_engine.evolution.loop (Epic 5.5). + +Verifies: + 1. parse_evolution_output — complete/failed token extraction + 2. run_evolution_loop — agent loop flow, recording, termination + 3. apply_with_retry — success, validation failure, retry, cleanup + 4. Constants: _MAX_EVOLUTION_ITERATIONS, _MAX_EVOLUTION_ATTEMPTS + 5. Backward compat: SkillEvolver delegates to loop functions +""" + +from __future__ import annotations + +import asyncio +from pathlib import Path +from typing import Optional +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +# Import module-level functions +from openspace.skill_engine.evolution.loop import ( + EVOLUTION_COMPLETE, + EVOLUTION_FAILED, + _MAX_EVOLUTION_ATTEMPTS, + _MAX_EVOLUTION_ITERATIONS, + apply_with_retry, + parse_evolution_output, + run_evolution_loop, +) + + +# --------------------------------------------------------------------------- +# Fakes +# --------------------------------------------------------------------------- + +class _FakeEvolutionType: + value = "FIX" + + +class _FakeSuggestion: + evolution_type = _FakeEvolutionType() + target_skill_ids = ["skill-a"] + direction = "fix it" + category = None + + +class _FakeTrigger: + value = "analysis" + + +class _FakeCtx: + suggestion = _FakeSuggestion() + trigger = _FakeTrigger() + available_tools = [] + recent_analyses = [] + skill_records = [] + skill_contents = [] + skill_dirs = [] + source_task_id = "task-1" + tool_issue_summary = "" + metric_summary = "" + + +class _FakeEvolver: + _model = "test-model" + _available_tools = [] + + def __init__(self): + self._llm_client = MagicMock() + self._llm_client.model = "fallback-model" + self._llm_client.complete = AsyncMock() + + def _parse_evolution_output(self, content): + return parse_evolution_output(content) + + def _format_skill_dir_content(self, skill_dir): + return "" + + +# --------------------------------------------------------------------------- +# parse_evolution_output +# --------------------------------------------------------------------------- + +class TestParseEvolutionOutput: + def test_complete_token_returns_content(self): + content = f"Here is the fix\n{EVOLUTION_COMPLETE}" + result, failure = parse_evolution_output(content) + assert result is not None + assert failure is None + assert "Here is the fix" in result + + def test_failed_token_returns_reason(self): + content = f"{EVOLUTION_FAILED} reason: cannot fix this skill" + result, failure = parse_evolution_output(content) + assert result is None + assert "cannot fix this skill" in failure + + def test_failed_no_reason(self): + content = f"{EVOLUTION_FAILED}" + result, failure = parse_evolution_output(content) + assert result is None + assert "no reason given" in failure.lower() + + def test_both_tokens_failure_wins(self): + """If both COMPLETE and FAILED appear, treat as failure (conservative).""" + content = f"some content {EVOLUTION_COMPLETE} but also {EVOLUTION_FAILED} oops" + result, failure = parse_evolution_output(content) + assert result is None + assert failure is not None + + def test_no_token_returns_defensive_failure(self): + result, failure = parse_evolution_output("just some text") + assert result is None + assert "No completion token" in failure + + def test_strips_markdown_fences(self): + content = f"```\nfixed content\n```\n{EVOLUTION_COMPLETE}" + result, failure = parse_evolution_output(content) + assert failure is None + assert "```" not in result + assert "fixed content" in result + + def test_reason_truncated_to_500(self): + long_reason = "x" * 1000 + content = f"{EVOLUTION_FAILED} {long_reason}" + _, failure = parse_evolution_output(content) + assert len(failure) <= 500 + + +# --------------------------------------------------------------------------- +# run_evolution_loop +# --------------------------------------------------------------------------- + +class TestRunEvolutionLoop: + @pytest.mark.asyncio + async def test_immediate_complete(self): + """LLM returns COMPLETE on first iteration → returns content.""" + evolver = _FakeEvolver() + evolver._llm_client.complete.return_value = { + "message": {"content": f"fixed\n{EVOLUTION_COMPLETE}"}, + "messages": [{"role": "user", "content": "prompt"}, {"role": "assistant", "content": f"fixed\n{EVOLUTION_COMPLETE}"}], + "has_tool_calls": False, + "tool_results": [], + } + + with patch("openspace.recording.RecordingManager") as mock_rec: + mock_rec.record_conversation_setup = AsyncMock() + mock_rec.record_iteration_context = AsyncMock() + result = await run_evolution_loop(evolver, "prompt", _FakeCtx()) + + assert result is not None + assert "fixed" in result + + @pytest.mark.asyncio + async def test_immediate_failed(self): + """LLM returns FAILED on first iteration → returns None.""" + evolver = _FakeEvolver() + evolver._llm_client.complete.return_value = { + "message": {"content": f"{EVOLUTION_FAILED} cannot do it"}, + "messages": [{"role": "user", "content": "prompt"}, {"role": "assistant", "content": f"{EVOLUTION_FAILED} cannot do it"}], + "has_tool_calls": False, + "tool_results": [], + } + + with patch("openspace.recording.RecordingManager") as mock_rec: + mock_rec.record_conversation_setup = AsyncMock() + mock_rec.record_iteration_context = AsyncMock() + result = await run_evolution_loop(evolver, "prompt", _FakeCtx()) + + assert result is None + + @pytest.mark.asyncio + async def test_llm_exception_returns_none(self): + """LLM call raises → returns None (graceful).""" + evolver = _FakeEvolver() + evolver._llm_client.complete.side_effect = RuntimeError("API down") + + with patch("openspace.recording.RecordingManager") as mock_rec: + mock_rec.record_conversation_setup = AsyncMock() + result = await run_evolution_loop(evolver, "prompt", _FakeCtx()) + + assert result is None + + @pytest.mark.asyncio + async def test_recording_called(self): + """Recording setup and iteration are called.""" + evolver = _FakeEvolver() + evolver._llm_client.complete.return_value = { + "message": {"content": f"ok\n{EVOLUTION_COMPLETE}"}, + "messages": [{"role": "assistant", "content": f"ok\n{EVOLUTION_COMPLETE}"}], + "has_tool_calls": False, + "tool_results": [], + } + + with patch("openspace.recording.RecordingManager") as mock_rec: + mock_rec.record_conversation_setup = AsyncMock() + mock_rec.record_iteration_context = AsyncMock() + await run_evolution_loop(evolver, "prompt", _FakeCtx()) + + mock_rec.record_conversation_setup.assert_awaited_once() + mock_rec.record_iteration_context.assert_awaited_once() + + @pytest.mark.asyncio + async def test_mro_parse_called(self): + """Loop calls evolver._parse_evolution_output (MRO).""" + evolver = _FakeEvolver() + evolver._parse_evolution_output = MagicMock(return_value=("content", None)) + evolver._llm_client.complete.return_value = { + "message": {"content": f"x {EVOLUTION_COMPLETE}"}, + "messages": [{"role": "assistant", "content": f"x {EVOLUTION_COMPLETE}"}], + "has_tool_calls": False, + "tool_results": [], + } + + with patch("openspace.recording.RecordingManager") as mock_rec: + mock_rec.record_conversation_setup = AsyncMock() + mock_rec.record_iteration_context = AsyncMock() + await run_evolution_loop(evolver, "prompt", _FakeCtx()) + + evolver._parse_evolution_output.assert_called_once() + + @pytest.mark.asyncio + async def test_model_fallback(self): + """When evolver._model is None, uses llm_client.model.""" + evolver = _FakeEvolver() + evolver._model = None + evolver._llm_client.complete.return_value = { + "message": {"content": f"ok\n{EVOLUTION_COMPLETE}"}, + "messages": [{"role": "assistant", "content": f"ok\n{EVOLUTION_COMPLETE}"}], + "has_tool_calls": False, + "tool_results": [], + } + + with patch("openspace.recording.RecordingManager") as mock_rec: + mock_rec.record_conversation_setup = AsyncMock() + mock_rec.record_iteration_context = AsyncMock() + await run_evolution_loop(evolver, "prompt", _FakeCtx()) + + call_kwargs = evolver._llm_client.complete.call_args[1] + assert call_kwargs["model"] == "fallback-model" + + +# --------------------------------------------------------------------------- +# apply_with_retry +# --------------------------------------------------------------------------- + +class _FakeEditResult: + def __init__(self, ok=True, error=None, content_snapshot=None, content_diff=None): + self.ok = ok + self.error = error + self.content_snapshot = content_snapshot or {} + self.content_diff = content_diff or "" + + +class TestApplyWithRetry: + @pytest.mark.asyncio + async def test_success_first_attempt(self): + """Apply succeeds + validation passes → returns result.""" + evolver = _FakeEvolver() + apply_fn = MagicMock(return_value=_FakeEditResult(ok=True)) + ctx = _FakeCtx() + + with patch("openspace.skill_engine.evolution.loop._validate_skill_dir", return_value=None), \ + patch("openspace.recording.RecordingManager"): + result = await apply_with_retry( + evolver, apply_fn=apply_fn, initial_content="content", + skill_dir=Path("/fake"), ctx=ctx, prompt="prompt", + ) + + assert result is not None + assert result.ok + apply_fn.assert_called_once_with("content") + + @pytest.mark.asyncio + async def test_apply_fails_then_retry_succeeds(self): + """First apply fails, retry LLM produces fixed content → success.""" + evolver = _FakeEvolver() + apply_fn = MagicMock(side_effect=[ + _FakeEditResult(ok=False, error="parse error"), + _FakeEditResult(ok=True), + ]) + evolver._llm_client.complete.return_value = { + "message": {"content": "fixed content"}, + "messages": [], + } + ctx = _FakeCtx() + + with patch("openspace.skill_engine.evolution.loop._validate_skill_dir", return_value=None), \ + patch("openspace.recording.RecordingManager") as mock_rec: + mock_rec.record_conversation_setup = AsyncMock() + mock_rec.record_iteration_context = AsyncMock() + result = await apply_with_retry( + evolver, apply_fn=apply_fn, initial_content="bad content", + skill_dir=Path("/fake"), ctx=ctx, prompt="prompt", + ) + + assert result is not None + assert result.ok + assert apply_fn.call_count == 2 + + @pytest.mark.asyncio + async def test_all_attempts_fail_returns_none(self): + """All attempts fail → returns None.""" + evolver = _FakeEvolver() + apply_fn = MagicMock(return_value=_FakeEditResult(ok=False, error="bad")) + evolver._llm_client.complete.return_value = { + "message": {"content": "still bad"}, + "messages": [], + } + ctx = _FakeCtx() + + with patch("openspace.skill_engine.evolution.loop._validate_skill_dir"), \ + patch("openspace.recording.RecordingManager") as mock_rec: + mock_rec.record_conversation_setup = AsyncMock() + mock_rec.record_iteration_context = AsyncMock() + result = await apply_with_retry( + evolver, apply_fn=apply_fn, initial_content="bad", + skill_dir=Path("/fake"), ctx=ctx, prompt="prompt", + ) + + assert result is None + assert apply_fn.call_count == _MAX_EVOLUTION_ATTEMPTS + + @pytest.mark.asyncio + async def test_validation_failure_triggers_retry(self): + """Apply OK but validation fails → treated as error, retries.""" + evolver = _FakeEvolver() + apply_fn = MagicMock(return_value=_FakeEditResult(ok=True)) + evolver._llm_client.complete.return_value = { + "message": {"content": "fixed"}, + "messages": [], + } + ctx = _FakeCtx() + + validate_results = iter(["missing frontmatter", None]) + with patch("openspace.skill_engine.evolution.loop._validate_skill_dir", side_effect=validate_results), \ + patch("openspace.recording.RecordingManager") as mock_rec: + mock_rec.record_conversation_setup = AsyncMock() + mock_rec.record_iteration_context = AsyncMock() + result = await apply_with_retry( + evolver, apply_fn=apply_fn, initial_content="incomplete", + skill_dir=Path("/fake"), ctx=ctx, prompt="prompt", + ) + + assert result is not None + assert apply_fn.call_count == 2 + + @pytest.mark.asyncio + async def test_cleanup_on_retry(self, tmp_path): + """cleanup_on_retry dir is removed before each retry.""" + target = tmp_path / "new-skill" + target.mkdir() + (target / "SKILL.md").write_text("old") + + evolver = _FakeEvolver() + apply_fn = MagicMock(side_effect=[ + _FakeEditResult(ok=False, error="bad"), + _FakeEditResult(ok=True), + ]) + evolver._llm_client.complete.return_value = { + "message": {"content": "fixed"}, + "messages": [], + } + ctx = _FakeCtx() + + with patch("openspace.skill_engine.evolution.loop._validate_skill_dir", return_value=None), \ + patch("openspace.recording.RecordingManager") as mock_rec: + mock_rec.record_conversation_setup = AsyncMock() + mock_rec.record_iteration_context = AsyncMock() + result = await apply_with_retry( + evolver, apply_fn=apply_fn, initial_content="bad", + skill_dir=target, ctx=ctx, prompt="prompt", + cleanup_on_retry=target, + ) + + assert result is not None + # The dir was cleaned up before the retry attempt + + @pytest.mark.asyncio + async def test_mro_format_skill_dir_called(self): + """Retry path calls evolver._format_skill_dir_content (MRO).""" + evolver = _FakeEvolver() + evolver._format_skill_dir_content = MagicMock(return_value="on disk content") + apply_fn = MagicMock(return_value=_FakeEditResult(ok=False, error="bad")) + evolver._llm_client.complete.return_value = { + "message": {"content": "fixed"}, + "messages": [], + } + ctx = _FakeCtx() + fake_dir = MagicMock() + fake_dir.is_dir.return_value = True + fake_dir.exists.return_value = False + + with patch("openspace.skill_engine.evolution.loop._validate_skill_dir"), \ + patch("openspace.recording.RecordingManager") as mock_rec: + mock_rec.record_conversation_setup = AsyncMock() + mock_rec.record_iteration_context = AsyncMock() + await apply_with_retry( + evolver, apply_fn=apply_fn, initial_content="bad", + skill_dir=fake_dir, ctx=ctx, prompt="prompt", + ) + + evolver._format_skill_dir_content.assert_called() + + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +class TestConstants: + def test_max_iterations(self): + assert _MAX_EVOLUTION_ITERATIONS == 5 + + def test_max_attempts(self): + assert _MAX_EVOLUTION_ATTEMPTS == 3 + + def test_logger_uses_evolver_namespace(self): + from openspace.skill_engine.evolution.loop import logger + assert "evolver" in logger.name + + +# --------------------------------------------------------------------------- +# Backward compat: SkillEvolver delegates +# --------------------------------------------------------------------------- + +class TestDelegationSeam: + def test_parse_evolution_output_is_staticmethod(self): + from openspace.skill_engine.evolver import SkillEvolver + assert isinstance( + SkillEvolver.__dict__["_parse_evolution_output"], + staticmethod, + ) + + def test_evolve_fix_method_exists(self): + from openspace.skill_engine.evolver import SkillEvolver + assert hasattr(SkillEvolver, "_evolve_fix") + assert asyncio.iscoroutinefunction(SkillEvolver._evolve_fix) + + def test_evolve_derived_method_exists(self): + from openspace.skill_engine.evolver import SkillEvolver + assert hasattr(SkillEvolver, "_evolve_derived") + assert asyncio.iscoroutinefunction(SkillEvolver._evolve_derived) + + def test_evolve_captured_method_exists(self): + from openspace.skill_engine.evolver import SkillEvolver + assert hasattr(SkillEvolver, "_evolve_captured") + assert asyncio.iscoroutinefunction(SkillEvolver._evolve_captured) + + def test_run_evolution_loop_method_exists(self): + from openspace.skill_engine.evolver import SkillEvolver + assert hasattr(SkillEvolver, "_run_evolution_loop") + assert asyncio.iscoroutinefunction(SkillEvolver._run_evolution_loop) + + def test_apply_with_retry_method_exists(self): + from openspace.skill_engine.evolver import SkillEvolver + assert hasattr(SkillEvolver, "_apply_with_retry") + assert asyncio.iscoroutinefunction(SkillEvolver._apply_with_retry) + + +# --------------------------------------------------------------------------- +# Size guard +# --------------------------------------------------------------------------- + +class TestSizeGuard: + def test_loop_module_size(self): + """loop.py should stay under 400 lines.""" + import openspace.skill_engine.evolution.loop as mod + src = Path(mod.__file__) + lines = src.read_text(encoding="utf-8").splitlines() + assert len(lines) <= 400, f"loop.py has {len(lines)} lines (limit 400)" diff --git a/tests/test_evolution_strategies.py b/tests/test_evolution_strategies.py new file mode 100644 index 00000000..567c5649 --- /dev/null +++ b/tests/test_evolution_strategies.py @@ -0,0 +1,317 @@ +"""Tests for openspace.skill_engine.evolution.strategies (Epic 5.5). + +Verifies: + 1. evolve_fix — builds prompt, calls loop, applies, persists, registers + 2. evolve_derived — single-parent + multi-parent merge paths + 3. evolve_captured — creates new skill from scratch + 4. Guard clauses: missing skill_records/contents/dirs → None + 5. MRO: all internal calls go through evolver._method() + 6. Backward compat: SkillEvolver delegates to strategy functions +""" + +from __future__ import annotations + +import asyncio +from pathlib import Path +from unittest.mock import AsyncMock, MagicMock, PropertyMock, patch + +import pytest + +from openspace.skill_engine.evolution.strategies import ( + evolve_captured, + evolve_derived, + evolve_fix, +) + + +# --------------------------------------------------------------------------- +# Fakes +# --------------------------------------------------------------------------- + +class _FakeEvolutionType: + value = "FIX" + + +class _FakeSuggestion: + evolution_type = _FakeEvolutionType() + target_skill_ids = ["skill-a"] + direction = "improve error handling" + category = None + + +class _FakeTrigger: + value = "analysis" + + +class _FakeLineage: + generation = 1 + origin = "FIXED" + parent_skill_ids = [] + + +class _FakeRecord: + skill_id = "skill-a__v1_abc" + name = "my-skill" + description = "A skill" + path = "/skills/my-skill/SKILL.md" + category = "WORKFLOW" + tags = ["test"] + visibility = "private" + creator_id = "user-1" + lineage = _FakeLineage() + tool_dependencies = ["tool-a"] + critical_tools = ["tool-a"] + + +class _FakeEditResult: + def __init__(self, ok=True): + self.ok = ok + self.error = None + self.content_snapshot = {"SKILL.md": "---\nname: my-skill\ndescription: A skill\n---\ncontent"} + self.content_diff = "diff here" + + +class _FakeCtx: + suggestion = _FakeSuggestion() + trigger = _FakeTrigger() + available_tools = [] + recent_analyses = [] + skill_records = [_FakeRecord()] + skill_contents = ["original content"] + skill_dirs = [Path("/skills/my-skill")] + source_task_id = "task-1" + tool_issue_summary = "" + metric_summary = "" + + +class _FakeEvolver: + _model = "test-model" + _available_tools = [] + + def __init__(self): + self._llm_client = MagicMock() + self._llm_client.model = "fallback-model" + self._store = MagicMock() + self._store.evolve_skill = AsyncMock() + self._store.save_record = AsyncMock() + self._registry = MagicMock() + self._registry._skill_dirs = [Path("/skills")] + self._registry.update_skill = MagicMock() + self._registry.add_skill = MagicMock() + + def _format_skill_dir_content(self, skill_dir): + return "" + + def _format_analysis_context(self, analyses): + return "(no context)" + + async def _run_evolution_loop(self, prompt, ctx): + return "---\nname: my-skill\n---\nnew content" + + async def _apply_with_retry(self, **kwargs): + return _FakeEditResult(ok=True) + + +# --------------------------------------------------------------------------- +# evolve_fix +# --------------------------------------------------------------------------- + +class TestEvolveFix: + @pytest.mark.asyncio + async def test_success(self): + """Happy path: FIX produces a new record with correct fields.""" + evolver = _FakeEvolver() + ctx = _FakeCtx() + + with patch("openspace.skill_engine.evolution.strategies.write_skill_id"), \ + patch("openspace.skill_engine.registry.SkillMeta"): + result = await evolve_fix(evolver, ctx) + + assert result is not None + assert result.name == "my-skill" + assert result.lineage.origin.value == "fixed" + assert result.lineage.generation == 2 # parent gen 1 + 1 + assert result.lineage.parent_skill_ids == ["skill-a__v1_abc"] + assert result.lineage.source_task_id == "task-1" + assert "tool-a" in result.tool_dependencies + evolver._store.evolve_skill.assert_awaited_once() + evolver._registry.update_skill.assert_called_once() + + @pytest.mark.asyncio + async def test_no_records_returns_none(self): + """Missing skill_records → None.""" + evolver = _FakeEvolver() + ctx = _FakeCtx() + ctx.skill_records = [] + + result = await evolve_fix(evolver, ctx) + assert result is None + + @pytest.mark.asyncio + async def test_loop_returns_none(self): + """Agent loop fails → None.""" + evolver = _FakeEvolver() + evolver._run_evolution_loop = AsyncMock(return_value=None) + ctx = _FakeCtx() + + result = await evolve_fix(evolver, ctx) + assert result is None + + @pytest.mark.asyncio + async def test_apply_fails_returns_none(self): + """Apply-retry fails → None.""" + evolver = _FakeEvolver() + evolver._apply_with_retry = AsyncMock(return_value=None) + ctx = _FakeCtx() + + result = await evolve_fix(evolver, ctx) + assert result is None + + @pytest.mark.asyncio + async def test_mro_format_called(self): + """Calls go through evolver._format_* (MRO).""" + evolver = _FakeEvolver() + evolver._format_skill_dir_content = MagicMock(return_value="dir content") + evolver._format_analysis_context = MagicMock(return_value="(context)") + ctx = _FakeCtx() + + with patch("openspace.skill_engine.evolution.strategies.write_skill_id"), \ + patch("openspace.skill_engine.registry.SkillMeta"): + await evolve_fix(evolver, ctx) + + evolver._format_skill_dir_content.assert_called_once() + evolver._format_analysis_context.assert_called_once() + + +# --------------------------------------------------------------------------- +# evolve_derived +# --------------------------------------------------------------------------- + +class TestEvolveDerived: + @pytest.mark.asyncio + async def test_single_parent_success(self): + """Single parent → enhanced skill.""" + evolver = _FakeEvolver() + ctx = _FakeCtx() + + with patch("openspace.skill_engine.evolution.strategies.write_skill_id"), \ + patch("openspace.skill_engine.registry.SkillMeta"): + result = await evolve_derived(evolver, ctx) + + assert result is not None + evolver._store.evolve_skill.assert_awaited_once() + evolver._registry.add_skill.assert_called_once() + + @pytest.mark.asyncio + async def test_no_records_returns_none(self): + """Missing skill_records → None.""" + evolver = _FakeEvolver() + ctx = _FakeCtx() + ctx.skill_records = [] + + result = await evolve_derived(evolver, ctx) + assert result is None + + @pytest.mark.asyncio + async def test_loop_returns_none(self): + """Agent loop fails → None.""" + evolver = _FakeEvolver() + evolver._run_evolution_loop = AsyncMock(return_value=None) + ctx = _FakeCtx() + + result = await evolve_derived(evolver, ctx) + assert result is None + + +# --------------------------------------------------------------------------- +# evolve_captured +# --------------------------------------------------------------------------- + +class TestEvolveCaptured: + @pytest.mark.asyncio + async def test_success(self): + """Happy path: CAPTURED produces a new record.""" + evolver = _FakeEvolver() + ctx = _FakeCtx() + ctx.skill_records = [] + ctx.skill_contents = [] + ctx.skill_dirs = [] + + with patch("openspace.skill_engine.evolution.strategies.write_skill_id"), \ + patch("openspace.skill_engine.registry.SkillMeta"): + result = await evolve_captured(evolver, ctx) + + assert result is not None + evolver._store.save_record.assert_awaited_once() + evolver._registry.add_skill.assert_called_once() + + @pytest.mark.asyncio + async def test_no_name_returns_none(self): + """LLM doesn't produce a name → None.""" + evolver = _FakeEvolver() + evolver._run_evolution_loop = AsyncMock(return_value="no frontmatter here") + ctx = _FakeCtx() + ctx.skill_records = [] + ctx.skill_contents = [] + ctx.skill_dirs = [] + + result = await evolve_captured(evolver, ctx) + assert result is None + + @pytest.mark.asyncio + async def test_no_skill_dirs_returns_none(self): + """No skill directories configured → None.""" + evolver = _FakeEvolver() + evolver._registry._skill_dirs = [] + ctx = _FakeCtx() + ctx.skill_records = [] + ctx.skill_contents = [] + ctx.skill_dirs = [] + + result = await evolve_captured(evolver, ctx) + assert result is None + + @pytest.mark.asyncio + async def test_apply_fails_returns_none(self): + """Apply-retry fails → None.""" + evolver = _FakeEvolver() + evolver._apply_with_retry = AsyncMock(return_value=None) + ctx = _FakeCtx() + ctx.skill_records = [] + ctx.skill_contents = [] + ctx.skill_dirs = [] + + result = await evolve_captured(evolver, ctx) + assert result is None + + +# --------------------------------------------------------------------------- +# Backward compat +# --------------------------------------------------------------------------- + +class TestDelegationSeam: + def test_evolve_fix_delegate(self): + """SkillEvolver._evolve_fix exists and is async.""" + from openspace.skill_engine.evolver import SkillEvolver + assert asyncio.iscoroutinefunction(SkillEvolver._evolve_fix) + + def test_evolve_derived_delegate(self): + from openspace.skill_engine.evolver import SkillEvolver + assert asyncio.iscoroutinefunction(SkillEvolver._evolve_derived) + + def test_evolve_captured_delegate(self): + from openspace.skill_engine.evolver import SkillEvolver + assert asyncio.iscoroutinefunction(SkillEvolver._evolve_captured) + + +# --------------------------------------------------------------------------- +# Size guard +# --------------------------------------------------------------------------- + +class TestSizeGuard: + def test_strategies_module_size(self): + """strategies.py should stay under 400 lines.""" + import openspace.skill_engine.evolution.strategies as mod + src = Path(mod.__file__) + lines = src.read_text(encoding="utf-8").splitlines() + assert len(lines) <= 400, f"strategies.py has {len(lines)} lines (limit 400)"