fix: detect APM-installed skills #400

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>

Author: Copilot

README.md

@@ -230,6 +230,18 @@ project/
     └── fixtures/
 ```
 
+*APM-managed skills* are detected from their compiled output without symlinks:
+```
+project/
+├── skills/{skill-name}/apm.yml
+├── skills/{skill-name}/.apm/skills/{skill-name}/SKILL.md
+└── skills/{skill-name}/eval.yaml
+```
+
+When both `skills/{skill-name}/SKILL.md` and the APM compiled
+`.apm/skills/{skill-name}/SKILL.md` exist for the same skill, the top-level
+`SKILL.md` takes precedence.
+
 *Standalone mode* (no `skills/` detected):
 ```
 {skill-name}/
@@ -729,8 +741,9 @@ With two refs, compares the first ref to the second.
 | `--threshold <n>` | Fail when any existing file increases by more than n percent (0 = disabled) |
 
 Use `--skills` to restrict comparison to SKILL.md files under configured skill
-roots (`skills/`, `.github/skills/`, and `paths.skills` from `.waza.yaml`). In
-skills mode the default base ref is `origin/main` (falling back to `main`).
+roots (`skills/`, `.github/skills/`, APM `.apm/skills/` outputs, and
+`paths.skills` from `.waza.yaml`). In skills mode the default base ref is
+`origin/main` (falling back to `main`).
 
 Use `--threshold` for CI gating — newly added files are exempt from threshold
 checks (no baseline) but still subject to absolute limit checks with `--strict`.

cmd/waza/cmd_run.go

@@ -446,6 +446,7 @@ func runCommandE(cmd *cobra.Command, args []string) error {
 type skillSpecPath struct {
 	evalSpecPath string
 	skillName    string
+	skillDir     string
 }
 
 type skillRunResult struct {
@@ -474,7 +475,11 @@ func resolveSpecPaths(args []string) ([]skillSpecPath, error) {
 		if err != nil {
 			return nil, err
 		}
-		return []skillSpecPath{{evalSpecPath: evalPath, skillName: wsCtx.Skills[0].Name}}, nil
+		return []skillSpecPath{{
+			evalSpecPath: evalPath,
+			skillName:    wsCtx.Skills[0].Name,
+			skillDir:     wsCtx.Skills[0].Dir,
+		}}, nil
 	}
 
 	// No args — workspace detection
@@ -493,7 +498,7 @@ func resolveSpecPaths(args []string) ([]skillSpecPath, error) {
 			fmt.Printf("⚠️  Skipping %s: %v\n", si.Name, err)
 			continue
 		}
-		paths = append(paths, skillSpecPath{evalSpecPath: evalPath, skillName: si.Name})
+		paths = append(paths, skillSpecPath{evalSpecPath: evalPath, skillName: si.Name, skillDir: si.Dir})
 	}
 
 	if len(paths) == 0 {
@@ -553,6 +558,9 @@ func printSkillRunSummary(results []skillRunResult) {
 // defaultSkills - skills found under the workspace folder, specified by .waza.yaml
 func runCommandForSpec(cmd *cobra.Command, sp skillSpecPath, defaultSkills []string) ([]modelResult, error) {
 	specPath := sp.evalSpecPath
+	if sp.skillDir != "" && !slices.Contains(defaultSkills, sp.skillDir) {
+		defaultSkills = append(defaultSkills, sp.skillDir)
+	}
 
 	// Load spec
 	spec, err := models.LoadEvalSpec(specPath)

cmd/waza/cmd_run_workspace_test.go

@@ -86,6 +86,46 @@ tasks:
 	assert.NoError(t, err, "should find eval.yaml for the named skill")
 }
 
+func TestRunCommand_WorkspaceDetection_APMInstalledSkill(t *testing.T) {
+	resetRunGlobals()
+
+	dir := t.TempDir()
+	sourceDir := filepath.Join(dir, "skills", "my-skill")
+	compiledDir := filepath.Join(sourceDir, ".apm", "skills", "my-skill")
+	require.NoError(t, os.MkdirAll(compiledDir, 0o755))
+
+	skillContent := "---\nname: my-skill\ndescription: \"test\"\n---\n# Body\n"
+	require.NoError(t, os.WriteFile(filepath.Join(compiledDir, "SKILL.md"), []byte(skillContent), 0o644))
+
+	taskDir := filepath.Join(sourceDir, "tasks")
+	require.NoError(t, os.MkdirAll(taskDir, 0o755))
+	task := "id: t1\nname: Task One\ninputs:\n  prompt: \"test\"\n"
+	require.NoError(t, os.WriteFile(filepath.Join(taskDir, "task.yaml"), []byte(task), 0o644))
+
+	spec := `name: apm-eval
+skill: my-skill
+version: "1.0"
+config:
+  trials_per_task: 1
+  timeout_seconds: 30
+  executor: mock
+  model: test-model
+tasks:
+  - "tasks/*.yaml"
+`
+	require.NoError(t, os.WriteFile(filepath.Join(sourceDir, "eval.yaml"), []byte(spec), 0o644))
+
+	t.Chdir(dir)
+
+	cmd := newRunCommand()
+	cmd.SetArgs(nil) // no args — workspace detection
+	cmd.SetOut(io.Discard)
+	cmd.SetErr(io.Discard)
+
+	err := cmd.Execute()
+	assert.NoError(t, err, "workspace detection should find APM-installed skill eval.yaml")
+}
+
 func TestRunCommand_ExplicitPath_BackwardCompat(t *testing.T) {
 	resetRunGlobals()
 

internal/workspace/workspace.go

@@ -77,6 +77,7 @@ type SkillInfo struct {
 	Dir       string // absolute path to the skill directory (containing SKILL.md)
 	SkillPath string // absolute path to SKILL.md
 	EvalPath  string // absolute path to the eval file (empty if not found)
+	SourceDir string // absolute path to the source skill directory when SKILL.md is compiled elsewhere
 }
 
 // WorkspaceContext represents the detected workspace.
@@ -92,8 +93,9 @@ type WorkspaceContext struct {
 // It checks:
 // 1. CWD for SKILL.md → single-skill
 // 2. Walk up parents for SKILL.md → single-skill (nested inside skill dir)
-// 3. Check for skills/ directory with SKILL.md descendants → multi-skill
-// 4. Scan CWD for child dirs containing SKILL.md → multi-skill
+// 3. Check known compiled skill output directories such as .apm/skills/
+// 4. Check for skills/ directory with SKILL.md descendants → multi-skill
+// 5. Scan CWD for child dirs containing SKILL.md → multi-skill
 func DetectContext(dir string, opts ...DetectOption) (*WorkspaceContext, error) {
 	o := defaultDetectOptions()
 	for _, fn := range opts {
@@ -116,7 +118,7 @@ func DetectContext(dir string, opts ...DetectOption) (*WorkspaceContext, error)
 		}, nil
 	}
 
-	// 2. Walk up parent directories looking for SKILL.md
+	// 2. Walk up parent directories looking for SKILL.md or known compiled outputs
 	current := absDir
 	for i := 0; i < maxParentWalk; i++ {
 		parent := filepath.Dir(current)
@@ -134,6 +136,11 @@ func DetectContext(dir string, opts ...DetectOption) (*WorkspaceContext, error)
 				EvalFile: o.evalFile,
 			}, nil
 		}
+		if skills, err := scanDirectAPMSkills(current); err != nil {
+			return nil, fmt.Errorf("scanning APM skills directory %s: %w", filepath.Join(current, ".apm", "skills"), err)
+		} else if len(skills) > 0 {
+			return contextFromSkills(current, skills, o), nil
+		}
 	}
 
 	// 3. Check for configured skills subdirectory with SKILL.md children
@@ -165,6 +172,20 @@ func DetectContext(dir string, opts ...DetectOption) (*WorkspaceContext, error)
 		skills = mergeSkillsByName(skills, githubSkills)
 	}
 
+	if isDir(skillsDir) {
+		apmSkills, err := scanForAPMSkillsUnder(skillsDir)
+		if err != nil {
+			return nil, fmt.Errorf("scanning APM skills under %s: %w", skillsDir, err)
+		}
+		skills = mergeSkillsByName(skills, apmSkills)
+	}
+
+	rootAPMSkills, err := scanDirectAPMSkills(absDir)
+	if err != nil {
+		return nil, fmt.Errorf("scanning APM skills directory %s: %w", filepath.Join(absDir, ".apm", "skills"), err)
+	}
+	skills = mergeSkillsByName(skills, rootAPMSkills)
+
 	if len(skills) > 0 {
 		return &WorkspaceContext{
 			Type:     ContextMultiSkill,
@@ -212,8 +233,9 @@ func FindSkill(ctx *WorkspaceContext, name string) (*SkillInfo, error) {
 
 // FindEval finds an eval file for a skill using priority order:
 // 1. {root}/evals/{skill-name}/{eval-file}  (separated convention)
-// 2. {skill-dir}/evals/{eval-file}          (nested subdir)
-// 3. {skill-dir}/{eval-file}                (co-located/legacy)
+// 2. {source-or-skill-dir}/evals/{eval-file} (nested subdir)
+// 3. {source-or-skill-dir}/{eval-file}       (co-located/legacy)
+// 4. {skill-dir}/evals/{eval-file} and {skill-dir}/{eval-file} for compiled skills
 // Returns empty string if none found (not an error).
 func FindEval(wsCtx *WorkspaceContext, skillName string) (string, error) {
 	si, err := FindSkill(wsCtx, skillName)
@@ -246,23 +268,45 @@ func FindEval(wsCtx *WorkspaceContext, skillName string) (string, error) {
 
 	// Priority 2: nested subdir inside skill directory
 	for _, evalFile := range evalFiles {
-		nested := filepath.Join(si.Dir, "evals", evalFile)
+		nested := filepath.Join(evalLookupDir(si), "evals", evalFile)
 		if isFile(nested) {
 			return nested, nil
 		}
 	}
 
 	// Priority 3: co-located / legacy
 	for _, evalFile := range evalFiles {
-		colocated := filepath.Join(si.Dir, evalFile)
+		colocated := filepath.Join(evalLookupDir(si), evalFile)
 		if isFile(colocated) {
 			return colocated, nil
 		}
 	}
 
+	if si.SourceDir != "" && !samePath(si.SourceDir, si.Dir) {
+		for _, evalFile := range evalFiles {
+			nested := filepath.Join(si.Dir, "evals", evalFile)
+			if isFile(nested) {
+				return nested, nil
+			}
+		}
+		for _, evalFile := range evalFiles {
+			colocated := filepath.Join(si.Dir, evalFile)
+			if isFile(colocated) {
+				return colocated, nil
+			}
+		}
+	}
+
 	return "", nil
 }
 
+func evalLookupDir(si *SkillInfo) string {
+	if si.SourceDir != "" {
+		return si.SourceDir
+	}
+	return si.Dir
+}
+
 func evalFilenames(configured string) []string {
 	if configured == "" {
 		configured = projectconfig.DefaultEvalFile
@@ -381,6 +425,53 @@ func scanForSkillsRecursive(parentDir string) ([]SkillInfo, error) {
 	return skills, nil
 }
 
+func scanDirectAPMSkills(dir string) ([]SkillInfo, error) {
+	apmSkillsDir := filepath.Join(dir, ".apm", "skills")
+	if !isDir(apmSkillsDir) {
+		return nil, nil
+	}
+	skills, err := scanForSkills(apmSkillsDir, true)
+	if err != nil {
+		return nil, err
+	}
+	for i := range skills {
+		skills[i].SourceDir = dir
+	}
+	return skills, nil
+}
+
+func scanForAPMSkillsUnder(parentDir string) ([]SkillInfo, error) {
+	var skills []SkillInfo
+	err := filepath.WalkDir(parentDir, func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return fmt.Errorf("error walking %s: %w", path, err)
+		}
+		if !d.IsDir() {
+			return nil
+		}
+		if path == parentDir {
+			return nil
+		}
+		name := d.Name()
+		if name == ".apm" {
+			apmSkills, scanErr := scanDirectAPMSkills(filepath.Dir(path))
+			if scanErr != nil {
+				return scanErr
+			}
+			skills = append(skills, apmSkills...)
+			return fs.SkipDir
+		}
+		if shouldSkipSkillScanDir(name) {
+			return fs.SkipDir
+		}
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	return skills, nil
+}
+
 func shouldSkipSkillScanDir(name string) bool {
 	return strings.HasPrefix(name, ".") || name == "node_modules" || name == "vendor"
 }
@@ -438,6 +529,20 @@ func mergeSkillsByName(base, additional []SkillInfo) []SkillInfo {
 	})
 }
 
+func contextFromSkills(root string, skills []SkillInfo, o detectOptions) *WorkspaceContext {
+	ctxType := ContextMultiSkill
+	if len(skills) == 1 {
+		ctxType = ContextSingleSkill
+	}
+	return &WorkspaceContext{
+		Type:     ctxType,
+		Root:     root,
+		Skills:   skills,
+		EvalsDir: o.evalsDir,
+		EvalFile: o.evalFile,
+	}
+}
+
 // LooksLikePath returns true if the string appears to be a file path
 // rather than a skill name. Exported so that CLI packages (cmd/waza,
 // cmd/waza/dev) can share the same heuristic without duplication.