From 703320611a8947143a411c3c37a820451e3c263a Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 3 Mar 2026 15:37:39 +0000 Subject: [PATCH 1/3] Initial plan From 3c5c1e87402b8bda4987730fdc1edb15ff626b55 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 3 Mar 2026 15:43:56 +0000 Subject: [PATCH 2/3] fix: --discover finds eval.yaml in project-layout evals/{name}/ directory Co-authored-by: spboyer <7681382+spboyer@users.noreply.github.com> --- internal/discovery/discovery.go | 14 ++++++++++---- internal/discovery/discovery_test.go | 29 ++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/internal/discovery/discovery.go b/internal/discovery/discovery.go index 4530eaa9..50b6acb6 100644 --- a/internal/discovery/discovery.go +++ b/internal/discovery/discovery.go @@ -22,7 +22,9 @@ func (d DiscoveredSkill) HasEval() bool { // Discover walks the given root directory and finds all skills with eval configs. // A skill is a directory containing SKILL.md. An eval config is eval.yaml either -// in the same directory, in an evals/ subdirectory, or in a tests/ subdirectory. +// in the same directory, in an evals/ subdirectory, in a tests/ subdirectory, or +// in a project-layout evals/{name}/ directory two levels above the skill directory +// (e.g. project-root/skills/{name}/SKILL.md → project-root/evals/{name}/eval.yaml). func Discover(root string) ([]DiscoveredSkill, error) { absRoot, err := filepath.Abs(root) if err != nil { @@ -60,7 +62,7 @@ func Discover(root string) ([]DiscoveredSkill, error) { if !info.IsDir() && info.Name() == "SKILL.md" { dir := filepath.Dir(path) name := filepath.Base(dir) - evalPath := findEvalConfig(dir) + evalPath := findEvalConfig(dir, name) skills = append(skills, DiscoveredSkill{ Name: name, @@ -80,12 +82,16 @@ func Discover(root string) ([]DiscoveredSkill, error) { } // findEvalConfig looks for eval.yaml in standard locations relative to a skill directory. -// Priority: tests/eval.yaml > evals/eval.yaml > eval.yaml -func findEvalConfig(skillDir string) string { +// Priority: tests/eval.yaml > evals/eval.yaml > eval.yaml > ../../evals/{name}/eval.yaml +// The last candidate handles the project layout produced by `waza new` in project mode, +// where SKILL.md lives at skills/{name}/SKILL.md and eval.yaml at evals/{name}/eval.yaml. +func findEvalConfig(skillDir, name string) string { candidates := []string{ filepath.Join(skillDir, "tests", "eval.yaml"), filepath.Join(skillDir, "evals", "eval.yaml"), filepath.Join(skillDir, "eval.yaml"), + // Project layout: project-root/skills/{name}/SKILL.md → project-root/evals/{name}/eval.yaml + filepath.Join(filepath.Dir(filepath.Dir(skillDir)), "evals", name, "eval.yaml"), } for _, c := range candidates { diff --git a/internal/discovery/discovery_test.go b/internal/discovery/discovery_test.go index deea6662..2fd95ec4 100644 --- a/internal/discovery/discovery_test.go +++ b/internal/discovery/discovery_test.go @@ -193,6 +193,35 @@ func TestDiscoverEvalsSubdir(t *testing.T) { } } +func TestDiscoverProjectLayout(t *testing.T) { + // Simulates the layout produced by `waza new` in project mode: + // project-root/skills/{name}/SKILL.md + // project-root/evals/{name}/eval.yaml + root := t.TempDir() + + setupSkillDir(t, filepath.Join(root, "skills", "my-skill")) + setupEvalFile(t, filepath.Join(root, "evals", "my-skill", "eval.yaml")) + + skills, err := Discover(root) + if err != nil { + t.Fatal(err) + } + + if len(skills) != 1 { + t.Fatalf("expected 1 skill, got %d", len(skills)) + } + if skills[0].Name != "my-skill" { + t.Errorf("expected my-skill, got %s", skills[0].Name) + } + if !skills[0].HasEval() { + t.Error("my-skill should have eval (project layout evals/{name}/eval.yaml)") + } + wantEvalPath := filepath.Join(root, "evals", "my-skill", "eval.yaml") + if skills[0].EvalPath != wantEvalPath { + t.Errorf("expected EvalPath %s, got %s", wantEvalPath, skills[0].EvalPath) + } +} + func TestDiscoverNonexistentRoot(t *testing.T) { _, err := Discover("/nonexistent/path/that/does/not/exist") if err == nil { From c58097c4feafd8d5fdffd935c5e64f26884d8b6b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 3 Mar 2026 18:46:51 +0000 Subject: [PATCH 3/3] fix: avoid exact path comparison in TestDiscoverProjectLayout (Windows symlink short paths) Co-authored-by: spboyer <7681382+spboyer@users.noreply.github.com> --- internal/discovery/discovery_test.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/internal/discovery/discovery_test.go b/internal/discovery/discovery_test.go index 2fd95ec4..61ac4d76 100644 --- a/internal/discovery/discovery_test.go +++ b/internal/discovery/discovery_test.go @@ -216,9 +216,17 @@ func TestDiscoverProjectLayout(t *testing.T) { if !skills[0].HasEval() { t.Error("my-skill should have eval (project layout evals/{name}/eval.yaml)") } - wantEvalPath := filepath.Join(root, "evals", "my-skill", "eval.yaml") - if skills[0].EvalPath != wantEvalPath { - t.Errorf("expected EvalPath %s, got %s", wantEvalPath, skills[0].EvalPath) + // Check structural aspects: eval.yaml should be inside an evals/my-skill/ directory. + // Avoid exact path comparison because filepath.EvalSymlinks can expand short + // Windows paths (e.g. RUNNER~1 → runneradmin), making exact matches unreliable. + if filepath.Base(skills[0].EvalPath) != "eval.yaml" { + t.Errorf("expected eval.yaml filename, got %s", filepath.Base(skills[0].EvalPath)) + } + if filepath.Base(filepath.Dir(skills[0].EvalPath)) != "my-skill" { + t.Errorf("expected eval inside my-skill dir, got %s", filepath.Dir(skills[0].EvalPath)) + } + if filepath.Base(filepath.Dir(filepath.Dir(skills[0].EvalPath))) != "evals" { + t.Errorf("expected eval inside evals/ dir, got %s", filepath.Dir(filepath.Dir(skills[0].EvalPath))) } }