Skip to content

Commit cd86a78

Browse files
bastoicabastoica
committed
refactor: Improve logic around directory layout, file path discovery, configs and checks
1 parent 763e637 commit cd86a78

File tree

1 file changed

+118
-69
lines changed
  • benchmarks/arteval_bench/data/benchmark/sosp24_wasabi/_agent_eval

1 file changed

+118
-69
lines changed

‎benchmarks/arteval_bench/data/benchmark/sosp24_wasabi/_agent_eval/main.py‎

Lines changed: 118 additions & 69 deletions
Original file line numberDiff line numberDiff line change
@@ -19,71 +19,113 @@
1919
from oracle_benchmark_prep import OracleBenchmarkPrep
2020
from oracle_experiment_runs import OracleExperimentRuns
2121

22-
_AGENT_EVAL_DIR = Path(__file__).resolve().parent
23-
_WASABI_HOME = _AGENT_EVAL_DIR.parent.resolve()
24-
_WASABI_REPO = (_WASABI_HOME / "wasabi" / "wasabi-testing").resolve()
25-
_WASABI_BENCH = (_WASABI_HOME / "benchmarks").resolve()
26-
27-
WASABI_CONFIG = EntryConfig(
28-
name="sosp24-wasabi",
29-
home_dir=_WASABI_HOME,
30-
repository_paths={
31-
"sosp24-wasabi": _WASABI_REPO,
32-
"benchmarks": _WASABI_BENCH,
33-
},
34-
results_paths={
35-
"results_root": _WASABI_REPO / "results",
36-
},
37-
ground_truth_paths={
38-
"bugs_ground_truth": _AGENT_EVAL_DIR / "refs" / "bugs_ground_truth.csv",
39-
},
40-
similarity_ratio=0.75,
41-
metadata={
42-
"maven_repo_dir":
43-
Path.home() / ".m2" / "repository",
44-
"weaving_plugin_signature":
45-
"aspectj-maven-plugin",
46-
"primary_artifact":
47-
"edu.uchicago.cs.systems:wasabi",
48-
"benchmarks": {
49-
"hadoop": {
50-
"repo_url": "https://github.com/apache/hadoop.git",
51-
"commit": "60867de",
52-
"pom_file": "pom.xml",
53-
"pom_backup": "pom-original.xml",
54-
},
55-
"hbase": {
56-
"repo_url": "https://github.com/apache/hbase.git",
57-
"commit": "89ca7f4",
58-
"pom_file": "pom.xml",
59-
"pom_backup": "pom-original.xml",
60-
},
61-
"hive": {
62-
"repo_url": "https://github.com/apache/hive.git",
63-
"commit": "e08a600",
64-
"pom_file": "pom.xml",
65-
"pom_backup": "pom-original.xml",
66-
},
67-
},
68-
"aspectj_markers": [
69-
"ajc$preClinit",
70-
"ajc$initFailureCause",
71-
"ajc$tjp",
72-
"ajc$before$",
73-
"ajc$after$",
74-
"ajc$around$",
75-
"ajc$interField$",
76-
"ajc$interMethod$",
77-
"org.aspectj.runtime.reflect.Factory",
78-
"org.aspectj.runtime.internal.AroundClosure",
79-
"org.aspectj.lang.JoinPoint",
80-
"org.aspectj.lang.JoinPoint$StaticPart",
81-
"org.aspectj.lang.ProceedingJoinPoint",
82-
"org.aspectj.lang.Signature",
83-
"org.aspectj.lang.NoAspectBoundException",
84-
],
85-
},
86-
)
22+
23+
def _resolve_workspace_paths() -> tuple[Path, Path, Path]:
24+
"""Resolve and validate _agent_eval/ and wasabi/ locations.
25+
26+
Priority:
27+
1) Environment variables: _AGENT_EVAL_DIR and _WASABI_HOME
28+
2) Derive from this file location assuming _agent_eval/ and wasabi/ are siblings
29+
"""
30+
try:
31+
env_agent_eval = os.environ.get("_AGENT_EVAL_DIR")
32+
env_wasabi_home = os.environ.get("_WASABI_HOME")
33+
34+
if env_agent_eval:
35+
agent_eval_dir = Path(env_agent_eval).expanduser().resolve()
36+
else:
37+
agent_eval_dir = Path(__file__).resolve().parent
38+
39+
if env_wasabi_home:
40+
wasabi_home = Path(env_wasabi_home).expanduser().resolve()
41+
else:
42+
wasabi_home = agent_eval_dir.parent.resolve()
43+
44+
if not agent_eval_dir.exists() or not agent_eval_dir.is_dir():
45+
raise RuntimeError(
46+
f"Invalid _agent_eval dir: {agent_eval_dir}\n"
47+
f"This runner expects _agent_eval/ and wasabi/ to be located in the same root directory.\n"
48+
f"Set _AGENT_EVAL_DIR to the directory containing main.py if needed."
49+
)
50+
51+
wasabi_repo_root = wasabi_home / "wasabi"
52+
if not wasabi_repo_root.exists() or not wasabi_repo_root.is_dir():
53+
raise RuntimeError(
54+
f"Invalid WASABI workspace: {wasabi_home}\n"
55+
f"Expected to find a 'wasabi/' directory at: {wasabi_repo_root}\n"
56+
f"This runner expects _agent_eval/ and wasabi/ to be located in the same root directory.\n"
57+
f"Set _WASABI_HOME to the workspace root if needed."
58+
)
59+
60+
workspace_root = wasabi_home
61+
return agent_eval_dir, wasabi_home, workspace_root
62+
63+
except OSError as exc:
64+
raise RuntimeError(f"Failed to resolve workspace paths: {exc}") from exc
65+
66+
67+
def _build_configs(*, agent_eval_dir: Path, workspace_root: Path) -> EntryConfig:
68+
"""Constructs EntryConfig for the WASABI evaluation bundle from resolved paths."""
69+
wasabi_repo = (workspace_root / "wasabi").resolve()
70+
benchmarks_dir = (workspace_root / "benchmarks").resolve()
71+
72+
return EntryConfig(
73+
name="sosp24-wasabi",
74+
home_dir=workspace_root,
75+
repository_paths={
76+
"sosp24-wasabi": wasabi_repo,
77+
"benchmarks": benchmarks_dir,
78+
},
79+
results_paths={
80+
"results_root": wasabi_repo / "results",
81+
},
82+
ground_truth_paths={
83+
"bugs_ground_truth": agent_eval_dir / "refs" / "bugs_ground_truth.csv",
84+
},
85+
similarity_ratio=0.75,
86+
metadata={
87+
"maven_repo_dir": Path.home() / ".m2" / "repository",
88+
"weaving_plugin_signature": "aspectj-maven-plugin",
89+
"primary_artifact": "edu.uchicago.cs.systems:wasabi",
90+
"benchmarks": {
91+
"hadoop": {
92+
"repo_url": "https://github.com/apache/hadoop.git",
93+
"commit": "60867de",
94+
"pom_file": "pom.xml",
95+
"pom_backup": "pom-original.xml",
96+
},
97+
"hbase": {
98+
"repo_url": "https://github.com/apache/hbase.git",
99+
"commit": "89ca7f4",
100+
"pom_file": "pom.xml",
101+
"pom_backup": "pom-original.xml",
102+
},
103+
"hive": {
104+
"repo_url": "https://github.com/apache/hive.git",
105+
"commit": "e08a600",
106+
"pom_file": "pom.xml",
107+
"pom_backup": "pom-original.xml",
108+
},
109+
},
110+
"aspectj_markers": [
111+
"ajc$preClinit",
112+
"ajc$initFailureCause",
113+
"ajc$tjp",
114+
"ajc$before$",
115+
"ajc$after$",
116+
"ajc$around$",
117+
"ajc$interField$",
118+
"ajc$interMethod$",
119+
"org.aspectj.runtime.reflect.Factory",
120+
"org.aspectj.runtime.internal.AroundClosure",
121+
"org.aspectj.lang.JoinPoint",
122+
"org.aspectj.lang.JoinPoint$StaticPart",
123+
"org.aspectj.lang.ProceedingJoinPoint",
124+
"org.aspectj.lang.Signature",
125+
"org.aspectj.lang.NoAspectBoundException",
126+
],
127+
},
128+
)
87129

88130

89131
def main(argv: list[str]) -> int:
@@ -95,19 +137,26 @@ def main(argv: list[str]) -> int:
95137
logger_name = os.environ.get("EVAL_LOGGER_NAME", "WASABI-AGENT-EVALUATOR")
96138
logger = get_logger(LoggerConfig(root_name=logger_name))
97139

98-
env_checker = OracleEnvSetup(config=WASABI_CONFIG, logger=logger)
140+
try:
141+
agent_eval_dir, _wasabi_home, workspace_root = _resolve_workspace_paths()
142+
wasabi_config = _build_configs(agent_eval_dir=agent_eval_dir, workspace_root=workspace_root)
143+
except RuntimeError as exc:
144+
# Keep failure message clean and actionable
145+
raise SystemExit(str(exc)) from exc
146+
147+
env_checker = OracleEnvSetup(config=wasabi_config, logger=logger)
99148
env_ok = env_checker.run(verbose=verbose)
100149
score += record_result(results, type(env_checker).__name__, env_ok)
101150

102-
build_checker = OracleArtifactBuild(config=WASABI_CONFIG, logger=logger)
151+
build_checker = OracleArtifactBuild(config=wasabi_config, logger=logger)
103152
build_ok = build_checker.run(verbose=verbose)
104153
score += record_result(results, type(build_checker).__name__, build_ok)
105154

106-
prep_checker = OracleBenchmarkPrep(config=WASABI_CONFIG, logger=logger)
155+
prep_checker = OracleBenchmarkPrep(config=wasabi_config, logger=logger)
107156
prep_ok = prep_checker.run(verbose=verbose)
108157
score += record_result(results, type(prep_checker).__name__, prep_ok)
109158

110-
runs_checker = OracleExperimentRuns(config=WASABI_CONFIG, logger=logger)
159+
runs_checker = OracleExperimentRuns(config=wasabi_config, logger=logger)
111160
runs_ok = runs_checker.run(verbose=verbose)
112161
score += record_result(results, type(runs_checker).__name__, runs_ok)
113162

0 commit comments

Comments
 (0)