asta-bench/pyproject.toml at main · allenai/asta-bench · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
[build-system]
requires = ["setuptools>=61", "wheel"]
build-backend = "setuptools.build_meta"

[project]
name = "astabench"
version = "0.5.1"
readme = "README.md"
requires-python = ">=3.11"
dependencies = [
    "inspect_ai==0.3.114",
    "agent-eval==0.1.44",
    "openai>=1.78.0", # required by inspect
    "pydantic>=2.11.4", # required by inspect
    "litellm",
    "datasets~=3.2.0",
    "huggingface_hub",
    "google-genai>=1.16.1",
    "nltk",
    "tabulate",
    "click==8.1.8",
    "nora_lib-impl~=1.1.0",
    "mcp~=1.10",
    "httpx~=0.28.1",
    "httpx-sse>=0.4.2", # need at least 0.4.2 for a key SSE decoding fix
    "h2~=4.2.0",
    "pandas",
    "scipy",
    "anthropic>=0.52.0",
    "platformdirs",
    "numpy"
]

[project.optional-dependencies]
dev = [
    "black==24.2.0",
    "isort",
    "autoflake",
    "pytest",
    "pytest-asyncio",
    "flake8==7.2.0",
    "flake8-pyproject==1.2.3",
    "mypy==1.15",
    "pandas-stubs",
]
azure = [
    "azure-ai-inference"
]

[project.scripts]
astabench = "astabench.cli:cli"
inspect = "inspect_ai._cli.main:main"

[project.entry-points.inspect_ai]
astabench = "astabench.evals._registry"

[tool.setuptools.packages.find]
where = [".", "inspect_evals/src"]
include = ["astabench*", "inspect_evals", "inspect_evals.core_bench", "inspect_evals.ds1000"]

[tool.uv]
conflicts = [
    [{extra = "sqa"}, {extra = "storm"}],
    [{extra = "sqa"}, {extra = "futurehouse"}],
    [{extra = "storm"}, {extra = "smolagents"}],
]
override-dependencies = [
    # sqa pins openai to a lower version than inspect requires
    "openai==1.78.0",

    # STORM pretends to require a lower version, but doesn't actually need it:
    # https://github.com/allenai/asta-bench/issues/31#issuecomment-3045978008
    "datasets~=3.2.0",
]

[tool.flake8]
select = [
    # F category for logical errors
    "F",

    # Serious "E" errors
    "E721", # `type(x) == <t>` instead of `isinstance(x, <t>)`
    "E901", "E902", "E999" # syntax errors
]

ignore = [
    "F401",  # module imported but unused
    "F541",  # f-string is missing placeholders
]

[tool.pylint.messages_control]
disable = [
    "line-too-long",                 # C0301 - handled by black
    "too-many-instance-attributes",  # R0902 - design choice
    "too-many-arguments",            # R0913 - design choice
    "too-many-locals",               # R0914 - design choice
    "too-many-branches",             # R0912 - design choice
    "too-many-statements",           # R0915 - design choice
    "too-few-public-methods",        # R0903 - design choice
    "no-else-return",                # R1705 - style preference
    "no-else-raise",                 # R1720 - style preference
]

[tool.setuptools.package-data]
"astabench" = ["config/*.yml", "evals/inspect_eval_wrappers/ds1000_splits.json", "util/sandbox/sandbox_compose.yaml", "util/sandbox/Dockerfile", "util/sandbox/*.txt", "evals/demo/code_execution/*.yaml", "evals/demo/arithmetic/data.json"]
"inspect_evals.core_bench" = [
    "*.json",
    "*.yaml",
    "Dockerfile",
    "*.md",
]

[tool.uv.sources]
knowledge-storm = { git = "https://github.com/gituser768/storm", branch = "dh-fix-youcom" }

[tool.pytest.ini_options]
testpaths = ["tests"]
# give each async test its own loop
asyncio_mode = "strict"
asyncio_default_fixture_loop_scope = "function"
# default to skipping expensive tests
addopts = "-m \"not expensive\""
markers = [
  "expensive: mark test as expensive (skipped by default)",
  "asta_tool_api_request: mark test as requiring Asta tool key (auto-skipped when ASTA_TOOL_KEY not set)"
]