frontier-evals/project/paperbench/pyproject.toml at main · openai/frontier-evals · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
[build-system]
requires = ["setuptools<78.0.0", "wheel"]
build-backend = "setuptools.build_meta"

[project]
name = "paperbench"
version = "1.0.0"
requires-python = ">=3.11"
dependencies = [
    "docker>=7.1.0",
    "python-dotenv>=1.0.0",
    "pyyaml>=6.0",
    "tqdm>=4.66.5",
    "tenacity>=9.0.0",
    "flask>=3.0.3",
    "scikit-learn>=1.5.2",
    "openai>=1.54.3",
    "pandas>=2.0.0",
    "drain3>=0.9.11",
    "blobfile>=3.0.0",
    "tiktoken>=0.9.0",
    "python-dateutil>=2.9.0",
    "unidecode>=1.3.8",
    "structlog>=25.1.0",
    "chz",
    "nanoeval",
    "nanoeval_alcatraz",
    "alcatraz",
    "preparedness-turn-completer",
    "pandas-stubs>=2.3.0.250703",
]

[tool.setuptools]
packages = { find = { include = ["paperbench", "paperbench.*"] } }

[tool.pyright]
extraPaths = [
    "../common/alcatraz",
    "../common/preparedness_turn_completer",
    "../common/nanoeval",
    "../common/nanoeval_alcatraz",
]
typeCheckingMode = "off" # we use mypy

[dependency-groups]
dev = [
    "mypy>=1.15.0",
    "poethepoet>=0.34.0",
    "pre-commit>=4.2.0",
    "pytest>=8.3.5",
    "pytest-asyncio>=0.26.0",
    "pytest-html>=4.1.1",
    "pytest-xdist>=3.6.1",
    "ruff>=0.11.7",
]

[tool.uv.sources]
chz = { git = "https://github.com/openai/chz.git" }
nanoeval = { path = "../common/nanoeval", editable = true }
nanoeval_alcatraz = { path = "../common/nanoeval_alcatraz", editable = true }
alcatraz = { path = "../common/alcatraz", editable = true }
preparedness-turn-completer = { path = "../common/preparedness_turn_completer", editable = true }

[tool.ruff]
extend = "../../pyproject.toml"
extend-exclude = ["paperbench/solvers/aisi-basic-agent/"]

[tool.mypy]
strict = true
local_partial_types = true
# These two make life easier if you use a lot of untyped libraries:
warn_return_any = false
allow_untyped_calls = true
# Do not use `ignore_missing_imports`, instead use:
disable_error_code = ["import-untyped"]
# If you don't want to force use of type annotations:
# allow_incomplete_defs = true
# allow_untyped_defs = true
# allow_any_generics = true
exclude = "^(paperbench/solvers/aisi-basic-agent|tests/integration/runs)(/|$)"
mypy_path = ["../common/alcatraz", "../common/preparedness_turn_completer", "../common/nanoeval", "../common/nanoeval_alcatraz"]
[[tool.mypy.overrides]]
module = [
    "alcatraz.*",
    "preparedness_turn_completer.*",
    "nanoeval.*",
    "nanoeval_alcatraz.*",
]
ignore_errors = true
warn_unused_ignores = false

[tool.pylint]
disable = ["all"]
# will eventually be supported by ruff: github.com/astral-sh/ruff/issues/2914
enable = ["cyclic-import"]
ignore-paths = [
    "^paperbench/solvers/aisi-basic-agent(/|$)",
    "^tests/integration/runs(/|$)",
]

[tool.poe.tasks]
lint = "ruff check --fix"
typecheck = "mypy paperbench tests"