asta-bench/Makefile at main · allenai/asta-bench · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
.PHONY: shell format mypy flake test build-image test-expensive solve-score-smoke solve-score-ci tag

# allow passing extra pytest args, e.g. make test-expensive PYTEST_ARGS="-k EVAL_NAME"
PYTEST_ARGS ?=

ASTABENCH_TAG    := astabench
CONTAINER_NAME   := astabench-container
DOCKER_SOCKET_PATH ?= $(if $(XDG_RUNTIME_DIR),$(XDG_RUNTIME_DIR)/docker.sock,/var/run/docker.sock)

ENV_ARGS :=

# Docker target for running a shell
TARGET := --target astabench-base

# Add each env var only if it's defined
ifdef OPENAI_API_KEY
  ENV_ARGS += -e OPENAI_API_KEY
endif

ifdef AZUREAI_OPENAI_API_KEY
  ENV_ARGS += -e AZUREAI_OPENAI_API_KEY
endif

ifdef HF_TOKEN
  ENV_ARGS += -e HF_TOKEN
endif

# Also support .env file if it exists
ifneq ("$(wildcard .env)","")
  ENV_ARGS += --env-file .env
endif

# -----------------------------------------------------------------------------
# Local vs CI environment vars
# -----------------------------------------------------------------------------
ifeq ($(IS_CI),true)
  LOCAL_MOUNTS :=
  ENV_ARGS += -e IS_CI
  TEST_RUN := docker run --rm $(ENV_ARGS) -v /var/run/docker.sock:/var/run/docker.sock $(ASTABENCH_TAG)
  BUILD_QUIET := --quiet
else
  LOCAL_MOUNTS := \
    -v $(DOCKER_SOCKET_PATH):/var/run/docker.sock \
    -v $$(pwd)/pyproject.toml:/astabench/pyproject.toml:ro \
    -v $$(pwd)/astabench:/astabench/astabench \
    -v $$(pwd)/tests:/astabench/tests \
    -v $$(pwd)/logs:/astabench/logs \
    -v astabench-cache:/root/.cache
  TEST_RUN := docker run --rm $(ENV_ARGS) $(LOCAL_MOUNTS) $(ASTABENCH_TAG)
  BUILD_QUIET ?=
endif

# -----------------------------------------------------------------------------
# Build the Docker image (primary target)
# -----------------------------------------------------------------------------
build-image:
	docker build $(BUILD_QUIET) $(TARGET) . --tag $(ASTABENCH_TAG) -f ./docker/Dockerfile

# -----------------------------------------------------------------------------
# Interactive shell in container
# -----------------------------------------------------------------------------
shell: build-image
	@git submodule update --init --recursive
	@docker run --rm -it --name $(CONTAINER_NAME) \
		$(LOCAL_MOUNTS) \
		-v astabench-home:/root/.astabench \
		$(ENV_ARGS) -p 7575:7575 \
		$(ASTABENCH_TAG) \
		/bin/bash

# -----------------------------------------------------------------------------
#  Formatting and linting
# -----------------------------------------------------------------------------
# NOTE: These commands aim to install only the dev dependencies, without the
# main package depedencies which require more complex setup, e.g., ~ssh.
# Ideally they would install the exact lib versions of the dev dependencies,
# but limiting to only dev dependencies in pyproject.toml in a DRY manner is not
# easy to do since pip has no mechanism and uv requires defining a seeparate section
# in pyproject.toml which pip cannot read.

ifneq ($(IS_CI),true)
format: build-image
endif

format:
	docker run --rm \
		-v $$(pwd):/astabench \
		$(ASTABENCH_TAG) \
		sh -c "pip install --no-cache-dir black && black --exclude '(inspect_evals|\.venv)' ."

ifneq ($(IS_CI),true)
mypy: build-image
endif

mypy:
	docker run --rm \
		-v $$(pwd):/astabench \
		$(ASTABENCH_TAG) \
		uv run mypy astabench/ tests/

ifneq ($(IS_CI),true)
flake: build-image
endif

flake:
	docker run --rm \
		$(ASTABENCH_TAG) \
		uv run flake8 astabench/ tests/

ifneq ($(IS_CI),true)
test: build-image
endif

test:
	@$(TEST_RUN) uv run --no-sync --extra dev \
		-m pytest $(PYTEST_ARGS) -vv /astabench/tests

ifneq ($(IS_CI),true)
test-expensive: build-image
endif

test-expensive:
	@$(TEST_RUN) uv run --no-sync --extra dev \
		-m pytest $(PYTEST_ARGS) -vv -o addopts= -m expensive /astabench/tests

ifneq ($(IS_CI),true)
solve-score-smoke: build-image
solve-score-ci: build-image
endif

solve-score-smoke:
	@$(MAKE) solve-score-ci

solve-score-ci:
	@$(TEST_RUN) ./scripts/smoke_solve_score.sh

tag:
	@VERSION=$$(grep '^version = ' pyproject.toml | sed 's/version = "\(.*\)"/\1/'); \
	echo "Creating and pushing tag v$$VERSION"; \
	git tag "v$$VERSION" && \
	git push origin "v$$VERSION"