Daily Convex Evaluations (OpenAI Models) #8

	name: Daily Convex Evaluations (OpenAI Models)

	on:
	schedule:
	- cron: '0 0 * * *' # Run at midnight UTC daily
	workflow_dispatch: # Allow manual triggering

	jobs:
	run-evals:
	runs-on: ubuntu-latest

	steps:
	- uses: actions/checkout@v4

	- name: Set up Node.js
	uses: actions/setup-node@v4
	with:
	node-version: '18'

	- name: Install Bun
	run: npm install -g [email protected]

	- name: Set up Python
	uses: actions/setup-python@v5
	with:
	python-version: '3.11'

	- name: Install PDM
	run: \|
	curl -sSL https://pdm.fming.dev/install-pdm.py \| python3 -

	- name: Install Python dependencies
	run: pdm install

	- name: Install Bun dependencies
	run: bun install

	- name: Create temp directory
	run: mkdir -p /tmp/convex-evals

	- name: Set main-branch-only environment variables
	if: github.ref == 'refs/heads/main'
	run: \|
	echo "CONVEX_EVAL_ENDPOINT=${{ secrets.CONVEX_EVAL_ENDPOINT }}" >> $GITHUB_ENV
	echo "CONVEX_AUTH_TOKEN=${{ secrets.CONVEX_AUTH_TOKEN }}" >> $GITHUB_ENV

	- name: Set OpenAI models
	run: \|
	echo "MODELS=gpt-4o,o3-mini" >> $GITHUB_ENV

	- name: Run evaluations
	env:
	ENVIRONMENT: ci
	OUTPUT_TEMPDIR: /tmp/convex-evals
	BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }}
	ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
	OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
	TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }}
	GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
	CONVEX_EVAL_ENDPOINT: ${{ secrets.CONVEX_EVAL_ENDPOINT }}
	CONVEX_AUTH_TOKEN: ${{ secrets.CONVEX_AUTH_TOKEN }}
	run: pdm run braintrust eval runner/eval_convex_coding.py

Provide feedback