Skip to content

Commit 5f18ddd

Browse files
claudeclaude
committed
Add automated CI test for Claude Code integration
Add a deterministic 'fruits' test job and comprehensive CI tests to validate that deepwork-generated commands work correctly with Claude Code. Changes: - Add fruits job fixture (identify + classify steps) for CI testing - Add integration tests for fruits workflow (8 tests) - Add e2e tests for Claude Code execution (3 tests, skipped without API key) - Add GitHub Actions workflow for automated testing: - validate-generation: Always runs, tests command generation - claude-code-e2e: Runs with ANTHROPIC_API_KEY, tests actual execution The fruits job is designed to be deterministic: - Input: comma-separated list of items (e.g., "apple, car, banana") - Step 1: Identify which items are fruits - Step 2: Classify fruits by category (citrus, tropical, etc.)
1 parent 9565e3b commit 5f18ddd

7 files changed

Lines changed: 915 additions & 0 deletions

File tree

‎.github/workflows/claude-code-test.yml‎

Lines changed: 213 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,213 @@
1+
name: Claude Code Integration Test
2+
3+
on:
4+
# Manual trigger for testing
5+
workflow_dispatch:
6+
inputs:
7+
debug:
8+
description: 'Enable debug logging'
9+
required: false
10+
default: 'false'
11+
type: boolean
12+
# Run on PRs that modify core code
13+
pull_request:
14+
branches: ["*"]
15+
paths:
16+
- 'src/deepwork/**'
17+
- 'tests/**'
18+
- '.github/workflows/claude-code-test.yml'
19+
# Scheduled run for continuous validation
20+
schedule:
21+
- cron: '0 6 * * 1' # Weekly on Monday at 6 AM UTC
22+
23+
jobs:
24+
# Job 1: Validate command generation (always runs)
25+
validate-generation:
26+
runs-on: ubuntu-latest
27+
steps:
28+
- uses: actions/checkout@v4
29+
30+
- name: Install uv
31+
uses: astral-sh/setup-uv@v4
32+
with:
33+
version: "latest"
34+
35+
- name: Set up Python
36+
uses: actions/setup-python@v5
37+
with:
38+
python-version: "3.11"
39+
40+
- name: Install dependencies
41+
run: uv sync --extra dev
42+
43+
- name: Run fruits workflow tests
44+
run: uv run pytest tests/integration/test_fruits_workflow.py -v
45+
46+
- name: Generate commands and validate structure
47+
run: |
48+
# Create a test environment
49+
mkdir -p test_project/.deepwork/jobs
50+
cp -r tests/fixtures/jobs/fruits test_project/.deepwork/jobs/
51+
52+
# Install deepwork into test project
53+
cd test_project
54+
git init
55+
git config user.email "test@test.com"
56+
git config user.name "Test"
57+
echo "# Test" > README.md
58+
git add . && git commit -m "init"
59+
60+
# Run deepwork sync to generate commands
61+
uv run --directory .. deepwork sync
62+
63+
# Validate generated commands exist
64+
echo "Checking generated commands..."
65+
ls -la .claude/commands/
66+
67+
# Verify command files exist
68+
test -f .claude/commands/fruits.identify.md || (echo "Missing fruits.identify.md" && exit 1)
69+
test -f .claude/commands/fruits.classify.md || (echo "Missing fruits.classify.md" && exit 1)
70+
71+
# Verify command content
72+
grep -q "# fruits.identify" .claude/commands/fruits.identify.md
73+
grep -q "raw_items" .claude/commands/fruits.identify.md
74+
grep -q "identified_fruits.md" .claude/commands/fruits.identify.md
75+
76+
grep -q "# fruits.classify" .claude/commands/fruits.classify.md
77+
grep -q "identified_fruits.md" .claude/commands/fruits.classify.md
78+
grep -q "classified_fruits.md" .claude/commands/fruits.classify.md
79+
80+
echo "Command generation validated successfully!"
81+
82+
# Job 2: End-to-end test with Claude Code (only when API key is available)
83+
claude-code-e2e:
84+
runs-on: ubuntu-latest
85+
needs: validate-generation
86+
if: github.event_name == 'workflow_dispatch' || github.event_name == 'schedule'
87+
env:
88+
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
89+
steps:
90+
- uses: actions/checkout@v4
91+
92+
- name: Check for API key
93+
id: check-key
94+
run: |
95+
if [ -z "$ANTHROPIC_API_KEY" ]; then
96+
echo "has_key=false" >> $GITHUB_OUTPUT
97+
echo "::warning::ANTHROPIC_API_KEY not set, skipping Claude Code e2e test"
98+
else
99+
echo "has_key=true" >> $GITHUB_OUTPUT
100+
fi
101+
102+
- name: Install Node.js (for Claude Code CLI)
103+
if: steps.check-key.outputs.has_key == 'true'
104+
uses: actions/setup-node@v4
105+
with:
106+
node-version: '20'
107+
108+
- name: Install Claude Code CLI
109+
if: steps.check-key.outputs.has_key == 'true'
110+
run: npm install -g @anthropic-ai/claude-code
111+
112+
- name: Install uv
113+
if: steps.check-key.outputs.has_key == 'true'
114+
uses: astral-sh/setup-uv@v4
115+
with:
116+
version: "latest"
117+
118+
- name: Set up Python
119+
if: steps.check-key.outputs.has_key == 'true'
120+
uses: actions/setup-python@v5
121+
with:
122+
python-version: "3.11"
123+
124+
- name: Install deepwork
125+
if: steps.check-key.outputs.has_key == 'true'
126+
run: uv sync
127+
128+
- name: Set up test project
129+
if: steps.check-key.outputs.has_key == 'true'
130+
run: |
131+
mkdir -p test_project/.deepwork/jobs
132+
cp -r tests/fixtures/jobs/fruits test_project/.deepwork/jobs/
133+
134+
cd test_project
135+
git init
136+
git config user.email "test@test.com"
137+
git config user.name "Test"
138+
echo "# CI Test Project" > README.md
139+
git add . && git commit -m "init"
140+
141+
# Generate commands
142+
uv run --directory .. deepwork sync
143+
144+
echo "Test project setup complete"
145+
ls -la .claude/commands/
146+
147+
- name: Run Claude Code - Identify Step
148+
if: steps.check-key.outputs.has_key == 'true'
149+
working-directory: test_project
150+
timeout-minutes: 5
151+
run: |
152+
# Run the identify step with a deterministic input
153+
# Using --print to output result, --yes to auto-accept
154+
claude --yes --print "/fruits.identify" <<EOF
155+
raw_items: apple, car, banana, chair, orange, table, mango, laptop
156+
EOF
157+
158+
# Verify output was created
159+
if [ -f "identified_fruits.md" ]; then
160+
echo "Identify step completed successfully!"
161+
echo "--- Output ---"
162+
cat identified_fruits.md
163+
else
164+
echo "ERROR: identified_fruits.md was not created"
165+
exit 1
166+
fi
167+
168+
- name: Run Claude Code - Classify Step
169+
if: steps.check-key.outputs.has_key == 'true'
170+
working-directory: test_project
171+
timeout-minutes: 5
172+
run: |
173+
# Run the classify step
174+
claude --yes --print "/fruits.classify"
175+
176+
# Verify output was created
177+
if [ -f "classified_fruits.md" ]; then
178+
echo "Classify step completed successfully!"
179+
echo "--- Output ---"
180+
cat classified_fruits.md
181+
else
182+
echo "ERROR: classified_fruits.md was not created"
183+
exit 1
184+
fi
185+
186+
- name: Validate outputs
187+
if: steps.check-key.outputs.has_key == 'true'
188+
working-directory: test_project
189+
run: |
190+
echo "=== Validating outputs ==="
191+
192+
# Check identified_fruits.md contains expected fruits
193+
echo "Checking identified_fruits.md..."
194+
grep -qi "apple" identified_fruits.md || (echo "Missing: apple" && exit 1)
195+
grep -qi "banana" identified_fruits.md || (echo "Missing: banana" && exit 1)
196+
grep -qi "orange" identified_fruits.md || (echo "Missing: orange" && exit 1)
197+
grep -qi "mango" identified_fruits.md || (echo "Missing: mango" && exit 1)
198+
199+
# Check classified_fruits.md has expected structure
200+
echo "Checking classified_fruits.md..."
201+
grep -qi "citrus\|tropical\|pome" classified_fruits.md || (echo "Missing fruit categories" && exit 1)
202+
203+
echo "All validations passed!"
204+
205+
- name: Upload test artifacts
206+
if: steps.check-key.outputs.has_key == 'true' && always()
207+
uses: actions/upload-artifact@v4
208+
with:
209+
name: claude-code-test-outputs
210+
path: |
211+
test_project/identified_fruits.md
212+
test_project/classified_fruits.md
213+
retention-days: 7

‎tests/e2e/__init__.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
"""End-to-end tests for DeepWork with Claude Code."""

0 commit comments

Comments
 (0)