|
| 1 | +name: Claude Code Integration Test |
| 2 | + |
| 3 | +on: |
| 4 | + # Manual trigger for testing |
| 5 | + workflow_dispatch: |
| 6 | + inputs: |
| 7 | + debug: |
| 8 | + description: 'Enable debug logging' |
| 9 | + required: false |
| 10 | + default: 'false' |
| 11 | + type: boolean |
| 12 | + # Run on PRs that modify core code |
| 13 | + pull_request: |
| 14 | + branches: ["*"] |
| 15 | + paths: |
| 16 | + - 'src/deepwork/**' |
| 17 | + - 'tests/**' |
| 18 | + - '.github/workflows/claude-code-test.yml' |
| 19 | + # Scheduled run for continuous validation |
| 20 | + schedule: |
| 21 | + - cron: '0 6 * * 1' # Weekly on Monday at 6 AM UTC |
| 22 | + |
| 23 | +# Ensure only one instance runs at a time per PR/branch |
| 24 | +concurrency: |
| 25 | + group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} |
| 26 | + cancel-in-progress: true |
| 27 | + |
| 28 | +jobs: |
| 29 | + # Job 1: Validate command generation from fixtures (always runs, no API key needed) |
| 30 | + validate-generation: |
| 31 | + runs-on: ubuntu-latest |
| 32 | + steps: |
| 33 | + - uses: actions/checkout@v4 |
| 34 | + |
| 35 | + - name: Install uv |
| 36 | + uses: astral-sh/setup-uv@v4 |
| 37 | + with: |
| 38 | + version: "latest" |
| 39 | + |
| 40 | + - name: Set up Python |
| 41 | + uses: actions/setup-python@v5 |
| 42 | + with: |
| 43 | + python-version: "3.11" |
| 44 | + |
| 45 | + - name: Install dependencies |
| 46 | + run: uv sync --extra dev |
| 47 | + |
| 48 | + - name: Run fruits workflow tests |
| 49 | + run: uv run pytest tests/integration/test_fruits_workflow.py -v |
| 50 | + |
| 51 | + - name: Generate commands and validate structure |
| 52 | + run: | |
| 53 | + # Create a test environment |
| 54 | + mkdir -p test_project/.deepwork/jobs |
| 55 | + mkdir -p test_project/.claude # Required for platform detection |
| 56 | + cp -r tests/fixtures/jobs/fruits test_project/.deepwork/jobs/ |
| 57 | +
|
| 58 | + # Set up git repo in test project |
| 59 | + cd test_project |
| 60 | + git init |
| 61 | + git config user.email "test@test.com" |
| 62 | + git config user.name "Test" |
| 63 | + echo "# Test" > README.md |
| 64 | + git add . && git commit -m "init" |
| 65 | + cd .. |
| 66 | +
|
| 67 | + # Run deepwork install to set up the project (this also runs sync) |
| 68 | + uv run deepwork install --platform claude --path test_project |
| 69 | +
|
| 70 | + # Validate generated commands exist |
| 71 | + echo "Checking generated commands..." |
| 72 | + ls -la test_project/.claude/commands/ |
| 73 | +
|
| 74 | + # Verify command files exist |
| 75 | + test -f test_project/.claude/commands/fruits.identify.md || (echo "Missing fruits.identify.md" && exit 1) |
| 76 | + test -f test_project/.claude/commands/fruits.classify.md || (echo "Missing fruits.classify.md" && exit 1) |
| 77 | +
|
| 78 | + # Verify command content |
| 79 | + grep -q "# fruits.identify" test_project/.claude/commands/fruits.identify.md |
| 80 | + grep -q "raw_items" test_project/.claude/commands/fruits.identify.md |
| 81 | + grep -q "identified_fruits.md" test_project/.claude/commands/fruits.identify.md |
| 82 | +
|
| 83 | + grep -q "# fruits.classify" test_project/.claude/commands/fruits.classify.md |
| 84 | + grep -q "identified_fruits.md" test_project/.claude/commands/fruits.classify.md |
| 85 | + grep -q "classified_fruits.md" test_project/.claude/commands/fruits.classify.md |
| 86 | +
|
| 87 | + echo "Command generation validated successfully!" |
| 88 | +
|
| 89 | + # Job 2: Full end-to-end test with Claude Code |
| 90 | + # Tests the COMPLETE workflow: define job -> implement -> execute |
| 91 | + claude-code-e2e: |
| 92 | + runs-on: ubuntu-latest |
| 93 | + needs: validate-generation |
| 94 | + if: github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' |
| 95 | + env: |
| 96 | + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} |
| 97 | + steps: |
| 98 | + - uses: actions/checkout@v4 |
| 99 | + |
| 100 | + - name: Check for API key |
| 101 | + id: check-key |
| 102 | + run: | |
| 103 | + if [ -z "$ANTHROPIC_API_KEY" ]; then |
| 104 | + echo "has_key=false" >> $GITHUB_OUTPUT |
| 105 | + echo "::warning::ANTHROPIC_API_KEY not set, skipping Claude Code e2e test" |
| 106 | + else |
| 107 | + echo "has_key=true" >> $GITHUB_OUTPUT |
| 108 | + fi |
| 109 | +
|
| 110 | + - name: Install Node.js (for Claude Code CLI) |
| 111 | + if: steps.check-key.outputs.has_key == 'true' |
| 112 | + uses: actions/setup-node@v4 |
| 113 | + with: |
| 114 | + node-version: '20' |
| 115 | + |
| 116 | + - name: Install Claude Code CLI |
| 117 | + if: steps.check-key.outputs.has_key == 'true' |
| 118 | + run: npm install -g @anthropic-ai/claude-code |
| 119 | + |
| 120 | + - name: Install uv |
| 121 | + if: steps.check-key.outputs.has_key == 'true' |
| 122 | + uses: astral-sh/setup-uv@v4 |
| 123 | + with: |
| 124 | + version: "latest" |
| 125 | + |
| 126 | + - name: Set up Python |
| 127 | + if: steps.check-key.outputs.has_key == 'true' |
| 128 | + uses: actions/setup-python@v5 |
| 129 | + with: |
| 130 | + python-version: "3.11" |
| 131 | + |
| 132 | + - name: Install deepwork |
| 133 | + if: steps.check-key.outputs.has_key == 'true' |
| 134 | + run: uv sync |
| 135 | + |
| 136 | + - name: Set up fresh test project |
| 137 | + if: steps.check-key.outputs.has_key == 'true' |
| 138 | + run: | |
| 139 | + # Create a fresh project with NO pre-existing job definitions |
| 140 | + mkdir -p test_project/.claude |
| 141 | +
|
| 142 | + cd test_project |
| 143 | + git init |
| 144 | + git config user.email "test@test.com" |
| 145 | + git config user.name "Test" |
| 146 | + echo "# CI Test Project - DeepWork E2E Test" > README.md |
| 147 | + git add . && git commit -m "init" |
| 148 | + cd .. |
| 149 | +
|
| 150 | + # Install deepwork (this sets up .deepwork/ with standard jobs only) |
| 151 | + uv run deepwork install --platform claude --path test_project |
| 152 | +
|
| 153 | + echo "Fresh test project setup complete" |
| 154 | + echo "Available commands:" |
| 155 | + ls -la test_project/.claude/commands/ |
| 156 | +
|
| 157 | + # STEP 1: Use /deepwork_jobs.define to CREATE the fruits job |
| 158 | + - name: Create job with /deepwork_jobs.define |
| 159 | + if: steps.check-key.outputs.has_key == 'true' |
| 160 | + working-directory: test_project |
| 161 | + timeout-minutes: 10 |
| 162 | + run: | |
| 163 | + echo "=== Running /deepwork_jobs.define to create fruits job ===" |
| 164 | +
|
| 165 | + # Provide detailed, deterministic instructions for creating the job |
| 166 | + claude --yes --print "/deepwork_jobs.define" <<'PROMPT_EOF' |
| 167 | + I want to create a simple job called "fruits" for identifying and classifying fruits. |
| 168 | +
|
| 169 | + Here are the EXACT specifications - please create the job.yml with these exact details: |
| 170 | +
|
| 171 | + Job name: fruits |
| 172 | + Version: 1.0.0 |
| 173 | + Summary: Identify and classify fruits from a mixed list of items |
| 174 | +
|
| 175 | + Description: A simple workflow that takes a list of mixed items, identifies which are fruits, then classifies them by category. Designed for CI testing. |
| 176 | +
|
| 177 | + Steps: |
| 178 | + 1. Step ID: identify |
| 179 | + Name: Identify Fruits |
| 180 | + Description: Filter a list of items to identify only the fruits |
| 181 | + Input: raw_items (user parameter) - A comma-separated list of items |
| 182 | + Output: identified_fruits.md |
| 183 | + Dependencies: none |
| 184 | +
|
| 185 | + 2. Step ID: classify |
| 186 | + Name: Classify Fruits |
| 187 | + Description: Organize identified fruits into categories (citrus, tropical, berries, etc.) |
| 188 | + Input: identified_fruits.md (file from step identify) |
| 189 | + Output: classified_fruits.md |
| 190 | + Dependencies: identify |
| 191 | +
|
| 192 | + Please create this job definition now. Do not ask questions - use these exact specifications. |
| 193 | + PROMPT_EOF |
| 194 | +
|
| 195 | + # Verify the job.yml was created |
| 196 | + echo "=== Checking job.yml was created ===" |
| 197 | + if [ -f ".deepwork/jobs/fruits/job.yml" ]; then |
| 198 | + echo "SUCCESS: job.yml created" |
| 199 | + cat .deepwork/jobs/fruits/job.yml |
| 200 | + else |
| 201 | + echo "ERROR: job.yml was not created" |
| 202 | + echo "Contents of .deepwork/jobs/:" |
| 203 | + ls -la .deepwork/jobs/ || echo "No jobs directory" |
| 204 | + exit 1 |
| 205 | + fi |
| 206 | +
|
| 207 | + # STEP 2: Use /deepwork_jobs.implement to generate step instructions |
| 208 | + - name: Generate step instructions with /deepwork_jobs.implement |
| 209 | + if: steps.check-key.outputs.has_key == 'true' |
| 210 | + working-directory: test_project |
| 211 | + timeout-minutes: 10 |
| 212 | + run: | |
| 213 | + echo "=== Running /deepwork_jobs.implement to generate step instructions ===" |
| 214 | +
|
| 215 | + claude --yes --print "/deepwork_jobs.implement" <<'PROMPT_EOF' |
| 216 | + Please implement the "fruits" job that was just defined. |
| 217 | +
|
| 218 | + For the identify step, create instructions that: |
| 219 | + - Parse the comma-separated raw_items input |
| 220 | + - Identify which items are fruits (apple, banana, orange, mango, grape, etc.) |
| 221 | + - Output a markdown file listing the identified fruits |
| 222 | +
|
| 223 | + For the classify step, create instructions that: |
| 224 | + - Read identified_fruits.md from the previous step |
| 225 | + - Classify fruits into categories: Citrus (orange, lemon), Tropical (banana, mango), Pome (apple, pear), Berries, etc. |
| 226 | + - Output a markdown file with fruits organized by category |
| 227 | +
|
| 228 | + Generate the step instruction files now. |
| 229 | + PROMPT_EOF |
| 230 | +
|
| 231 | + # Verify step files were created |
| 232 | + echo "=== Checking step files were created ===" |
| 233 | + if [ -f ".deepwork/jobs/fruits/steps/identify.md" ] && [ -f ".deepwork/jobs/fruits/steps/classify.md" ]; then |
| 234 | + echo "SUCCESS: Step instruction files created" |
| 235 | + echo "--- identify.md ---" |
| 236 | + cat .deepwork/jobs/fruits/steps/identify.md |
| 237 | + echo "" |
| 238 | + echo "--- classify.md ---" |
| 239 | + cat .deepwork/jobs/fruits/steps/classify.md |
| 240 | + else |
| 241 | + echo "ERROR: Step files were not created" |
| 242 | + ls -la .deepwork/jobs/fruits/steps/ || echo "No steps directory" |
| 243 | + exit 1 |
| 244 | + fi |
| 245 | +
|
| 246 | + # Run sync to generate the slash commands |
| 247 | + echo "=== Running deepwork sync to generate commands ===" |
| 248 | + cd .. |
| 249 | + uv run deepwork sync --path test_project |
| 250 | +
|
| 251 | + echo "=== Checking generated commands ===" |
| 252 | + ls -la test_project/.claude/commands/ |
| 253 | +
|
| 254 | + if [ -f "test_project/.claude/commands/fruits.identify.md" ] && [ -f "test_project/.claude/commands/fruits.classify.md" ]; then |
| 255 | + echo "SUCCESS: Slash commands generated" |
| 256 | + else |
| 257 | + echo "ERROR: Slash commands were not generated" |
| 258 | + exit 1 |
| 259 | + fi |
| 260 | +
|
| 261 | + # STEP 3: Execute the generated /fruits.identify command |
| 262 | + - name: Run /fruits.identify |
| 263 | + if: steps.check-key.outputs.has_key == 'true' |
| 264 | + working-directory: test_project |
| 265 | + timeout-minutes: 5 |
| 266 | + run: | |
| 267 | + echo "=== Running /fruits.identify with test input ===" |
| 268 | +
|
| 269 | + claude --yes --print "/fruits.identify" <<'PROMPT_EOF' |
| 270 | + raw_items: apple, car, banana, chair, orange, table, mango, laptop, grape, bicycle |
| 271 | + PROMPT_EOF |
| 272 | +
|
| 273 | + # Verify output was created |
| 274 | + if [ -f "identified_fruits.md" ]; then |
| 275 | + echo "SUCCESS: identified_fruits.md created" |
| 276 | + echo "--- Output ---" |
| 277 | + cat identified_fruits.md |
| 278 | + else |
| 279 | + echo "ERROR: identified_fruits.md was not created" |
| 280 | + exit 1 |
| 281 | + fi |
| 282 | +
|
| 283 | + # STEP 4: Execute the generated /fruits.classify command |
| 284 | + - name: Run /fruits.classify |
| 285 | + if: steps.check-key.outputs.has_key == 'true' |
| 286 | + working-directory: test_project |
| 287 | + timeout-minutes: 5 |
| 288 | + run: | |
| 289 | + echo "=== Running /fruits.classify ===" |
| 290 | +
|
| 291 | + claude --yes --print "/fruits.classify" |
| 292 | +
|
| 293 | + # Verify output was created |
| 294 | + if [ -f "classified_fruits.md" ]; then |
| 295 | + echo "SUCCESS: classified_fruits.md created" |
| 296 | + echo "--- Output ---" |
| 297 | + cat classified_fruits.md |
| 298 | + else |
| 299 | + echo "ERROR: classified_fruits.md was not created" |
| 300 | + exit 1 |
| 301 | + fi |
| 302 | +
|
| 303 | + # STEP 5: Validate the complete workflow output |
| 304 | + - name: Validate complete workflow |
| 305 | + if: steps.check-key.outputs.has_key == 'true' |
| 306 | + working-directory: test_project |
| 307 | + run: | |
| 308 | + echo "=== Validating complete workflow ===" |
| 309 | +
|
| 310 | + # Check identified_fruits.md contains expected fruits |
| 311 | + echo "Checking identified_fruits.md..." |
| 312 | + grep -qi "apple" identified_fruits.md || (echo "Missing: apple" && exit 1) |
| 313 | + grep -qi "banana" identified_fruits.md || (echo "Missing: banana" && exit 1) |
| 314 | + grep -qi "orange" identified_fruits.md || (echo "Missing: orange" && exit 1) |
| 315 | + grep -qi "mango" identified_fruits.md || (echo "Missing: mango" && exit 1) |
| 316 | + grep -qi "grape" identified_fruits.md || (echo "Missing: grape" && exit 1) |
| 317 | + echo " ✓ All expected fruits found in identified_fruits.md" |
| 318 | +
|
| 319 | + # Check classified_fruits.md has expected structure |
| 320 | + echo "Checking classified_fruits.md..." |
| 321 | + grep -qi "citrus\|tropical\|pome\|berr" classified_fruits.md || (echo "Missing fruit categories" && exit 1) |
| 322 | + echo " ✓ Fruit categories found in classified_fruits.md" |
| 323 | +
|
| 324 | + # Verify job structure was created correctly |
| 325 | + echo "Checking job structure..." |
| 326 | + test -f .deepwork/jobs/fruits/job.yml || (echo "Missing job.yml" && exit 1) |
| 327 | + test -f .deepwork/jobs/fruits/steps/identify.md || (echo "Missing identify.md" && exit 1) |
| 328 | + test -f .deepwork/jobs/fruits/steps/classify.md || (echo "Missing classify.md" && exit 1) |
| 329 | + echo " ✓ Job structure is complete" |
| 330 | +
|
| 331 | + echo "" |
| 332 | + echo "==========================================" |
| 333 | + echo " ALL E2E TESTS PASSED SUCCESSFULLY!" |
| 334 | + echo "==========================================" |
| 335 | + echo "" |
| 336 | + echo "Workflow tested:" |
| 337 | + echo " 1. /deepwork_jobs.define - Created job specification" |
| 338 | + echo " 2. /deepwork_jobs.implement - Generated step instructions" |
| 339 | + echo " 3. /fruits.identify - Executed identify step" |
| 340 | + echo " 4. /fruits.classify - Executed classify step" |
| 341 | + echo "" |
| 342 | +
|
| 343 | + - name: Upload test artifacts |
| 344 | + if: steps.check-key.outputs.has_key == 'true' && always() |
| 345 | + uses: actions/upload-artifact@v4 |
| 346 | + with: |
| 347 | + name: claude-code-e2e-outputs |
| 348 | + path: | |
| 349 | + test_project/.deepwork/jobs/fruits/ |
| 350 | + test_project/.claude/commands/fruits.*.md |
| 351 | + test_project/identified_fruits.md |
| 352 | + test_project/classified_fruits.md |
| 353 | + retention-days: 7 |
0 commit comments