Skip to content

Commit 103d016

Browse files
dshkoldshkol
authored
Merge pull request #1 from dshkol/feature/autonomous-pipeline
Feature/autonomous pipeline
2 parents 9ac29eb + 64b41b8 commit 103d016

15 files changed

Lines changed: 1700 additions & 59 deletions

‎.github/workflows/daily.yml‎

Lines changed: 214 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,214 @@
1+
name: The D-AI-LY - Automated Pipeline
2+
3+
on:
4+
# Run daily at 8am Eastern (13:00 UTC)
5+
schedule:
6+
- cron: '0 13 * * *'
7+
8+
# Allow manual trigger
9+
workflow_dispatch:
10+
inputs:
11+
table:
12+
description: 'Specific table number (optional)'
13+
required: false
14+
type: string
15+
dry_run:
16+
description: 'Dry run (discovery only)'
17+
required: false
18+
type: boolean
19+
default: false
20+
21+
env:
22+
NODE_VERSION: '20'
23+
R_VERSION: '4.3'
24+
25+
jobs:
26+
discover-and-fetch:
27+
runs-on: ubuntu-latest
28+
outputs:
29+
selected_table: ${{ steps.discover.outputs.table }}
30+
has_update: ${{ steps.discover.outputs.has_update }}
31+
32+
steps:
33+
- name: Checkout repository
34+
uses: actions/checkout@v4
35+
36+
- name: Setup R
37+
uses: r-lib/actions/setup-r@v2
38+
with:
39+
r-version: ${{ env.R_VERSION }}
40+
41+
- name: Install R dependencies
42+
run: |
43+
install.packages(c("cansim", "dplyr", "tidyr", "jsonlite"), repos = "https://cloud.r-project.org")
44+
shell: Rscript {0}
45+
46+
- name: Run discovery
47+
id: discover
48+
run: |
49+
mkdir -p output
50+
51+
if [ -n "${{ github.event.inputs.table }}" ]; then
52+
echo "Using specified table: ${{ github.event.inputs.table }}"
53+
echo "table=${{ github.event.inputs.table }}" >> $GITHUB_OUTPUT
54+
echo "has_update=true" >> $GITHUB_OUTPUT
55+
else
56+
echo "Running topic discovery..."
57+
Rscript r-tools/discover_topics.R --configured --json --output=output
58+
59+
if [ -f output/discovery_results.json ]; then
60+
TABLE=$(python3 -c "
61+
import json
62+
with open('output/discovery_results.json') as f:
63+
data = json.load(f)
64+
if data.get('recommendation'):
65+
print(data['recommendation']['table_number'])
66+
")
67+
if [ -n "$TABLE" ]; then
68+
echo "table=$TABLE" >> $GITHUB_OUTPUT
69+
echo "has_update=true" >> $GITHUB_OUTPUT
70+
echo "Recommended table: $TABLE"
71+
else
72+
echo "has_update=false" >> $GITHUB_OUTPUT
73+
echo "No newsworthy updates found"
74+
fi
75+
else
76+
echo "has_update=false" >> $GITHUB_OUTPUT
77+
fi
78+
fi
79+
80+
- name: Fetch table data
81+
if: steps.discover.outputs.has_update == 'true' && github.event.inputs.dry_run != 'true'
82+
run: |
83+
TABLE="${{ steps.discover.outputs.table }}"
84+
echo "Fetching data for table: $TABLE"
85+
Rscript r-tools/fetch_table.R "$TABLE" output
86+
87+
- name: Upload data artifact
88+
if: steps.discover.outputs.has_update == 'true' && github.event.inputs.dry_run != 'true'
89+
uses: actions/upload-artifact@v4
90+
with:
91+
name: cansim-data
92+
path: output/
93+
retention-days: 7
94+
95+
create-issue:
96+
needs: discover-and-fetch
97+
if: needs.discover-and-fetch.outputs.has_update == 'true' && github.event.inputs.dry_run != 'true'
98+
runs-on: ubuntu-latest
99+
100+
steps:
101+
- name: Checkout repository
102+
uses: actions/checkout@v4
103+
104+
- name: Download data artifact
105+
uses: actions/download-artifact@v4
106+
with:
107+
name: cansim-data
108+
path: output/
109+
110+
- name: Get table info
111+
id: table_info
112+
run: |
113+
TABLE="${{ needs.discover-and-fetch.outputs.selected_table }}"
114+
115+
# Extract title from discovery results
116+
if [ -f output/discovery_results.json ]; then
117+
TITLE=$(python3 -c "
118+
import json
119+
with open('output/discovery_results.json') as f:
120+
data = json.load(f)
121+
if data.get('recommendation'):
122+
print(data['recommendation'].get('title', 'Unknown'))
123+
")
124+
else
125+
TITLE="Unknown"
126+
fi
127+
128+
echo "table=$TABLE" >> $GITHUB_OUTPUT
129+
echo "title=$TITLE" >> $GITHUB_OUTPUT
130+
131+
- name: Create GitHub Issue for manual generation
132+
uses: actions/github-script@v7
133+
with:
134+
script: |
135+
const table = '${{ steps.table_info.outputs.table }}';
136+
const title = '${{ steps.table_info.outputs.title }}';
137+
const date = new Date().toISOString().split('T')[0];
138+
139+
const issueTitle = `[D-AI-LY] Generate article for ${table} - ${date}`;
140+
141+
const issueBody = `## The D-AI-LY - Article Generation Required
142+
143+
**Date:** ${date}
144+
**Table:** ${table}
145+
**Title:** ${title}
146+
147+
### Data Ready
148+
149+
The discovery and data fetch steps completed successfully. The data artifact is attached to the workflow run.
150+
151+
### To Generate the Article
152+
153+
Run Claude Code locally with:
154+
155+
\`\`\`bash
156+
# Navigate to the project
157+
cd ~/Projects/the-daily
158+
159+
# Pull latest changes (includes data files)
160+
git pull
161+
162+
# Run the generator
163+
claude "/the-daily-generator ${table}"
164+
165+
# Or run the full pipeline
166+
./automation/run_pipeline.sh --table=${table}
167+
\`\`\`
168+
169+
### Why Manual?
170+
171+
This workflow creates an issue instead of generating automatically because:
172+
- Article generation uses Claude Code via Max subscription (no API costs)
173+
- Generation runs locally on your machine
174+
175+
### Workflow Run
176+
177+
[View workflow run](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})
178+
179+
---
180+
*This issue was automatically created by the D-AI-LY pipeline.*
181+
`;
182+
183+
await github.rest.issues.create({
184+
owner: context.repo.owner,
185+
repo: context.repo.repo,
186+
title: issueTitle,
187+
body: issueBody,
188+
labels: ['d-ai-ly', 'automation']
189+
});
190+
191+
notify:
192+
needs: [discover-and-fetch, create-issue]
193+
if: always()
194+
runs-on: ubuntu-latest
195+
196+
steps:
197+
- name: Summary
198+
run: |
199+
echo "## D-AI-LY Pipeline Summary" >> $GITHUB_STEP_SUMMARY
200+
echo "" >> $GITHUB_STEP_SUMMARY
201+
echo "- **Date**: $(date '+%Y-%m-%d %H:%M:%S UTC')" >> $GITHUB_STEP_SUMMARY
202+
echo "- **Selected Table**: ${{ needs.discover-and-fetch.outputs.selected_table || 'None' }}" >> $GITHUB_STEP_SUMMARY
203+
echo "- **Has Update**: ${{ needs.discover-and-fetch.outputs.has_update }}" >> $GITHUB_STEP_SUMMARY
204+
echo "" >> $GITHUB_STEP_SUMMARY
205+
206+
if [ "${{ needs.create-issue.result }}" == "success" ]; then
207+
echo "✅ Data fetched and issue created for manual generation." >> $GITHUB_STEP_SUMMARY
208+
echo "" >> $GITHUB_STEP_SUMMARY
209+
echo "Check GitHub Issues for generation instructions." >> $GITHUB_STEP_SUMMARY
210+
elif [ "${{ needs.discover-and-fetch.outputs.has_update }}" == "false" ]; then
211+
echo "ℹ️ No newsworthy updates found today." >> $GITHUB_STEP_SUMMARY
212+
else
213+
echo "⚠️ Pipeline completed with status: ${{ needs.create-issue.result }}" >> $GITHUB_STEP_SUMMARY
214+
fi

0 commit comments

Comments
 (0)