Skip to content

Commit ea91d63

Browse files
pratik-bhatt-nvpratik-bhatt-nv
authored
Add CI Workflow (#15)
* add workflow * add workflow * add workflow * update notebook * update runner
1 parent 093310a commit ea91d63

2 files changed

Lines changed: 170 additions & 16 deletions

File tree

‎.github/workflows/ci.yaml‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
name: Jupyter Notebook Runner
2+
3+
on:
4+
push:
5+
branches:
6+
- main
7+
workflow_dispatch:
8+
9+
concurrency:
10+
group: ${{ github.workflow }}-${{ github.ref }}
11+
cancel-in-progress: true
12+
13+
jobs:
14+
run-notebook:
15+
runs-on: arc-runner-set-oke-org-nv-ai-bp
16+
env:
17+
NOTEBOOK_PATH: ./launchable/PDFtoPodcast.ipynb
18+
PYTHON_VERSION: 3.12
19+
steps:
20+
- name: Checkout repository
21+
uses: actions/checkout@v3
22+
23+
- name: Set up Python
24+
uses: actions/setup-python@v4
25+
with:
26+
python-version: ${{ env.PYTHON_VERSION }}
27+
cache: 'pip'
28+
cache-dependency-path: |
29+
requirements.txt
30+
**/*.ipynb
31+
32+
- name: Install uv
33+
run: |
34+
# Check if uv is installed, if not, install it
35+
if ! command -v uv &> /dev/null; then
36+
echo "Installing uv..."
37+
curl -LsSf https://astral.sh/uv/install.sh | sh
38+
fi
39+
40+
# Add ~/.local/bin to PATH to make `uv` accessible
41+
echo "PATH=$PATH:$HOME/.local/bin" >> $GITHUB_ENV
42+
43+
- name: Install dependencies
44+
run: |
45+
python -m pip install --upgrade pip
46+
pip install papermill jupyter
47+
# Install Docker and Docker Compose in a single step
48+
curl -fsSL https://get.docker.com -o get-docker.sh
49+
sudo sh get-docker.sh
50+
sudo apt-get update
51+
sudo apt-get install -y docker-compose-plugin docker-compose build-essential
52+
53+
- name: Run Jupyter Notebook
54+
env:
55+
${{ secrets }}
56+
run: |
57+
# Verify required environment variables
58+
required_vars=("ELEVENLABS_API_KEY" "NVIDIA_API_KEY")
59+
for var in "${required_vars[@]}"; do
60+
if [ -z "${!var}" ]; then
61+
echo "Error: $var is not set"
62+
exit 1
63+
fi
64+
done
65+
66+
OUTPUT_NOTEBOOK="result.ipynb"
67+
echo "Executing notebook: $NOTEBOOK_PATH"
68+
papermill "$NOTEBOOK_PATH" "$OUTPUT_NOTEBOOK" --log-output --log-level DEBUG
69+
70+
- name: Convert result to html format
71+
if: always()
72+
run: |
73+
OUTPUT_NOTEBOOK="result.ipynb"
74+
jupyter nbconvert --to html "$OUTPUT_NOTEBOOK"
75+
76+
- name: Upload the result notebook as artifact
77+
if: always()
78+
uses: actions/upload-artifact@v4
79+
with:
80+
name: result-notebook
81+
path: "result.html"
82+
retention-days: 30
83+

‎launchable/PDFtoPodcast.ipynb‎

Lines changed: 87 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -112,11 +112,11 @@
112112
"Required variables:\n",
113113
"- `ELEVENLABS_API_KEY`: Your ElevenLabs API key\n",
114114
"- `NVIDIA_API_KEY`: Your NVIDIA API key\n",
115-
"- `MAX_CONCURRENT_REQUESTS`: Number of concurrent requests allowed (recommended: 1 for local development)\n",
115+
"- `MAX_CONCURRENT_REQUESTS` = Number of concurrent requests allowed (recommended: 1 for local development)\n",
116116
"\n",
117117
"> **Note**: While production environments use the NVIDIA Eleven Labs API key which supports concurrent requests, for local development we recommend setting `MAX_CONCURRENT_REQUESTS=1` to avoid rate limiting issues. You can obtain a free testing API key from [ElevenLabs](https://elevenlabs.io).\n",
118118
"\n",
119-
"Run the code cell below to create your `.env` file. Make sure to replace the placeholder values with your actual API keys."
119+
"Run the code cell below to create your `.env` file. Make sure to update variables `ELEVENLABS_API_KEY` and `NVIDIA_API_KEY`. "
120120
]
121121
},
122122
{
@@ -126,7 +126,7 @@
126126
"metadata": {},
127127
"outputs": [],
128128
"source": [
129-
"%%bash\n",
129+
"%%bash \n",
130130
"\n",
131131
"cd pdf-to-podcast/\n",
132132
"\n",
@@ -138,15 +138,13 @@
138138
"\n",
139139
"# Create new .env file\n",
140140
"cat > .env << EOL\n",
141-
"ELEVENLABS_API_KEY=<ENTER-KEY>\n",
142-
"NVIDIA_API_KEY=<ENTER_KEY>\n",
141+
"ELEVENLABS_API_KEY=$ELEVENLABS_API_KEY\n",
142+
"NVIDIA_API_KEY=$NVIDIA_API_KEY\n",
143143
"MAX_CONCURRENT_REQUESTS=1\n",
144144
"EOL\n",
145145
"\n",
146146
"echo \"Created .env file. Please edit it with your actual API keys.\"\n",
147-
"echo -e \"\\nCurrent .env contents:\"\n",
148-
"echo \"----------------------------------------\"\n",
149-
"cat .env"
147+
"echo \"----------------------------------------\""
150148
]
151149
},
152150
{
@@ -171,7 +169,6 @@
171169
"%%bash\n",
172170
"\n",
173171
"cd pdf-to-podcast/\n",
174-
"\n",
175172
"make uv"
176173
]
177174
},
@@ -188,7 +185,51 @@
188185
"```\n",
189186
"cd pdf-to-podcast/\n",
190187
"make all-services\n",
191-
"```"
188+
"```\n",
189+
"Or Run below command for starting services in detach mode"
190+
]
191+
},
192+
{
193+
"cell_type": "code",
194+
"execution_count": null,
195+
"id": "44fb2da5",
196+
"metadata": {},
197+
"outputs": [],
198+
"source": [
199+
"%%bash\n",
200+
"\n",
201+
"cd pdf-to-podcast/\n",
202+
"make all-services DETACH=1\n",
203+
"\n",
204+
"wait_for_services() {\n",
205+
" local max_wait_time=$1 # Max wait time in seconds\n",
206+
" local check_interval=$2 # Interval to check in seconds\n",
207+
"\n",
208+
" local elapsed_time=0\n",
209+
"\n",
210+
" while [ $elapsed_time -lt $max_wait_time ]; do\n",
211+
" # Check if any containers are up\n",
212+
" if docker-compose ps | grep -q \"Up\"; then\n",
213+
" echo \"Services are up and running!\"\n",
214+
" return 0 # Return success if services are up\n",
215+
" fi\n",
216+
" # Wait for the next check interval\n",
217+
" sleep $check_interval\n",
218+
" elapsed_time=$((elapsed_time + check_interval))\n",
219+
" echo \"Waiting for services to come up... $((elapsed_time / 60)) min elapsed\"\n",
220+
" done\n",
221+
"\n",
222+
" # If max wait time is reached and services are not up, print a timeout message\n",
223+
" echo \"Timeout reached. Services did not come up in $((max_wait_time / 60)) minutes.\"\n",
224+
" return 1 # Return failure if services did not come up in time\n",
225+
"}\n",
226+
"\n",
227+
"# Maximum wait time in seconds (15 minutes)\n",
228+
"MAX_WAIT_TIME=900 # 15 minutes\n",
229+
"CHECK_INTERVAL=10 # Check every 10 seconds\n",
230+
"\n",
231+
"# Call the function\n",
232+
"wait_for_services $MAX_WAIT_TIME $CHECK_INTERVAL"
192233
]
193234
},
194235
{
@@ -198,7 +239,8 @@
198239
"metadata": {},
199240
"outputs": [],
200241
"source": [
201-
"!docker ps --format \"table {{.ID}}\\t{{.Names}}\\t{{.Status}}\""
242+
"%%bash\n",
243+
"docker ps --format \"table {{.ID}}\\t{{.Names}}\\t{{.Status}}\""
202244
]
203245
},
204246
{
@@ -237,7 +279,27 @@
237279
"metadata": {},
238280
"outputs": [],
239281
"source": [
240-
"!curl localhost:8002/health"
282+
"import requests\n",
283+
"\n",
284+
"def make_request(url, params=None, headers=None):\n",
285+
"\n",
286+
" try:\n",
287+
" # Send GET request with optional parameters and headers\n",
288+
" response = requests.get(url, params=params, headers=headers)\n",
289+
" \n",
290+
" # Check if the request was successful (status code 200)\n",
291+
" if response.status_code != 200:\n",
292+
" raise Exception(f\"Request failed with status code {response.status_code}. Response: {response.text}\")\n",
293+
" \n",
294+
" # Return the response object if successful\n",
295+
" return response\n",
296+
" \n",
297+
" except requests.exceptions.RequestException as e:\n",
298+
" print(f\"Error occurred during the request: {e}\")\n",
299+
" raise Exception(f\"Request failed: {e}\")\n",
300+
"\n",
301+
"response = make_request(\"http://localhost:8002/health\")\n",
302+
"print(response.text)"
241303
]
242304
},
243305
{
@@ -475,7 +537,13 @@
475537
"metadata": {},
476538
"outputs": [],
477539
"source": [
478-
"!curl \"localhost:8002/output/{job_id}?userId=test-userid\" --output temp_audio.mp3\n",
540+
"response = make_request(f\"http://localhost:8002/output/{job_id}?userId=test-userid\")\n",
541+
"if response.content:\n",
542+
" with open(\"temp_audio.mp3\", 'wb') as f:\n",
543+
" f.write(response.content)\n",
544+
" print(\"MP3 file saved successfully as temp_audio.mp3\")\n",
545+
"else:\n",
546+
" print(\"No content received in the response.\")\n",
479547
"Audio(\"temp_audio.mp3\")"
480548
]
481549
},
@@ -508,7 +576,8 @@
508576
"metadata": {},
509577
"outputs": [],
510578
"source": [
511-
"!curl \"localhost:8002/saved_podcast/{job_id}/transcript?userId=test-userid\""
579+
"response = make_request(f\"http://localhost:8002/saved_podcast/{job_id}/transcript?userId=test-userid\")\n",
580+
"print(response.text)"
512581
]
513582
},
514583
{
@@ -544,7 +613,8 @@
544613
},
545614
"outputs": [],
546615
"source": [
547-
"!curl \"localhost:8002/saved_podcast/{job_id}/history?userId=test-userid\""
616+
"response = make_request(f\"http://localhost:8002/saved_podcast/{job_id}/history?userId=test-userid\")\n",
617+
"print(response.text)\n"
548618
]
549619
},
550620
{
@@ -554,7 +624,8 @@
554624
"metadata": {},
555625
"outputs": [],
556626
"source": [
557-
"!curl \"localhost:8002/saved_podcast/{job_id}/metadata?userId=test-userid\""
627+
"response = make_request(f\"http://localhost:8002/saved_podcast/{job_id}/metadata?userId=test-userid\")\n",
628+
"print(response.text)\n"
558629
]
559630
},
560631
{

0 commit comments

Comments
 (0)