Skip to content

Commit bd45a56

Browse files
dineshpintodineshpinto
authored
fix: audit and fix example scripts for issue #109 (#115)
2 parents adb13ff + 47080e6 commit bd45a56

16 files changed

+1161
-53
lines changed

‎.github/workflows/docker-scripts.yml‎

Lines changed: 231 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,231 @@
1+
name: Docker Scripts Validation
2+
3+
on:
4+
push:
5+
branches: ["main"]
6+
paths:
7+
- "Dockerfile"
8+
- "scripts/**"
9+
- "pyproject.toml"
10+
- "uv.lock"
11+
pull_request:
12+
branches: ["main"]
13+
paths:
14+
- "Dockerfile"
15+
- "scripts/**"
16+
- "pyproject.toml"
17+
- "uv.lock"
18+
19+
permissions:
20+
contents: read
21+
22+
jobs:
23+
validate-docker-scripts:
24+
runs-on: ubuntu-latest
25+
26+
strategy:
27+
matrix:
28+
script:
29+
- name: "PDF Ingestion"
30+
extras: "pdf"
31+
script_file: "ingest_pdf.py"
32+
test_timeout: "300" # 5 minutes
33+
34+
env:
35+
# Test environment variables
36+
LOG_LEVEL: INFO
37+
AGENT__GEMINI_MODEL: "gemini-2.0-flash"
38+
AGENT__GEMINI_API_KEY: ${{ secrets.AGENT__GEMINI_API_KEY }}
39+
ECOSYSTEM__WEB3_PROVIDER_URL: "https://stylish-light-theorem.flare-mainnet.quiknode.pro/ext/bc/C/rpc"
40+
INGESTION__CHUNK_SIZE: 5000
41+
TEE__SIMULATE_ATTESTATION_TOKEN: true
42+
43+
steps:
44+
- name: Checkout repository
45+
uses: actions/checkout@v5
46+
47+
- name: Set up Docker Buildx
48+
uses: docker/setup-buildx-action@v3
49+
50+
- name: Build Docker image for ${{ matrix.script.name }}
51+
run: |
52+
docker build \
53+
--build-arg EXTRAS=${{ matrix.script.extras }} \
54+
--build-arg SCRIPT=${{ matrix.script.script_file }} \
55+
--tag fai-script-${{ matrix.script.extras }} \
56+
--cache-from type=gha \
57+
--cache-to type=gha,mode=max \
58+
.
59+
60+
- name: Validate script exists in image
61+
run: |
62+
docker run --rm fai-script-${{ matrix.script.extras }} \
63+
test -f "/app/scripts/${{ matrix.script.script_file }}"
64+
65+
- name: Test script startup (dry run)
66+
timeout-minutes: 5
67+
run: |
68+
# Simple validation that the script exists and dependencies are available
69+
docker run --rm \
70+
-e LOG_LEVEL="$LOG_LEVEL" \
71+
-e AGENT__GEMINI_MODEL="$AGENT__GEMINI_MODEL" \
72+
-e AGENT__GEMINI_API_KEY="$AGENT__GEMINI_API_KEY" \
73+
-e ECOSYSTEM__WEB3_PROVIDER_URL="$ECOSYSTEM__WEB3_PROVIDER_URL" \
74+
-e INGESTION__CHUNK_SIZE="$INGESTION__CHUNK_SIZE" \
75+
-e TEE__SIMULATE_ATTESTATION_TOKEN="$TEE__SIMULATE_ATTESTATION_TOKEN" \
76+
fai-script-${{ matrix.script.extras }} \
77+
python -c "
78+
import sys
79+
import os
80+
81+
# Test that script file exists
82+
script_path = '/app/scripts/${{ matrix.script.script_file }}'
83+
if not os.path.exists(script_path):
84+
print(f'❌ Script not found: {script_path}')
85+
sys.exit(1)
86+
print(f'✅ Script exists: {script_path}')
87+
88+
# Test that required dependencies are available
89+
if '${{ matrix.script.extras }}' == 'pdf':
90+
try:
91+
import PIL
92+
import fitz # pymupdf
93+
import pytesseract
94+
print('✅ PDF dependencies available')
95+
except ImportError as e:
96+
print(f'❌ PDF dependency missing: {e}')
97+
sys.exit(1)
98+
99+
print('✅ Script validation completed successfully')
100+
"
101+
102+
- name: Test container health
103+
run: |
104+
# Test that the container can start and the Python environment is healthy
105+
docker run --rm fai-script-${{ matrix.script.extras }} \
106+
python -c "
107+
import sys
108+
print(f'Python version: {sys.version}')
109+
print(f'Python path: {sys.path}')
110+
111+
# Test core dependencies (some modules may require optional deps)
112+
try:
113+
import flare_ai_kit
114+
print('✅ flare-ai-kit imported successfully')
115+
except ImportError as e:
116+
print(f'⚠️ flare-ai-kit import issue (may need more extras): {e}')
117+
# Test basic Python packages instead
118+
import httpx, pydantic, structlog
119+
print('✅ Core Python dependencies available')
120+
121+
# Test that uv environment is working
122+
import subprocess
123+
result = subprocess.run(['/app/.venv/bin/python', '--version'],
124+
capture_output=True, text=True)
125+
print(f'Virtual env Python: {result.stdout.strip()}')
126+
127+
print('✅ Container health check passed')
128+
"
129+
130+
- name: Test script dependencies for ${{ matrix.script.name }}
131+
run: |
132+
# Test that the specific extras are properly installed
133+
docker run --rm fai-script-${{ matrix.script.extras }} \
134+
python -c "
135+
import sys
136+
137+
extras = '${{ matrix.script.extras }}'
138+
print(f'Testing dependencies for extras: {extras}')
139+
140+
if 'pdf' in extras:
141+
try:
142+
import PIL
143+
import fitz
144+
import pytesseract
145+
print('✅ PDF dependencies (PIL, fitz, pytesseract) available')
146+
except ImportError as e:
147+
print(f'❌ PDF dependency missing: {e}')
148+
sys.exit(1)
149+
150+
if 'rag' in extras:
151+
try:
152+
import qdrant_client
153+
import dulwich
154+
print('✅ RAG dependencies (qdrant_client, dulwich) available')
155+
except ImportError as e:
156+
print(f'❌ RAG dependency missing: {e}')
157+
sys.exit(1)
158+
159+
if 'a2a' in extras:
160+
try:
161+
import fastapi
162+
print('✅ A2A dependencies (fastapi) available')
163+
except ImportError as e:
164+
print(f'❌ A2A dependency missing: {e}')
165+
sys.exit(1)
166+
167+
print('✅ All expected dependencies are available')
168+
"
169+
170+
171+
172+
validate-build-args:
173+
runs-on: ubuntu-latest
174+
175+
steps:
176+
- name: Checkout repository
177+
uses: actions/checkout@v5
178+
179+
- name: Set up Docker Buildx
180+
uses: docker/setup-buildx-action@v3
181+
182+
- name: Test build without extras
183+
run: |
184+
docker build \
185+
--build-arg SCRIPT=ingest_pdf.py \
186+
--tag fai-script-base \
187+
.
188+
189+
- name: Test build with multiple extras
190+
run: |
191+
docker build \
192+
--build-arg EXTRAS=pdf,rag \
193+
--build-arg SCRIPT=ingest_pdf.py \
194+
--tag fai-script-multi \
195+
.
196+
197+
- name: Validate multi-extras build
198+
run: |
199+
docker run --rm fai-script-multi \
200+
python -c "
201+
import PIL, fitz, pytesseract # PDF deps
202+
import qdrant_client, dulwich # RAG deps
203+
print('✅ Multiple extras build successful')
204+
"
205+
206+
validate-documentation:
207+
runs-on: ubuntu-latest
208+
209+
steps:
210+
- name: Checkout repository
211+
uses: actions/checkout@v5
212+
213+
- name: Check documentation exists
214+
run: |
215+
test -f docs/docker_scripts_guide.md
216+
echo "✅ Docker scripts guide exists"
217+
218+
- name: Validate README updates
219+
run: |
220+
grep -q "parametric Dockerfile" README.md
221+
grep -q "EXTRAS" README.md
222+
grep -q "docker_scripts_guide.md" README.md
223+
echo "✅ README contains Docker scripts documentation"
224+
225+
- name: Check scripts directory structure
226+
run: |
227+
test -d scripts
228+
test -f scripts/ingest_pdf.py
229+
test -d scripts/data
230+
test -f scripts/data/create_sample_invoice.py
231+
echo "✅ Scripts directory structure is correct"

‎Dockerfile‎

Lines changed: 80 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,99 @@
1+
# Parametric Dockerfile for running scripts with specific extras
2+
# Usage:
3+
# docker build -t fai-script-pdf --build-arg EXTRAS=pdf --build-arg SCRIPT=ingest_pdf.py .
4+
# docker run --rm -it -v "$PWD/data:/app/scripts/data" fai-script-pdf
5+
6+
# Build arguments for parametric behavior
7+
ARG EXTRAS=""
8+
ARG SCRIPT="ingest_pdf.py"
9+
110
# Add <builder-digest> in prod
211
FROM ghcr.io/astral-sh/uv:python3.12-bookworm-slim AS builder
12+
13+
# Pass build args to builder stage
14+
ARG EXTRAS
15+
ARG SCRIPT
16+
317
ENV UV_COMPILE_BYTECODE=1 \
418
UV_LINK_MODE=copy \
519
UV_PYTHON_DOWNLOADS=0
20+
621
WORKDIR /app
22+
23+
# Install system dependencies for PDF processing (if needed)
24+
RUN apt-get update && apt-get install -y --no-install-recommends \
25+
tesseract-ocr \
26+
tesseract-ocr-eng \
27+
poppler-utils \
28+
&& rm -rf /var/lib/apt/lists/*
29+
30+
# Copy dependency files first for better caching
31+
COPY uv.lock pyproject.toml ./
32+
33+
# Install dependencies based on EXTRAS parameter
734
RUN --mount=type=cache,target=/root/.cache/uv \
8-
--mount=type=bind,source=uv.lock,target=uv.lock \
9-
--mount=type=bind,source=pyproject.toml,target=pyproject.toml \
10-
uv sync --locked --no-install-project --all-extras --no-dev --no-editable
35+
if [ -n "$EXTRAS" ]; then \
36+
echo "Installing with extras: $EXTRAS"; \
37+
# Convert comma-separated extras to space-separated for uv
38+
EXTRAS_ARGS=$(echo "$EXTRAS" | sed 's/,/ --extra /g'); \
39+
echo "Installing extras: $EXTRAS_ARGS"; \
40+
uv sync --locked --no-install-project --extra $EXTRAS_ARGS --no-dev --no-editable; \
41+
else \
42+
echo "Installing base dependencies only"; \
43+
uv sync --locked --no-install-project --no-dev --no-editable; \
44+
fi
45+
46+
# Copy the entire project
1147
COPY . /app
48+
49+
# Install the project itself
1250
RUN --mount=type=cache,target=/root/.cache/uv \
13-
uv sync --locked --all-extras --no-dev --no-editable
51+
if [ -n "$EXTRAS" ]; then \
52+
# Convert comma-separated extras to space-separated for uv
53+
EXTRAS_ARGS=$(echo "$EXTRAS" | sed 's/,/ --extra /g'); \
54+
echo "Installing project with extras: $EXTRAS_ARGS"; \
55+
uv sync --locked --extra $EXTRAS_ARGS --no-dev --no-editable; \
56+
else \
57+
uv sync --locked --no-dev --no-editable; \
58+
fi
59+
60+
# Clean up cache
1461
RUN rm -rf /root/.cache/uv /root/.cache/pip
1562

1663
# Add <runtime-digest> in prod
1764
FROM python:3.12-slim-bookworm AS runtime
1865

66+
# Pass build args to runtime stage
67+
ARG EXTRAS
68+
ARG SCRIPT
69+
70+
# Install runtime system dependencies for PDF processing (if needed)
71+
RUN apt-get update && apt-get install -y --no-install-recommends \
72+
tesseract-ocr \
73+
tesseract-ocr-eng \
74+
poppler-utils \
75+
&& rm -rf /var/lib/apt/lists/*
76+
1977
ENV PIP_NO_CACHE_DIR=1 \
20-
UV_PYTHON_DOWNLOADS=0
78+
UV_PYTHON_DOWNLOADS=0 \
79+
SCRIPT_NAME="$SCRIPT"
80+
81+
# Create non-root user
2182
RUN groupadd -r app && \
2283
useradd -r -g app -d /nonexistent -s /usr/sbin/nologin app
23-
USER app
24-
WORKDIR /app
84+
85+
# Copy built application from builder stage
2586
COPY --from=builder --chown=app:app /app /app
87+
88+
# Set working directory and PATH
89+
WORKDIR /app
2690
ENV PATH="/app/.venv/bin:$PATH"
27-
CMD ["/app/.venv/bin/flare-ai-kit"]
91+
92+
# Switch to non-root user
93+
USER app
94+
95+
# Validate that the script exists
96+
RUN test -f "/app/scripts/$SCRIPT" || (echo "Error: Script /app/scripts/$SCRIPT not found" && exit 1)
97+
98+
# Default command runs the specified script
99+
CMD ["sh", "-c", "cd /app/scripts && python \"$SCRIPT_NAME\""]

0 commit comments

Comments
 (0)