Skip to content

Commit 15a11ab

Browse files
saquibb8saquibb8
committed
Add aggregator Lambda code and deploy workflow
1 parent 2ae4f26 commit 15a11ab

7 files changed

Lines changed: 327 additions & 3 deletions

File tree

‎.github/workflows/deploy-lambda.yml‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
name: Deploy Lambda Function
2+
3+
# WHEN: Only runs when you manually trigger it from the Actions tab.
4+
# Nobody can trigger this by pushing code or opening a PR.
5+
on:
6+
workflow_dispatch:
7+
inputs:
8+
source_folder:
9+
description: 'Source folder name under data-engineering/src/ (e.g., saayam-org-aggregator)'
10+
required: true
11+
type: string
12+
lambda_function_name:
13+
description: 'Exact AWS Lambda function name (e.g., saayam-org-aggregator)'
14+
required: true
15+
type: string
16+
17+
jobs:
18+
deploy:
19+
runs-on: ubuntu-latest
20+
21+
steps:
22+
23+
# STEP 1: Security gate — only YOUR username gets through.
24+
- name: Check if authorized to deploy
25+
run: |
26+
if [ "${{ github.actor }}" != "saquibb8" ]; then
27+
echo "❌ Unauthorized. Only saquibb8 can deploy."
28+
exit 1
29+
fi
30+
echo "✅ Authorized: ${{ github.actor }}"
31+
32+
# STEP 2: Pull the latest code from the repo.
33+
- name: Checkout code
34+
uses: actions/checkout@v4
35+
with:
36+
ref: main # Always deploy from main branch
37+
38+
# STEP 3: Set up Python (matches your repo's Python 3.10+ requirement).
39+
- name: Set up Python
40+
uses: actions/setup-python@v5
41+
with:
42+
python-version: '3.10'
43+
44+
# STEP 4: Build the deployment package.
45+
# This installs dependencies + copies source code into one folder.
46+
- name: Build deployment package
47+
run: |
48+
mkdir package
49+
50+
# Install Lambda-specific dependencies (if requirements.txt exists)
51+
if [ -f data-engineering/src/${{ inputs.source_folder }}/requirements.txt ]; then
52+
echo "📦 Installing dependencies..."
53+
pip install -r data-engineering/src/${{ inputs.source_folder }}/requirements.txt -t package/ --quiet
54+
fi
55+
56+
# Copy the Lambda's source code
57+
echo "📂 Copying data-engineering/src/${{ inputs.source_folder }}/ ..."
58+
cp -r data-engineering/src/${{ inputs.source_folder }}/* package/
59+
60+
# Copy shared utilities (so imports like "from utils.db_client import ..." work)
61+
if [ -d data-engineering/src/utils ]; then
62+
echo "📂 Copying data-engineering/src/utils/ ..."
63+
cp -r data-engineering/src/utils package/
64+
fi
65+
66+
# Copy shared models (so imports like "from models.fraud_requests import ..." work)
67+
if [ -d data-engineering/src/models ]; then
68+
echo "📂 Copying data-engineering/src/models/ ..."
69+
cp -r data-engineering/src/models package/
70+
fi
71+
72+
# STEP 5: Zip everything up.
73+
- name: Create zip file
74+
run: |
75+
cd package
76+
zip -r ../deployment.zip . --quiet
77+
cd ..
78+
echo "📦 Package size: $(du -h deployment.zip | cut -f1)"
79+
80+
# STEP 6: Set up AWS credentials from your GitHub Secrets.
81+
- name: Configure AWS credentials
82+
uses: aws-actions/configure-aws-credentials@v4
83+
with:
84+
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
85+
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
86+
aws-region: ${{ secrets.AWS_REGION }}
87+
88+
# STEP 7: Deploy to Lambda.
89+
- name: Deploy to AWS Lambda
90+
run: |
91+
echo "🚀 Deploying to: ${{ inputs.lambda_function_name }}"
92+
93+
aws lambda update-function-code \
94+
--function-name "${{ inputs.lambda_function_name }}" \
95+
--zip-file fileb://deployment.zip \
96+
--publish \
97+
--output table
98+
99+
echo ""
100+
echo "✅ ${{ inputs.lambda_function_name }} deployed successfully!"

‎data-engineering/deploy-lambda.yml‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
name: Deploy Lambda Function
2+
3+
# WHEN: Only runs when you manually trigger it from the Actions tab.
4+
# Nobody can trigger this by pushing code or opening a PR.
5+
on:
6+
workflow_dispatch:
7+
inputs:
8+
source_folder:
9+
description: 'Source folder name under src/ (e.g., aggregator)'
10+
required: true
11+
type: string
12+
lambda_function_name:
13+
description: 'Exact AWS Lambda function name (e.g., saayam-org-aggregator)'
14+
required: true
15+
type: string
16+
17+
jobs:
18+
deploy:
19+
runs-on: ubuntu-latest
20+
21+
steps:
22+
23+
# STEP 1: Security gate — only YOUR username gets through.
24+
# Replace YOUR_GITHUB_USERNAME below with your actual GitHub username.
25+
- name: Check if authorized to deploy
26+
run: |
27+
if [ "${{ github.actor }}" != "saquibb8" ]; then
28+
echo "❌ Unauthorized. Only saquibb8 can deploy."
29+
exit 1
30+
fi
31+
echo "✅ Authorized: ${{ github.actor }}"
32+
33+
# STEP 2: Pull the latest code from the repo.
34+
- name: Checkout code
35+
uses: actions/checkout@v4
36+
with:
37+
ref: main # Always deploy from main branch
38+
39+
# STEP 3: Set up Python (matches your repo's Python 3.10+ requirement).
40+
- name: Set up Python
41+
uses: actions/setup-python@v5
42+
with:
43+
python-version: '3.10'
44+
45+
# STEP 4: Build the deployment package.
46+
# This installs dependencies + copies source code into one folder.
47+
- name: Build deployment package
48+
run: |
49+
mkdir package
50+
51+
# Install Lambda-specific dependencies (if requirements.txt exists)
52+
if [ -f src/${{ inputs.source_folder }}/requirements.txt ]; then
53+
echo "📦 Installing dependencies..."
54+
pip install -r src/${{ inputs.source_folder }}/requirements.txt -t package/ --quiet
55+
fi
56+
57+
# Copy the Lambda's source code
58+
echo "📂 Copying src/${{ inputs.source_folder }}/ ..."
59+
cp -r src/${{ inputs.source_folder }}/* package/
60+
61+
# Copy shared utilities (so imports like "from utils.db_client import ..." work)
62+
if [ -d src/utils ]; then
63+
echo "📂 Copying src/utils/ ..."
64+
cp -r src/utils package/
65+
fi
66+
67+
# Copy shared models (so imports like "from models.fraud_requests import ..." work)
68+
if [ -d src/models ]; then
69+
echo "📂 Copying src/models/ ..."
70+
cp -r src/models package/
71+
fi
72+
73+
# STEP 5: Zip everything up.
74+
- name: Create zip file
75+
run: |
76+
cd package
77+
zip -r ../deployment.zip . --quiet
78+
cd ..
79+
echo "📦 Package size: $(du -h deployment.zip | cut -f1)"
80+
81+
# STEP 6: Set up AWS credentials from your GitHub Secrets.
82+
- name: Configure AWS credentials
83+
uses: aws-actions/configure-aws-credentials@v4
84+
with:
85+
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
86+
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
87+
aws-region: ${{ secrets.AWS_REGION }}
88+
89+
# STEP 7: Deploy to Lambda.
90+
- name: Deploy to AWS Lambda
91+
run: |
92+
echo "🚀 Deploying to: ${{ inputs.lambda_function_name }}"
93+
94+
aws lambda update-function-code \
95+
--function-name "${{ inputs.lambda_function_name }}" \
96+
--zip-file fileb://deployment.zip \
97+
--publish \
98+
--output table
99+
100+
echo ""
101+
echo "✅ ${{ inputs.lambda_function_name }} deployed successfully!"

‎data-engineering/src/aggregator/requirements.txt‎

Lines changed: 0 additions & 3 deletions
This file was deleted.

data-engineering/src/aggregator/__init__.py renamed to data-engineering/src/saayam-org-aggregator/__init__.py

File renamed without changes.

‎data-engineering/src/saayam-org-aggregator/helpers.py‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
import subprocess
2+
import boto3
3+
import sys
4+
import json
5+
from aws_lambda_powertools.utilities import parameters
6+
import pandas as pd
7+
8+
subprocess.call([sys.executable, "-m", "pip", "install", "pg8000", "-t", "/tmp/"])
9+
sys.path.insert(0, "/tmp/")
10+
import pg8000
11+
12+
GEN_AI_LAMBDA = "More_Org_GenAI_Py_v3126"
13+
14+
def get_orgs_from_db(location, category):
15+
try:
16+
creds = json.loads(parameters.get_parameter(
17+
'/dev/saayam/db/Virginia/Analytics/user',
18+
decrypt=True,
19+
max_age=3600
20+
))
21+
database = creds['DATABASE NAME']
22+
23+
conn = pg8000.connect(
24+
host=creds['HOST'],
25+
user=creds['USERNAME'],
26+
password=creds['PASSWORD'],
27+
database=database,
28+
port=creds['PORT'],
29+
ssl_context=True
30+
)
31+
32+
df = pd.read_sql(
33+
f"SELECT * FROM {database}.organizations WHERE mission = '{category}' AND city_name = '{location}'",
34+
conn
35+
)
36+
conn.close()
37+
return df
38+
39+
except parameters.GetParameterError as e:
40+
raise Exception(f'Failed to retrieve DB credentials: {str(e)}')
41+
except pg8000.DatabaseError as e:
42+
raise Exception(f'Database error: {str(e)}')
43+
except Exception as e:
44+
raise Exception(f'Error fetching from DB: {str(e)}')
45+
46+
47+
def get_ai_orgs(subject, description, location):
48+
try:
49+
response = boto3.client('lambda').invoke(
50+
FunctionName=GEN_AI_LAMBDA,
51+
InvocationType='RequestResponse',
52+
Payload=json.dumps({
53+
"subject": subject,
54+
"description": description,
55+
"location": location
56+
})
57+
)
58+
59+
payload = json.loads(response['Payload'].read())
60+
61+
if payload.get('statusCode') != 200:
62+
raise Exception(f'GenAI Lambda returned error: {payload}')
63+
64+
return pd.DataFrame(payload['body']['organizations'])
65+
66+
except boto3.exceptions.Boto3Error as e:
67+
raise Exception(f'Failed to invoke GenAI Lambda: {str(e)}')
68+
except (KeyError, TypeError) as e:
69+
raise Exception(f'Unexpected response structure from GenAI Lambda: {str(e)}')
70+
except Exception as e:
71+
raise Exception(f'Error fetching AI orgs: {str(e)}')
72+
73+
74+
def merge_organizations(db_organizations, genAI_organizations):
75+
try:
76+
db_organizations = db_organizations.rename(columns={
77+
'org_name': 'name',
78+
'city_name': 'location',
79+
'phone': 'contact'
80+
})[['name', 'location', 'contact', 'email', 'web_url', 'mission', 'source']]
81+
82+
genAI_organizations = genAI_organizations.rename(columns={
83+
'organization_name': 'name'
84+
})[['name', 'location', 'contact', 'email', 'web_url', 'mission', 'source']]
85+
86+
return pd.concat([db_organizations, genAI_organizations], ignore_index=True)
87+
88+
except KeyError as e:
89+
raise Exception(f'Missing expected column during merge: {str(e)}')
90+
except Exception as e:
91+
raise Exception(f'Error merging organizations: {str(e)}')

‎data-engineering/src/saayam-org-aggregator/lambda_function.py‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
import json
2+
from helpers import get_ai_orgs, get_orgs_from_db, merge_organizations
3+
4+
def lambda_handler(event, context):
5+
try:
6+
raw_body = event.get("body")
7+
body = json.loads(raw_body) if isinstance(raw_body, str) else event
8+
9+
subject = body.get("subject")
10+
description = body.get("description")
11+
location = body.get("location")
12+
category = body.get("category")
13+
14+
if not location or not category:
15+
return {
16+
'statusCode': 400,
17+
'body': json.dumps({'error': 'location and category are required fields'})
18+
}
19+
20+
db_organizations = get_orgs_from_db(location, category)
21+
genAI_organizations = get_ai_orgs(subject, description, location)
22+
combined_list = merge_organizations(db_organizations, genAI_organizations)
23+
24+
return {
25+
'statusCode': 200,
26+
'body': combined_list.to_dict(orient='records')
27+
}
28+
29+
except json.JSONDecodeError as e:
30+
return {'statusCode': 400, 'body': json.dumps({'error': f'Invalid JSON in request body: {str(e)}'})}
31+
except Exception as e:
32+
return {'statusCode': 500, 'body': json.dumps({'error': f'Internal server error: {str(e)}'})}

‎data-engineering/src/saayam-org-aggregator/requirements.txt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
aws-lambda-powertools>=2.0.0
2+
pandas>=2.0.0
3+
pg8000>=1.30.0

0 commit comments

Comments
 (0)