Skip to content

Commit 1303dff

Browse files
chris0765chris0765
authored
Merge pull request #600 from ddps-lab/main
pull main to azure-collector branch
2 parents e8564fc + 5871fdf commit 1303dff

239 files changed

Lines changed: 46306 additions & 1969 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/aws-lambda-sync.yml‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
name: deploy AWS files to lambda
2+
on:
3+
push:
4+
branches:
5+
- "main"
6+
paths:
7+
- "collector/spot-dataset/aws/lambda/**"
8+
- "utility/slack_msg_sender.py"
9+
workflow_dispatch:
10+
11+
env:
12+
AWS_ACCESS_KEY_ID: ${{ secrets.SPOTRANK_ACCESS_KEY_ID }}
13+
AWS_SECRET_ACCESS_KEY: ${{ secrets.SPOTRANK_SECRET_ACCESS_KEY }}
14+
AWS_DEFAULT_REGION: "us-west-2"
15+
16+
jobs:
17+
deploy_source:
18+
name: deploy lambda from source
19+
runs-on: ubuntu-latest
20+
steps:
21+
- name: Checkout source code
22+
uses: actions/checkout@v4
23+
24+
- name: Zip lambda function code
25+
run: |
26+
set -e
27+
28+
declare -A LAMBDA_PATHS=(
29+
["ondemand_price_collector_lambda.zip"]="collector/spot-dataset/aws/lambda/ondemand_price"
30+
["spotlake_post_processing_lambda.zip"]="collector/spot-dataset/aws/lambda/post_processing_data"
31+
["spot_if_collector_lambda.zip"]="collector/spot-dataset/aws/lambda/spot_if"
32+
["spot_price_collector_lambda.zip"]="collector/spot-dataset/aws/lambda/spot_price"
33+
["binpacked_workloads_generator_lambda.zip"]="collector/spot-dataset/aws/lambda/workload"
34+
)
35+
36+
for zip_name in "${!LAMBDA_PATHS[@]}"; do
37+
echo "Creating ZIP: $zip_name"
38+
zip -j "$zip_name" utility/slack_msg_sender.py
39+
zip -j "$zip_name" "${LAMBDA_PATHS[$zip_name]}"/*
40+
done
41+
42+
- name: Deploy to AWS Lambda
43+
run: |
44+
set -e
45+
46+
declare -A LAMBDA_FUNCTIONS=(
47+
["ondemand_price_collector"]="ondemand_price_collector_lambda.zip"
48+
["spotlake_post_processing"]="spotlake_post_processing_lambda.zip"
49+
["spot_if_collector"]="spot_if_collector_lambda.zip"
50+
["spot_price_collector"]="spot_price_collector_lambda.zip"
51+
["binpacked_workloads_generator"]="binpacked_workloads_generator_lambda.zip"
52+
)
53+
54+
for function_name in "${!LAMBDA_FUNCTIONS[@]}"; do
55+
echo "Deploying: $function_name"
56+
aws lambda update-function-code --function-name "$function_name" --zip-file "fileb://${LAMBDA_FUNCTIONS[$function_name]}"
57+
done

‎.github/workflows/azure-lambda-sync.yml‎

Lines changed: 21 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,11 @@ on:
44
branches:
55
- 'main'
66
paths:
7-
- 'collector/spot-dataset/azure/lambda/current_collector/**'
8-
- 'utility/slack_msg_sender.py'
7+
- 'collector/spot-dataset/azure/lambda/current_collector/lambda_function.py'
8+
- 'collector/spot-dataset/azure/lambda/current_collector/load_price.py'
9+
- 'collector/spot-dataset/azure/lambda/current_collector/load_if.py'
910
- 'const_config.py'
11+
- 'collector/spot-dataset/azure/lambda/current_collector/utils/**'
1012

1113
env:
1214
AWS_ACCESS_KEY_ID: ${{ secrets.SPOTRANK_ACCESS_KEY_ID }}
@@ -19,14 +21,25 @@ jobs:
1921
runs-on: ubuntu-latest
2022
steps:
2123
- name: checkout source code
22-
uses: actions/checkout@v1
23-
- name: Zip lambda_function code
24+
uses: actions/checkout@v4
25+
- name: Zip Lambda function code
2426
run: |
25-
zip -j ./collector/spot-dataset/azure/lambda/current_collector/azure_lambda.zip ./collector/spot-dataset/azure/lambda/current_collector/* ./const_config.py ./utility/slack_msg_sender.py
27+
rm -f azure_lambda.zip
28+
rm -f ./collector/spot-dataset/azure/lambda/current_collector/azure_lambda.zip
29+
30+
zip -j ./collector/spot-dataset/azure/lambda/current_collector/azure_lambda.zip \
31+
./collector/spot-dataset/azure/lambda/current_collector/lambda_function.py \
32+
./collector/spot-dataset/azure/lambda/current_collector/load_price.py \
33+
./collector/spot-dataset/azure/lambda/current_collector/load_if.py \
34+
./const_config.py
35+
2636
cd ./collector/spot-dataset/azure/lambda/current_collector/
27-
zip -r azure_lambda.zip ./utill/*
37+
zip -r azure_lambda.zip ./utils/*
38+
39+
2840
cd ../../../../../
2941
mv ./collector/spot-dataset/azure/lambda/current_collector/azure_lambda.zip ./
30-
- name: Deploy to lambda
42+
43+
- name: Deploy to AWS Lambda
3144
run: |
32-
aws lambda update-function-code --function-name azure-collector --zip-file fileb://azure_lambda.zip
45+
aws lambda update-function-code --function-name azure-collector --zip-file fileb://azure_lambda.zip

‎.github/workflows/azure-sps-lambda-sync.yml‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
name: deploy azure sps files to lambda
2+
on:
3+
push:
4+
branches:
5+
- 'main'
6+
paths:
7+
- 'collector/spot-dataset/azure/lambda/current_collector/lambda_function_sps.py'
8+
- 'collector/spot-dataset/azure/lambda/current_collector/load_price.py'
9+
- 'collector/spot-dataset/azure/lambda/current_collector/load_sps.py'
10+
- 'const_config.py'
11+
- 'collector/spot-dataset/azure/lambda/current_collector/utils/**'
12+
- 'collector/spot-dataset/azure/lambda/current_collector/sps_module/**'
13+
env:
14+
AWS_ACCESS_KEY_ID: ${{ secrets.SPOTRANK_ACCESS_KEY_ID }}
15+
AWS_SECRET_ACCESS_KEY: ${{ secrets.SPOTRANK_SECRET_ACCESS_KEY }}
16+
AWS_DEFAULT_REGION: 'us-west-2'
17+
18+
jobs:
19+
deploy_source:
20+
name: deploy lambda from source
21+
runs-on: ubuntu-latest
22+
steps:
23+
- name: checkout source code
24+
uses: actions/checkout@v4
25+
- name: Zip Lambda function code
26+
run: |
27+
rm -f azure_sps_lambda.zip
28+
rm -f ./collector/spot-dataset/azure/lambda/current_collector/azure_sps_lambda.zip
29+
30+
mkdir -p /tmp/lambda_collector
31+
32+
cp ./collector/spot-dataset/azure/lambda/current_collector/lambda_function_sps.py /tmp/lambda_collector/lambda_function.py
33+
cp ./collector/spot-dataset/azure/lambda/current_collector/load_price.py /tmp/lambda_collector/
34+
cp ./collector/spot-dataset/azure/lambda/current_collector/load_sps.py /tmp/lambda_collector/
35+
cp ./const_config.py /tmp/lambda_collector/
36+
37+
zip -j ./collector/spot-dataset/azure/lambda/current_collector/azure_sps_lambda.zip /tmp/lambda_collector/*
38+
39+
rm -rf /tmp/lambda_collector
40+
41+
cd ./collector/spot-dataset/azure/lambda/current_collector/
42+
zip -r azure_sps_lambda.zip ./utils/*
43+
zip -r azure_sps_lambda.zip ./sps_module/*
44+
45+
cd ../../../../../
46+
mv ./collector/spot-dataset/azure/lambda/current_collector/azure_sps_lambda.zip ./
47+
48+
- name: Deploy to AWS Lambda
49+
run: |
50+
aws lambda update-function-code --function-name azure-sps-collector-az-true --zip-file fileb://azure_sps_lambda.zip
51+
aws lambda update-function-code --function-name azure-sps-collector-az-false --zip-file fileb://azure_sps_lambda.zip

‎.github/workflows/gcp-lambda-sync.yml‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,16 +2,16 @@ name: deploy gcp files to lambda
22
on:
33
push:
44
branches:
5-
- 'main'
5+
- "main"
66
paths:
7-
- 'collector/spot-dataset/gcp/lambda/**'
8-
- 'utility/slack_msg_sender.py'
9-
- 'const_config.py'
7+
- "collector/spot-dataset/gcp/lambda/**"
8+
- "utility/slack_msg_sender.py"
9+
- "const_config.py"
1010

1111
env:
1212
AWS_ACCESS_KEY_ID: ${{ secrets.SPOTRANK_ACCESS_KEY_ID }}
1313
AWS_SECRET_ACCESS_KEY: ${{ secrets.SPOTRANK_SECRET_ACCESS_KEY }}
14-
AWS_DEFAULT_REGION: 'us-west-2'
14+
AWS_DEFAULT_REGION: "us-west-2"
1515

1616
jobs:
1717
deploy_source:
@@ -20,10 +20,13 @@ jobs:
2020
steps:
2121
- name: checkout source code
2222
uses: actions/checkout@v1
23+
- name: Create JSON file from environment variable
24+
run: echo "${{ secrets.GCP_KEY_JSON_BASE64 }}" | base64 --decode > ${{ secrets.GCP_JSON_FILENAME }}
2325
- name: Zip lambda_function code
2426
run: |
2527
zip -r gcp_lambda.zip ./utility/slack_msg_sender.py
2628
zip -j gcp_lambda.zip ./collector/spot-dataset/gcp/lambda/* ./const_config.py
29+
zip -j gcp_lambda.zip ${{ secrets.GCP_JSON_FILENAME }}
2730
- name: Deploy to lambda
2831
run: |
2932
aws lambda update-function-code --function-name gcp-collector --zip-file fileb://gcp_lambda.zip

‎.github/workflows/sync.yml‎

Lines changed: 0 additions & 24 deletions
This file was deleted.

‎.github/workflows/utilty-dataset-generator-ecr-push.yml‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
name: CI/CD
2+
on:
3+
push:
4+
branches:
5+
- main
6+
paths:
7+
- "utility/montly_share_raw_dataset_generator/**"
8+
9+
env:
10+
IMAGE_TAG: ${{ github.sha }}
11+
12+
jobs:
13+
build-and-push:
14+
name: Build and Push to ECR
15+
runs-on: ubuntu-latest
16+
steps:
17+
- name: Checkout Code
18+
uses: actions/checkout@v3
19+
20+
- name: Configure AWS credentials
21+
uses: aws-actions/configure-aws-credentials@v2
22+
with:
23+
aws-access-key-id: ${{ secrets.SPOTRANK_AWS_ACCESS_ID }}
24+
aws-secret-access-key: ${{ secrets.SPOTRANK_AWS_SECRET_KEY }}
25+
aws-region: ${{ secrets.SPOTRANK_AWS_REGION }}
26+
27+
- name: Log in to Amazon ECR
28+
uses: aws-actions/amazon-ecr-login@v2
29+
30+
- name: Build and Push Docker Image
31+
run: |
32+
cd utility/montly_share_raw_dataset_generator
33+
docker build -t ${{ secrets.SPOTRANK_AWS_ECR_REPOSITORY }} .
34+
docker push ${{ secrets.SPOTRANK_AWS_ECR_REPOSITORY }}:$IMAGE_TAG
35+
36+
- name: Change to latest tag & Push Docker Image
37+
run: |
38+
docker tag ${{ secrets.SPOTRANK_AWS_ECR_REPOSITORY }}:$IMAGE_TAG ${{ secrets.SPOTRANK_AWS_ECR_REPOSITORY }}:latest
39+
docker push ${{ secrets.SPOTRANK_AWS_ECR_REPOSITORY }}:latest

‎.github/workflows/web-sync.yml‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
name: Sync Demo-Web html to S3
2+
3+
on:
4+
push:
5+
branches:
6+
- "main"
7+
paths:
8+
- "frontend/**"
9+
workflow_dispatch:
10+
11+
env:
12+
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_HYUDDPS_WEB_ACCESS_KEY_ID }}
13+
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_HYUDDPS_WEB_SECRET_ACCESS_KEY }}
14+
AWS_DEFAULT_REGION: "us-west-2"
15+
16+
jobs:
17+
sync:
18+
runs-on: ubuntu-latest
19+
steps:
20+
- uses: actions/checkout@v2
21+
22+
- name: Setup Node.js
23+
uses: actions/setup-node@v3
24+
with:
25+
node-version: '18'
26+
cache: 'npm'
27+
cache-dependency-path: './frontend/package-lock.json'
28+
29+
- name: Install dependencies
30+
run: |
31+
cd frontend
32+
npm ci
33+
34+
- name: Build Next.js app
35+
run: |
36+
cd frontend
37+
npm run build
38+
39+
- name: "Sync with AWS S3"
40+
run: aws s3 sync ./frontend/out/ s3://${{ secrets.AWS_SPOTLAKE_S3_BUCKET_NAME }} --acl public-read --delete
41+
42+
- name: "Create Invalidation"
43+
run: aws cloudfront create-invalidation --distribution-id ${{ secrets.AWS_SPOTLAKE_CLOUDFRONT_DISTRIBUTION_ID }} --paths "/*"

‎.gitignore‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,10 @@
11
__pycache__
22
.DS_Store
3-
/frontend/node_modules/
3+
/frontend/node_modules/
4+
**/*binpacked_workloads.pkl.gz
5+
**/*test.py
6+
**/.vscode
7+
**/*.terraform
8+
**/*.terraform.lock.hcl
9+
**/*.tfstate
10+
**/*.tfstate.backup

‎collector/spot-dataset/aws/batch-test/Dockerfile‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
2+
3+
# Stage 2: Final image
4+
FROM python:3.12-slim
5+
6+
ARG AWS_ACCESS_KEY_ID
7+
ARG AWS_SECRET_ACCESS_KEY
8+
9+
ENV AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID
10+
ENV AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY
11+
12+
WORKDIR /app
13+
14+
# Install system dependencies
15+
RUN apt-get update && apt-get install -y \
16+
curl \
17+
unzip \
18+
procps \
19+
&& rm -rf /var/lib/apt/lists/*
20+
21+
# Install AWS CLI based on architecture
22+
RUN ARCH=$(uname -m) && \
23+
if [ "$ARCH" = "x86_64" ]; then \
24+
URL="https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip"; \
25+
elif [ "$ARCH" = "aarch64" ]; then \
26+
URL="https://awscli.amazonaws.com/awscli-exe-linux-aarch64.zip"; \
27+
else \
28+
echo "Unsupported architecture: $ARCH" && exit 1; \
29+
fi && \
30+
curl "$URL" -o "awscliv2.zip" && \
31+
unzip awscliv2.zip && \
32+
./aws/install && \
33+
rm -rf awscliv2.zip aws
34+
35+
# Install Python dependencies
36+
RUN pip install --no-cache-dir \
37+
boto3 \
38+
pandas \
39+
numpy \
40+
pyyaml \
41+
ortools \
42+
requests
43+
44+
# Download and install spotinfo based on architecture
45+
# Download and install spotinfo based on architecture
46+
RUN ARCH=$(uname -m) && \
47+
if [ "$ARCH" = "x86_64" ]; then \
48+
URL="https://github.com/alexei-led/spotinfo/releases/download/2.2.0/spotinfo_linux_amd64"; \
49+
elif [ "$ARCH" = "aarch64" ]; then \
50+
URL="https://github.com/alexei-led/spotinfo/releases/download/2.2.0/spotinfo_linux_arm64"; \
51+
else \
52+
echo "Unsupported architecture: $ARCH" && exit 1; \
53+
fi && \
54+
curl -L $URL -o /usr/local/bin/spotinfo && \
55+
chmod +x /usr/local/bin/spotinfo
56+
57+
# Copy application code
58+
# Assuming build context is the project root (spotlake/)
59+
COPY collector/spot-dataset/aws/batch-test /app/collector/spot-dataset/aws/batch-test
60+
# COPY const_config.py /app/
61+
COPY utility /app/utility
62+
RUN chmod +x /app/collector/spot-dataset/aws/batch-test/scripts/run_collection.sh
63+
64+
# Set PYTHONPATH to include /app so imports work
65+
ENV PYTHONPATH=/app
66+
67+
# Default entrypoint (can be overridden)
68+
CMD ["python3", "--version"]

0 commit comments

Comments
 (0)