Skip to content

Commit bc99d2c

Browse files
bjaggbjagg
committed
Initial public release
0 parents  commit bc99d2c

829 files changed

Lines changed: 181571 additions & 0 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.aws/config‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
[profile AWS-Gates-CLIR]
2+
sso_start_url = https://uniconsso.awsapps.com/start
3+
sso_region = us-east-1
4+
sso_account_id = 381492161417
5+
sso_role_name = AdministratorAccess
6+
region = us-west-2
7+
output = json
8+

‎.dockerignore‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
# Ignore everything except what we need
2+
*
3+
4+
# Include only what's needed for the build
5+
!projects/
6+
!bases/
7+
!components/
8+
# TODO: Remember to update the following as dev progresses
9+
!development/dagster/
10+
!orchestrators/dagster/lif-orchestrator/
11+
12+
# Exclude development files
13+
**/.venv/
14+
**/__pycache__/
15+
**/*.pyc
16+
**/.git/
17+
**/node_modules/
18+
**/.DS_Store

‎.github/actions/build-and-push/action.yml‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
name: Build and Push
2+
description: Build and Push a Docker Image to AWS ECR
3+
inputs:
4+
working-dir:
5+
description: The application build directory
6+
required: true
7+
github-actions-role:
8+
description: The AWS role to assume for GitHub Actions
9+
required: true
10+
role-session-name:
11+
description: The session tag to apply to the role session
12+
required: true
13+
aws-region:
14+
description: The AWS region of the target ECR registry
15+
required: true
16+
ecr-repository:
17+
description: The target ECR repo within the ECR registry
18+
required: true
19+
build-args:
20+
description: Docker build arguments (optional)
21+
required: false
22+
default: ''
23+
24+
runs:
25+
using: composite
26+
steps:
27+
- name: Configure AWS credentials
28+
uses: aws-actions/configure-aws-credentials@v5
29+
with:
30+
audience: sts.amazonaws.com
31+
role-to-assume: ${{ inputs.github-actions-role }}
32+
role-session-name: ${{ inputs.role-session-name }}
33+
aws-region: ${{ env.AWS_REGION }}
34+
35+
- name: Login to Amazon ECR
36+
id: login-ecr
37+
uses: aws-actions/amazon-ecr-login@v2
38+
39+
- name: image tag
40+
id: vars
41+
shell: bash
42+
run: |
43+
sha_short=$(git rev-parse --short HEAD)
44+
date_string=$(date +%Y-%m-%d-%H-%M-%S)
45+
tag=${date_string}-${sha_short}
46+
echo "tag=$tag" >> $GITHUB_OUTPUT
47+
48+
- name: Set up QEMU
49+
uses: docker/setup-qemu-action@v3
50+
51+
- name: Set up Docker Buildx
52+
uses: docker/setup-buildx-action@v3
53+
54+
- name: Build and push
55+
id: build-and-push
56+
uses: docker/build-push-action@v6
57+
with:
58+
context: ${{ inputs.working-dir }}
59+
platforms: linux/amd64
60+
build-args: ${{ inputs.build-args }}
61+
push: true
62+
tags: |
63+
${{ steps.login-ecr.outputs.registry }}/${{ inputs.ecr-repository }}:${{ steps.vars.outputs.tag }}
64+
${{ steps.login-ecr.outputs.registry }}/${{ inputs.ecr-repository }}:latest
65+
cache-from: type=registry,ref=${{ steps.login-ecr.outputs.registry }}/${{ inputs.ecr-repository }}:latest
66+
cache-to: type=inline

‎.github/actions/setup-poetry/action.yml‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
name: Setup Poetry
2+
description: Install and setup Poetry & Earthmover
3+
inputs:
4+
working-dir:
5+
description: The application build directory
6+
required: true
7+
runs:
8+
using: composite
9+
steps:
10+
- name: Setup Python
11+
uses: actions/setup-python@v4
12+
with:
13+
python-version: 3.12.0
14+
15+
- name: Install poetry
16+
shell: bash
17+
working-directory: ${{ inputs.working-dir }}
18+
run: |
19+
python -m pip install poetry==1.8.3
20+
python -m pip install earthmover
21+
python -m poetry config virtualenvs.in-project true
22+
23+
- name: Cache the virtualenv
24+
uses: actions/cache@v4
25+
with:
26+
path: ${{ inputs.working-dir }}/.venv
27+
key: ${{ runner.os }}-venv-${{ hashFiles('**/poetry.lock') }}
28+

‎.github/actions/update-cluster/action.yml‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
name: Update ECS Cluster
2+
description: Force a deployment on an ECS cluster
3+
inputs:
4+
github-actions-role:
5+
description: The AWS role to assume for GitHub Actions
6+
required: true
7+
role-session-name:
8+
description: The session tag to apply to the role session
9+
required: true
10+
aws-region:
11+
description: The AWS region of the target ECR registry
12+
required: true
13+
ecs-cluster:
14+
description: The ECS Cluster that contains the service
15+
required: true
16+
ecs-service:
17+
description: The target ECS Service
18+
required: true
19+
20+
runs:
21+
using: composite
22+
steps:
23+
- name: Configure AWS credentials
24+
uses: aws-actions/configure-aws-credentials@v5
25+
with:
26+
audience: sts.amazonaws.com
27+
role-to-assume: ${{ inputs.github-actions-role }}
28+
role-session-name: ${{ inputs.role-session-name }}
29+
aws-region: ${{ env.AWS_REGION }}
30+
31+
- id: install-aws-cli
32+
uses: unfor19/install-aws-cli-action@v1
33+
with:
34+
version: 2
35+
verbose: false
36+
arch: amd64
37+
38+
- name: Force ECS service deployment
39+
id: vars
40+
shell: bash
41+
run: |
42+
if aws ecs list-clusters | \
43+
grep -q ${{ inputs.ecs-cluster }}; then
44+
echo "cluster \"${{ inputs.ecs-cluster }}\" exists"
45+
else
46+
echo "cluster \"${{ inputs.ecs-cluster }}\" does not exist"
47+
exit 0
48+
fi
49+
50+
if aws ecs list-services \
51+
--cluster ${{ inputs.ecs-cluster }} | \
52+
grep -q ${{ inputs.ecs-service }}; then
53+
echo "service \"${{ inputs.ecs-service }}\" exists"
54+
else
55+
echo "service \"${{ inputs.ecs-service }}\" does not exist"
56+
exit 0
57+
fi
58+
59+
aws ecs update-service \
60+
--cluster ${{ inputs.ecs-cluster }} \
61+
--service ${{ inputs.ecs-service }} \
62+
--force-new-deployment

‎.github/workflows/dagster.yml‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
name: Deploy Dagster Webserver/Daemon to Amazon ECS
2+
3+
env:
4+
AWS_REGION: us-east-1
5+
GHA_ROLE: "arn:aws:iam::381492161417:role/lif-github-actions-dev"
6+
ECS_CLUSTER: dev
7+
8+
permissions:
9+
id-token: write
10+
contents: read
11+
12+
on:
13+
workflow_dispatch:
14+
push:
15+
branches: [ "main" ]
16+
paths:
17+
- .github/workflows/dagster.yml
18+
- projects/dagster_oss_ecs/**
19+
20+
jobs:
21+
22+
dagster-webserver:
23+
runs-on: ubuntu-latest
24+
defaults:
25+
run:
26+
shell: bash
27+
env:
28+
WORKING_DIR: ./
29+
ECR_REPOSITORY: lif/dev/dagster
30+
steps:
31+
- name: Checkout repository
32+
uses: actions/checkout@v4
33+
34+
- name: Prepare for Docker build
35+
shell: bash
36+
run: |
37+
cp projects/dagster_oss_ecs/Dockerfile.dagster Dockerfile
38+
39+
- name: Build and push docker image
40+
uses: ./.github/actions/build-and-push
41+
with:
42+
working-dir: ${{ env.WORKING_DIR }}
43+
github-actions-role: ${{ env.GHA_ROLE }}
44+
role-session-name: dagster-build
45+
aws-region: ${{ env.AWS_REGION }}
46+
ecr-repository: ${{ env.ECR_REPOSITORY }}
47+
48+
- name: Update the Dagster Webserver ECS Service
49+
uses: ./.github/actions/update-cluster
50+
with:
51+
github-actions-role: ${{ env.GHA_ROLE }}
52+
role-session-name: dagster-webserver-ecs
53+
aws-region: ${{ env.AWS_REGION }}
54+
ecs-cluster: ${{ env.ECS_CLUSTER }}
55+
ecs-service: dagster-webserver-FARGATE
56+
57+
- name: Update the Dagster Daemon ECS Service
58+
uses: ./.github/actions/update-cluster
59+
with:
60+
github-actions-role: ${{ env.GHA_ROLE }}
61+
role-session-name: dagster-daemon-ecs
62+
aws-region: ${{ env.AWS_REGION }}
63+
ecs-cluster: ${{ env.ECS_CLUSTER }}
64+
ecs-service: dagster-daemon-FARGATE

‎.github/workflows/dagster_cloud_agent.yml‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
name: Deploy Dagster Cloud Agent to Amazon ECS
2+
3+
env:
4+
AWS_REGION: us-east-1
5+
GHA_ROLE: "arn:aws:iam::381492161417:role/lif-github-actions-dev"
6+
ECS_CLUSTER: dev
7+
ECS_SERVICE: dagster-cloud-agent-FARGATE
8+
9+
permissions:
10+
id-token: write
11+
contents: read
12+
13+
on:
14+
workflow_dispatch:
15+
push:
16+
branches: [ "main", "feature/dagster-init" ]
17+
paths:
18+
- .github/workflows/dagster_cloud_agent.yml
19+
- cloudformation/dev-dagster-cloud-agent.params
20+
- cloudformation/dagster-cloud-agent-taskdef-includes.yml
21+
- cloudformation/dagster-cloud-agent-taskrole-includes.yml
22+
- projects/dagster_plus_hybrid/dagster_cloud_agent/**
23+
24+
jobs:
25+
26+
dagster-cloud-agent:
27+
runs-on: ubuntu-latest
28+
defaults:
29+
run:
30+
shell: bash
31+
env:
32+
WORKING_DIR: ./projects/dagster_plus_hybrid/dagster_cloud_agent
33+
ECR_REPOSITORY: lif/dev/dagster_cloud_agent
34+
steps:
35+
- uses: actions/checkout@v4
36+
37+
- uses: ./.github/actions/build-and-push
38+
with:
39+
working-dir: ${{ env.WORKING_DIR }}
40+
github-actions-role: ${{ env.GHA_ROLE }}
41+
role-session-name: dagster-cloud-agent-build
42+
aws-region: ${{ env.AWS_REGION }}
43+
ecr-repository: ${{ env.ECR_REPOSITORY }}
44+
45+
- uses: ./.github/actions/update-cluster
46+
with:
47+
github-actions-role: ${{ env.GHA_ROLE }}
48+
role-session-name: dagster-cloud-agent-ecs
49+
aws-region: ${{ env.AWS_REGION }}
50+
ecs-cluster: ${{ env.ECS_CLUSTER }}
51+
ecs-service: ${{ env.ECS_SERVICE }}
52+

‎.github/workflows/dagster_code_location.yml‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
name: Deploy Dagster Code Location to Amazon ECS
2+
3+
env:
4+
AWS_REGION: us-east-1
5+
GHA_ROLE: "arn:aws:iam::381492161417:role/lif-github-actions-dev"
6+
ECS_CLUSTER: dev
7+
ECS_SERVICE: dagster-code-location-FARGATE
8+
9+
permissions:
10+
id-token: write
11+
contents: read
12+
13+
on:
14+
workflow_dispatch:
15+
push:
16+
branches: [ "main" ]
17+
paths:
18+
- .github/workflows/dagster_code_location.yml
19+
- projects/dagster_oss_ecs/**
20+
- components/lif/data_source_adapters/**
21+
- components/lif/datatypes/**
22+
- components/lif/exceptions/**
23+
- components/lif/logging/**
24+
- components/lif/query_planner_service/**
25+
- orchestrators/dagster/lif-orchestrator/**
26+
27+
jobs:
28+
29+
dagster-code-location:
30+
runs-on: ubuntu-latest
31+
defaults:
32+
run:
33+
shell: bash
34+
env:
35+
WORKING_DIR: ./
36+
ECR_REPOSITORY: lif/dev/dagster_code_location
37+
steps:
38+
- name: Checkout repository
39+
uses: actions/checkout@v4
40+
41+
- name: Build Orchestrator
42+
working-directory: orchestrators/dagster/lif-orchestrator
43+
shell: bash
44+
run: |
45+
curl -LsSf https://astral.sh/uv/install.sh | sh
46+
uv sync
47+
48+
- name: Prepare for Docker build
49+
shell: bash
50+
run: |
51+
cp projects/dagster_oss_ecs/Dockerfile.code_location Dockerfile
52+
53+
- name: Build and push docker image
54+
uses: ./.github/actions/build-and-push
55+
with:
56+
working-dir: ${{ env.WORKING_DIR }}
57+
github-actions-role: ${{ env.GHA_ROLE }}
58+
role-session-name: dagster-code-location-build
59+
aws-region: ${{ env.AWS_REGION }}
60+
ecr-repository: ${{ env.ECR_REPOSITORY }}
61+
62+
- name: Update the Dagster Code Location ECS Service
63+
uses: ./.github/actions/update-cluster
64+
with:
65+
github-actions-role: ${{ env.GHA_ROLE }}
66+
role-session-name: dagster-code-location-ecs
67+
aws-region: ${{ env.AWS_REGION }}
68+
ecs-cluster: ${{ env.ECS_CLUSTER }}
69+
ecs-service: ${{ env.ECS_SERVICE }}

0 commit comments

Comments
 (0)