Skip to content

Commit bda6140

Browse files
jonhopper-dataengineersjonhopper-dataengineers
committed
added base actions
1 parent 428a838 commit bda6140

File tree

12 files changed

+527
-0
lines changed

12 files changed

+527
-0
lines changed

‎setup/database-name-branch-based/action.yml

+47
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
name: Determine the deployment database name
2+
description: Geneates the database in which to deploy into
3+
inputs:
4+
source-database-name:
5+
description: The root name of the database to copy from (excluding dev, test, prod)
6+
required: true
7+
include-branch:
8+
description: Whether to include the branch name in the database name
9+
required: false
10+
default: $true
11+
outputs:
12+
database-name:
13+
description: The name of the database to use for deployment
14+
value: ${{ steps.determine-database-name.outputs.database-name }}
15+
runs:
16+
using: "composite"
17+
steps:
18+
- name: Get branch name (merge)
19+
if: github.event_name != 'pull_request' && include-branch == $true
20+
shell: bash
21+
run: |
22+
echo "BRANCH_NAME=$(echo ${GITHUB_REF#refs/heads/} | tr / -)" >> $GITHUB_ENV
23+
24+
- name: Get branch name (pull request)
25+
if: github.event_name == 'pull_request' && include-branch == $true
26+
shell: bash
27+
run: |
28+
echo "BRANCH_NAME=$(echo ${GITHUB_HEAD_REF} | tr / -)" >> $GITHUB_ENV
29+
30+
- name: BRANCH_NAME
31+
id: determine-database-name
32+
if: include-branch == $true
33+
shell: bash
34+
run: |
35+
echo "::set-output name=database-name::${{ inputs.source-database-name }}_${{ env.BRANCH_NAME }}"
36+
37+
- name: BRANCH_NAME
38+
id: determine-database-name
39+
if: include-branch == $false
40+
shell: bash
41+
run: |
42+
echo "::set-output name=database-name::${{ inputs.source-database-name }}"
43+
44+
- name: show target name
45+
shell: bash
46+
run: |
47+
echo "Target: ${{ steps.determine-database-name.outputs.database-name }}"

‎setup/database-name-generate/action.yml

+27
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
name: Prepare Unit Test Database
2+
description: Creates a new Database name for unit testing
3+
inputs:
4+
source-database-name:
5+
description: The name of the database to copy from
6+
required: true
7+
outputs:
8+
unit-test-database:
9+
description: The name of the database to use for unit testing
10+
value: ${{ steps.create-database-name.outputs.database-name }}
11+
runs:
12+
using: "composite"
13+
steps:
14+
- name: create new unit test database name
15+
id: create-database-name
16+
shell: bash
17+
run: |
18+
sudo apt-get update
19+
sudo apt-get install uuid -y
20+
# Set DB Name
21+
DB_NAME=$(uuid | tr '-' '_')
22+
echo "::set-output name=database-name::${{ inputs.source-database-name }}_UT_$(echo $DB_NAME)"
23+
24+
- name: show target name
25+
shell: bash
26+
run: |
27+
echo "Target: ${{ steps.create-database-name.outputs.database-name }}"

‎setup/dbt/action.yml

+26
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
name: Dbt Setup
2+
description: Setups Dbt Components
3+
inputs:
4+
working-directory:
5+
description: The directory where the DBT project is located
6+
required: true
7+
default: './dbt'
8+
9+
runs:
10+
using: "composite"
11+
steps:
12+
- name: install Python
13+
uses: "actions/setup-python@v2"
14+
with:
15+
python-version: 3.8
16+
17+
- name: install components
18+
shell: bash
19+
run: pip install -r .deployrequirements
20+
working-directory: ${{ inputs.working-directory }}
21+
22+
- name: setup dbt dependencies
23+
shell: bash
24+
run: |
25+
dbt deps
26+
working-directory: ${{ inputs.working-directory }}

‎setup/schemachange/action.yml

+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
name: Schemachange Setup
2+
description: Setups Schemachange Components
3+
inputs:
4+
working-directory:
5+
description: The directory where the Schemachange project is located
6+
required: true
7+
default: './schemachange'
8+
9+
runs:
10+
using: "composite"
11+
steps:
12+
- name: install Python
13+
uses: "actions/setup-python@v2"
14+
with:
15+
python-version: 3.8
16+
17+
- name: install components
18+
shell: bash
19+
run: pip install -r .deployrequirements
20+
working-directory: ${{ inputs.working-directory }}

‎tasks/database-clone/action.yml

+66
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
name: Clone Database to Unit Test
2+
description: Generates a test database
3+
inputs:
4+
working-directory:
5+
description: The directory where the scripts are located
6+
required: true
7+
source-database:
8+
description: The source database to clone
9+
required: true
10+
target-database:
11+
description: The target database name to clone to
12+
required: true
13+
snowflake-account:
14+
description: The Snowflake account name
15+
required: true
16+
snowflake-username:
17+
description: The Snowflake username
18+
required: true
19+
snowflake-password:
20+
description: The Snowflake password
21+
required: true
22+
snowflake-role:
23+
description: The Snowflake role
24+
required: true
25+
snowflake-warehouse:
26+
description: The Snowflake warehouse
27+
required: true
28+
data-lake-url:
29+
description: The Data Lake URL
30+
required: false
31+
storage-integration-name:
32+
description: The Storage Integration Name
33+
required: false
34+
35+
runs:
36+
using: "composite"
37+
steps:
38+
- name: Find and replace target & source database name
39+
run: |
40+
sed -i 's/TARGET_DATABASE_NAME/${{ inputs.target-database }}/g' database-clone.sql
41+
sed -i 's/SOURCE_DATABASE_NAME/${{ inputs.source-database }}/g' database-clone.sql
42+
shell: pwsh
43+
working-directory: ${{ inputs.working-directory }}
44+
- name: Download SnowSQL
45+
shell: bash
46+
run: curl -O https://sfc-repo.snowflakecomputing.com/snowsql/bootstrap/1.2/linux_x86_64/snowsql-1.2.9-linux_x86_64.bash
47+
- name: Install SnowSQL
48+
shell: bash
49+
run: SNOWSQL_DEST=~/snowflake SNOWSQL_LOGIN_SHELL=~/.profile bash snowsql-1.2.9-linux_x86_64.bash
50+
- name: Execute SQL against Snowflake
51+
shell: bash
52+
run: |
53+
~/snowflake/snowsql \
54+
-a $SNOWSQL_ACCOUNT \
55+
-u $SNOWSQL_USER \
56+
-d $SNOWSQL_DATABASE \
57+
-w $SNOWSQL_WAREHOUSE \
58+
-f database-clone.sql
59+
env:
60+
SNOWSQL_ACCOUNT: ${{ inputs.snowflake-account }}
61+
SNOWSQL_USER: ${{ inputs.snowflake-username }}
62+
SNOWSQL_PWD: ${{ inputs.snowflake-password }}
63+
SNOWSQL_DATABASE: ${{ inputs.source-database }}
64+
SNOWSQL_WAREHOUSE: ${{ inputs.snowflake-warehouse }}
65+
working-directory: ${{ inputs.working-directory }}
66+

‎tasks/database-destroy/action.yml

+55
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
name: Removes Unit Test Database
2+
description: Drops the unit-test database
3+
inputs:
4+
working-directory:
5+
description: The directory where the scripts are located
6+
required: true
7+
target-database:
8+
description: The target database name to clone to
9+
required: true
10+
snowflake-account:
11+
description: The Snowflake account name
12+
required: true
13+
snowflake-username:
14+
description: The Snowflake username
15+
required: true
16+
snowflake-password:
17+
description: The Snowflake password
18+
required: true
19+
snowflake-role:
20+
description: The Snowflake role
21+
required: true
22+
snowflake-warehouse:
23+
description: The Snowflake warehouse
24+
required: true
25+
26+
runs:
27+
using: "composite"
28+
steps:
29+
- name: Find and replace target
30+
run: |
31+
sed -i 's/TARGET_DATABASE_NAME/${{ inputs.target-database }}/g' database-destroy.sql
32+
shell: pwsh
33+
working-directory: ${{ inputs.working-directory }}
34+
- name: Download SnowSQL
35+
shell: bash
36+
run: curl -O https://sfc-repo.snowflakecomputing.com/snowsql/bootstrap/1.2/linux_x86_64/snowsql-1.2.9-linux_x86_64.bash
37+
- name: Install SnowSQL
38+
shell: bash
39+
run: SNOWSQL_DEST=~/snowflake SNOWSQL_LOGIN_SHELL=~/.profile bash snowsql-1.2.9-linux_x86_64.bash
40+
- name: Execute SQL against Snowflake
41+
shell: bash
42+
run: |
43+
~/snowflake/snowsql \
44+
-a $SNOWSQL_ACCOUNT \
45+
-u $SNOWSQL_USER \
46+
-w $SNOWSQL_WAREHOUSE \
47+
-f database-destroy.sql
48+
env:
49+
SNOWSQL_ACCOUNT: ${{ inputs.snowflake-account }}
50+
SNOWSQL_USER: ${{ inputs.snowflake-username }}
51+
SNOWSQL_PWD: ${{ inputs.snowflake-password }}
52+
SNOWSQL_DATABASE: ${{ inputs.target-database }}
53+
SNOWSQL_WAREHOUSE: ${{ inputs.snowflake-warehouse }}
54+
working-directory: ${{ inputs.working-directory }}
55+

‎tasks/dbt-apply-tag/action.yml

+50
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
name: Run DBT
2+
description: Runs DBT
3+
inputs:
4+
target:
5+
description: The target to run DBT against
6+
required: true
7+
tag:
8+
description: The tag to apply
9+
required: true
10+
working-directory:
11+
description: The directory where the DBT project is located
12+
required: true
13+
profiles-directory:
14+
description: The directory where the DBT profiles are located
15+
required: true
16+
snowflake-account:
17+
description: The snowflake account to use
18+
required: true
19+
snowflake-username:
20+
description: The snowflake username to use
21+
required: true
22+
snowflake-password:
23+
description: The snowflake password to use
24+
required: true
25+
snowflake-role:
26+
description: The snowflake role to use
27+
required: true
28+
snowflake-warehouse:
29+
description: The snowflake warehouse to use
30+
required: true
31+
snowflake-target-database:
32+
description: The snowflake target database to use
33+
required: true
34+
35+
36+
runs:
37+
using: "composite"
38+
steps:
39+
- name: Run DBT
40+
shell: bash
41+
run: |
42+
dbt run --profiles-dir=${{ inputs.profiles-directory }} --target=${{ inputs.target }} -models tag:${{ inputs.tag }}
43+
working-directory: ${{ inputs.working-directory }}
44+
env:
45+
snowflake-account: ${{ inputs.snowflake-account }}
46+
snowflake-username: ${{ inputs.snowflake-username }}
47+
snowflake-role: ${{ inputs.snowflake-role }}
48+
snowflake-warehouse: ${{ inputs.snowflake-warehouse }}
49+
snowflake-target-database: ${{ inputs.snowflake-target-database }}
50+
snowflake-password: ${{ inputs.snowflake-password }}

‎tasks/dbt-apply/action.yml

+46
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
name: Run DBT
2+
description: Runs DBT
3+
inputs:
4+
target:
5+
description: The target to run DBT against
6+
required: true
7+
working-directory:
8+
description: The directory where the DBT project is located
9+
required: true
10+
profiles-directory:
11+
description: The directory where the DBT profiles are located
12+
required: true
13+
snowflake-account:
14+
description: The snowflake account to use
15+
required: true
16+
snowflake-username:
17+
description: The snowflake username to use
18+
required: true
19+
snowflake-password:
20+
description: The snowflake password to use
21+
required: true
22+
snowflake-role:
23+
description: The snowflake role to use
24+
required: true
25+
snowflake-warehouse:
26+
description: The snowflake warehouse to use
27+
required: true
28+
snowflake-target-database:
29+
description: The snowflake target database to use
30+
required: true
31+
32+
runs:
33+
using: "composite"
34+
steps:
35+
- name: Run DBT
36+
shell: bash
37+
run: |
38+
dbt run --profiles-dir=${{ inputs.profiles-directory }} --target=${{ inputs.target }} --exclude tag:daily tag:unit_test
39+
working-directory: ${{ inputs.working-directory }}
40+
env:
41+
snowflake-account: ${{ inputs.snowflake-account }}
42+
snowflake-username: ${{ inputs.snowflake-username }}
43+
snowflake-role: ${{ inputs.snowflake-role }}
44+
snowflake-warehouse: ${{ inputs.snowflake-warehouse }}
45+
snowflake-target-database: ${{ inputs.snowflake-target-database }}
46+
snowflake-password: ${{ inputs.snowflake-password }}

‎tasks/dbt-source-freshness/action.yml

+47
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
name: Source Freshness Checks
2+
description: Checks the source freshness of the DBT project
3+
inputs:
4+
target:
5+
description: The target to run DBT against
6+
required: true
7+
working-directory:
8+
description: The directory where the DBT project is located
9+
required: true
10+
profiles-directory:
11+
description: The directory where the DBT profiles are located
12+
required: true
13+
snowflake-account:
14+
description: The snowflake account to use
15+
required: true
16+
snowflake-username:
17+
description: The snowflake username to use
18+
required: true
19+
snowflake-password:
20+
description: The snowflake password to use
21+
required: true
22+
snowflake-role:
23+
description: The snowflake role to use
24+
required: true
25+
snowflake-warehouse:
26+
description: The snowflake warehouse to use
27+
required: true
28+
snowflake-target-database:
29+
description: The snowflake target database to use
30+
required: true
31+
32+
33+
runs:
34+
using: "composite"
35+
steps:
36+
- name: Run Source Freshness Checks
37+
shell: bash
38+
run: |
39+
dbt source snapshot-freshness --profiles-dir=${{ inputs.profiles-directory }} --target=${{ inputs.target }}
40+
working-directory: ${{ inputs.working-directory }}
41+
env:
42+
snowflake-account: ${{ inputs.snowflake-account }}
43+
snowflake-username: ${{ inputs.snowflake-username }}
44+
snowflake-role: ${{ inputs.snowflake-role }}
45+
snowflake-warehouse: ${{ inputs.snowflake-warehouse }}
46+
snowflake-target-database: ${{ inputs.snowflake-target-database }}
47+
snowflake-password: ${{ inputs.snowflake-password }}

0 commit comments

Comments
 (0)