Skip to content

Commit 440015d

Browse files
jonhopper-dataengineersjonhopper-dataengineers
committed
added new tasks for documentation and cloning
1 parent 239b21f commit 440015d

File tree

7 files changed

+263
-9
lines changed

7 files changed

+263
-9
lines changed

‎README.md

+36
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,38 @@
11
# Snowflake.DataOps.Actions
22
Github Actions Designed to work with the workflows created for the Snowflake.DataOps Project
3+
4+
## Snowflake Cloning
5+
6+
| Action | Description | Usage |
7+
| -------- | ----------- | ---- |
8+
| DataEngineersNZ/Snowflake.DataOps.Actions/setup/database-name-branch-based | | |
9+
| DataEngineersNZ/Snowflake.DataOps.Actions/tasks/database-clone | | |
10+
| DataEngineersNZ/Snowflake.DataOps.Actions/tasks/database-destroy | | |
11+
| DataEngineersNZ/Snowflake.DataOps.Actions/tasks/database-permissions | | |
12+
13+
## dbt Actions
14+
15+
| Action | Description | Usage |
16+
| -------------------------------------------------------------------------- | ----------- | ----- |
17+
| DataEngineersNZ/Snowflake.DataOps.Actions/setup/dbt | | |
18+
| DataEngineersNZ/Snowflake.DataOps.Actions/tasks/dbt-apply | | |
19+
| DataEngineersNZ/Snowflake.DataOps.Actions/tasks/dbt-apply-tag | | |
20+
| DataEngineersNZ/Snowflake.DataOps.Actions/tasks/dbt-docs-generate | | |
21+
| DataEngineersNZ/Snowflake.DataOps.Actions/tasks/dbt-docs-publish | | |
22+
| DataEngineersNZ/Snowflake.DataOps.Actions/tasks/dbt-manifest-clone | | |
23+
| DataEngineersNZ/Snowflake.DataOps.Actions/tasks/dbt-manifest-download | | |
24+
| DataEngineersNZ/Snowflake.DataOps.Actions/tasks/dbt-manifest-upload | | |
25+
| DataEngineersNZ/Snowflake.DataOps.Actions/tasks/dbt-source-freshness | | |
26+
| DataEngineersNZ/Snowflake.DataOps.Actions/tasks/dbt-test | | |
27+
28+
29+
## Schemachange Actions
30+
| Action | Description | Usage |
31+
| -------------------------------------------------------------------------- | ----------- | ----- |
32+
| DataEngineersNZ/Snowflake.DataOps.Actions/setup/schemachange | | |
33+
| DataEngineersNZ/Snowflake.DataOps.Actions/tasks/schemachange-apply | | |
34+
35+
## Linting Actions
36+
| Action | Description | Usage |
37+
| -------------------------------------------------------------------------- | ----------- | ----- |
38+
| DataEngineersNZ/Snowflake.DataOps.Actions/tasks/lint-sql | | |

‎tasks/dbt-apply/action.yml

+18-5
Original file line numberDiff line numberDiff line change
@@ -34,10 +34,16 @@ runs:
3434
steps:
3535
- name: Run DBT without Unit-tets
3636
if: inputs.target != 'unit-test'
37-
shell: bash
37+
shell: pwsh
3838
run: |
39-
dbt seed --profiles-dir=${{ inputs.profiles-directory }} --target=${{ inputs.target }} --exclude path:seeds/unit_tests
40-
dbt run --profiles-dir=${{ inputs.profiles-directory }} --target=${{ inputs.target }} --exclude tag:daily tag:unit_test
39+
dbt seed --profiles-dir=${{ inputs.profiles-directory }} --target=${{ inputs.target }} --exclude path:seeds/unit_tests
40+
if(Test-Path -Path ".\manifest.json" -PathType Leaf) {
41+
echo " ==> Previous Manifest file found, running dbt run for release"
42+
dbt run -s state:modified --defer --state . --profiles-dir=${{ inputs.profiles-directory }} --target=${{ inputs.target }} --exclude tag:daily tag:unit_test
43+
} else {
44+
echo " ==> No Previous Manifest file found, running dbt run fo release"
45+
dbt run --profiles-dir=${{ inputs.profiles-directory }} --target=${{ inputs.target }} --exclude tag:daily tag:unit_test
46+
}
4147
working-directory: ${{ inputs.working-directory }}
4248
env:
4349
snowflake-account: ${{ inputs.snowflake-account }}
@@ -49,10 +55,17 @@ runs:
4955

5056
- name: Run DBT with unit-tests
5157
if: inputs.target == 'unit-test'
52-
shell: bash
58+
shell: pwsh
5359
run: |
5460
dbt seed --profiles-dir=${{ inputs.profiles-directory }} --target=${{ inputs.target }}
55-
dbt run --profiles-dir=${{ inputs.profiles-directory }} --target=${{ inputs.target }} --exclude tag:daily
61+
if(Test-Path -Path ".\manifest.json" -PathType Leaf) {
62+
echo " ==> Previous Manifest file found, running dbt run for unit tests"
63+
dbt run -s state:modified --defer --state . --profiles-dir=${{ inputs.profiles-directory }} --target=${{ inputs.target }} --exclude tag:daily
64+
} else {
65+
echo " ==> No Previous Manifest file found, running dbt run for unit tests"
66+
dbt run --profiles-dir=${{ inputs.profiles-directory }} --target=${{ inputs.target }} --exclude tag:daily
67+
}
68+
5669
working-directory: ${{ inputs.working-directory }}
5770
env:
5871
snowflake-account: ${{ inputs.snowflake-account }}

‎tasks/dbt-docs-upload-azure-storage/action.yml renamed to ‎tasks/dbt-docs-publish/action.yml

+13-4
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
name: Upload Dbt Docs to Azure Storage
2-
description: Uploads the documents required for the Dbt docs to Azure Storage
1+
name: Publish Dbt Docs to Azure Storage
2+
description: Publish the dbt docs to Azure Storage
33
inputs:
44
azure_credentials:
55
description: The azure azure_credentials to be used
@@ -40,6 +40,12 @@ runs:
4040
run: |
4141
echo "BRANCH_NAME=$(echo ${GITHUB_HEAD_REF} | tr / -)" >> $GITHUB_ENV
4242
43+
- name: Get Manifest and Catalog Paths
44+
shell: bash
45+
run: |
46+
echo "MANIFEST_FILE=$(echo ${{ env.BRANCH_NAME }}/manifest.json)" >> $GITHUB_ENV
47+
echo "CATALOG_FILE=$(echo ${{ env.BRANCH_NAME }}/catalog.json)" >> $GITHUB_ENV
48+
4349
- name: Authenticate to Azure as a Service Principal
4450
uses: azure/login@v1
4551
with:
@@ -48,8 +54,11 @@ runs:
4854
- name: Find and replace the placeholders
4955
shell: pwsh
5056
run: |
51-
sed -i 's/<<BRANCH_NAME>>/${{ env.BRANCH_NAME }}/g' manifest.json
52-
sed -i 's/Welcome to the auto-generated documentation for your dbt project!/Welcome to the auto-generated documentation for your dbt project for the ${{ env.BRANCH_NAME }} branch!/g ' manifest.json
57+
sed -i 's|<<BRANCH_NAME>>|${{ env.BRANCH_NAME }}|g' manifest.json
58+
if("${{ inputs.branchBasedFolder }}" -eq "yes") {
59+
sed -i 's|manifest.json|${{ env.MANIFEST_FILE }}|g' index.html
60+
sed -i 's|catalog.json|${{ env.CATALOG_FILE }}|g' index.html
61+
}
5362
working-directory: ${{ inputs.working-directory }}/${{ inputs.target-directory }}
5463

5564

‎tasks/dbt-manifest-clone/action.yml

+50
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
name: Clone the Manifest for the cloned database
2+
description: Clones an existing dbt manifest.json for the database
3+
inputs:
4+
azure_credentials:
5+
description: The azure azure_credentials to be used
6+
required: true
7+
resource-group:
8+
description: The resource group where the storage account is located
9+
required: true
10+
storage-account:
11+
description: The name of the storage account
12+
required: true
13+
snowflake-source-database:
14+
description: The name of the database to get the manifest for
15+
required: true
16+
snowflake-target-database:
17+
description: The name of the database to set the manifest for
18+
required: true
19+
20+
runs:
21+
using: "composite"
22+
steps:
23+
- name: Authenticate to Azure as a Service Principal
24+
uses: azure/login@v1
25+
with:
26+
creds: ${{ inputs.azure_credentials }}
27+
28+
- name: Clone Manifest file
29+
uses: azure/CLI@v1
30+
with:
31+
inlineScript: |
32+
if (az storage blob exists --container-name 'manifests' --name ${{ inputs.snowflake-source-database }}_manifest.json --account-name ${{ inputs.storage-account }} --auth-mode login)
33+
then
34+
echo " ==> DBT manifest exists in the container, copying file"
35+
az storage blob copy start \
36+
--account-name ${{ inputs.storage-account }} \
37+
--destination-container 'manifests' \
38+
--destination-blob '${{ inputs.snowflake-target-database }}_manifest.json' \
39+
--source-account-name ${{ inputs.storage-account }} \
40+
--source-container 'manifests' \
41+
--source-blob '${{ inputs.snowflake-source-database }}_manifest.json' \
42+
--auth-mode login
43+
fi
44+
45+
# Azure logout
46+
- name: logout
47+
shell: bash
48+
run: |
49+
az logout
50+
if: always()

‎tasks/dbt-manifest-destroy/action.yml

+45
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
name: Removes the Manifest for the database
2+
description: Removes an existing dbt manifest.json from the storage account
3+
inputs:
4+
azure_credentials:
5+
description: The azure azure_credentials to be used
6+
required: true
7+
resource-group:
8+
description: The resource group where the storage account is located
9+
required: true
10+
storage-account:
11+
description: The name of the storage account
12+
required: true
13+
snowflake-target-database:
14+
description: The name of the database to set the manifest for
15+
required: true
16+
17+
runs:
18+
using: "composite"
19+
steps:
20+
- name: Authenticate to Azure as a Service Principal
21+
uses: azure/login@v1
22+
with:
23+
creds: ${{ inputs.azure_credentials }}
24+
25+
- name: Clone Manifest file
26+
uses: azure/CLI@v1
27+
with:
28+
inlineScript: |
29+
if (az storage blob exists --container-name 'manifests' --name ${{ inputs.snowflake-target-database }}_manifest.json --account-name ${{ inputs.storage-account }} --auth-mode login)
30+
then
31+
echo " ==> DBT manifest exists in the container, copying file"
32+
az storage blob delete \
33+
--account-name ${{ inputs.storage-account }} \
34+
--container-name 'manifests' \
35+
--name '${{ inputs.snowflake-target-database }}_manifest.json' \
36+
--delete-snapshots include \
37+
--auth-mode login
38+
fi
39+
40+
# Azure logout
41+
- name: logout
42+
shell: bash
43+
run: |
44+
az logout
45+
if: always()

‎tasks/dbt-manifest-download/action.yml

+46
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
name: Download dbt manifest
2+
description: Downloads the dbt manifest.json file to the current working directory
3+
inputs:
4+
azure_credentials:
5+
description: The azure azure_credentials to be used
6+
required: true
7+
target-directory:
8+
description: The directory where the DBT output is located
9+
required: true
10+
working-directory:
11+
description: The directory where dbt is located
12+
required: true
13+
resource-group:
14+
description: The resource group where the storage account is located
15+
required: true
16+
storage-account:
17+
description: The name of the storage account
18+
required: true
19+
snowflake-target-database:
20+
description: The name of the database to get the manifest for
21+
required: true
22+
23+
runs:
24+
using: "composite"
25+
steps:
26+
- name: Authenticate to Azure as a Service Principal
27+
uses: azure/login@v1
28+
with:
29+
creds: ${{ inputs.azure_credentials }}
30+
31+
- name: Download DBT Docs
32+
uses: azure/CLI@v1
33+
with:
34+
inlineScript: |
35+
if (az storage blob exists --container-name 'manifests' --name ${{ inputs.snowflake-target-database }}_manifest.json --account-name ${{ inputs.storage-account }} --auth-mode login)
36+
then
37+
echo " ==> DBT manifest exists in the container, downloading"
38+
az storage blob download --account-name ${{ inputs.storage-account }} --auth-mode login --container 'manifests' --name '${{ inputs.snowflake-target-database }}_manifest.json' --file '${{ inputs.working-directory }}/manifest.json'
39+
fi
40+
41+
# Azure logout
42+
- name: logout
43+
shell: bash
44+
run: |
45+
az logout
46+
if: always()

‎tasks/dbt-manifest-upload/action.yml

+55
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
name: Upload Manifest File to Azure Storage
2+
description: Uploads the dbt manifest.json to Azure Storage
3+
inputs:
4+
azure_credentials:
5+
description: The azure azure_credentials to be used
6+
required: true
7+
target-directory:
8+
description: The directory where the DBT output is located
9+
required: true
10+
working-directory:
11+
description: The directory where dbt is located
12+
required: true
13+
resource-group:
14+
description: The resource group where the storage account is located
15+
required: true
16+
storage-account:
17+
description: The name of the storage account
18+
required: true
19+
snowflake-target-database:
20+
description: The name of the database to get the manifest for
21+
required: true
22+
23+
runs:
24+
using: "composite"
25+
steps:
26+
- name: Authenticate to Azure as a Service Principal
27+
uses: azure/login@v1
28+
with:
29+
creds: ${{ inputs.azure_credentials }}
30+
31+
- name: Upload DBT Manifest for database
32+
uses: azure/CLI@v1
33+
with:
34+
inlineScript: |
35+
az storage blob upload --account-name ${{ inputs.storage-account }} \
36+
--auth-mode login \
37+
-f '${{ inputs.working-directory }}/${{ inputs.target-directory }}/manifest.json' \
38+
-c 'manifests' -n '${{ env.snowflake-target-database }}_manifest.json' \
39+
--content-type 'application/json' \
40+
--overwrite
41+
42+
- name: add the IP address of the agent to the allowed list
43+
uses: azure/CLI@v1
44+
if: always()
45+
with:
46+
inlineScript: |
47+
echo "Remove this IP address from the rules: ${{ env.AGENT_IP}}"
48+
az storage account network-rule remove --resource-group ${{ inputs.resource-group }} --account-name ${{ inputs.storage-account }} --ip-address ${{ env.AGENT_IP}}
49+
50+
# Azure logout
51+
- name: logout
52+
shell: bash
53+
run: |
54+
az logout
55+
if: always()

0 commit comments

Comments
 (0)