Skip to content

Commit 239b21f

Browse files
jonhopper-dataengineersjonhopper-dataengineers
committed
added blob upload and publish docs / test results as artifacts
1 parent d6c7115 commit 239b21f

File tree

3 files changed

+111
-4
lines changed

3 files changed

+111
-4
lines changed

‎tasks/dbt-docs-generate/action.yml

+3-3
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,6 @@ runs:
5050
with:
5151
name: dbt-docs
5252
path: |
53-
./dbt/target/catalog.json
54-
./dbt/target/index.html
55-
./dbt/target/manifest.json
53+
${{ inputs.working-directory }}/target/catalog.json
54+
${{ inputs.working-directory }}/target/index.html
55+
${{ inputs.working-directory }}/target/manifest.json

‎tasks/dbt-docs-upload-azure-storage/action.yml

+100
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
name: Upload Dbt Docs to Azure Storage
2+
description: Uploads the documents required for the Dbt docs to Azure Storage
3+
inputs:
4+
azure_credentials:
5+
description: The azure azure_credentials to be used
6+
required: true
7+
target-directory:
8+
description: The directory where the DBT output is located
9+
required: true
10+
working-directory:
11+
description: The directory where dbt is located
12+
required: true
13+
resource-group:
14+
description: The resource group where the storage account is located
15+
required: true
16+
storage-account:
17+
description: The name of the storage account
18+
required: true
19+
container:
20+
description: The name of the storage container
21+
required: false
22+
default: '$web'
23+
branchBasedFolder:
24+
description: The sub folder to add the files to in the storage account
25+
required: false
26+
default: 'yes'
27+
28+
runs:
29+
using: "composite"
30+
steps:
31+
- name: Get branch name (merge)
32+
if: github.event_name != 'pull_request'
33+
shell: bash
34+
run: |
35+
echo "BRANCH_NAME=$(echo ${GITHUB_REF#refs/heads/} | tr / -)" >> $GITHUB_ENV
36+
37+
- name: Get branch name (pull request)
38+
if: github.event_name == 'pull_request'
39+
shell: bash
40+
run: |
41+
echo "BRANCH_NAME=$(echo ${GITHUB_HEAD_REF} | tr / -)" >> $GITHUB_ENV
42+
43+
- name: Authenticate to Azure as a Service Principal
44+
uses: azure/login@v1
45+
with:
46+
creds: ${{ inputs.azure_credentials }}
47+
48+
- name: Find and replace the placeholders
49+
shell: pwsh
50+
run: |
51+
sed -i 's/<<BRANCH_NAME>>/${{ env.BRANCH_NAME }}/g' manifest.json
52+
sed -i 's/Welcome to the auto-generated documentation for your dbt project!/Welcome to the auto-generated documentation for your dbt project for the ${{ env.BRANCH_NAME }} branch!/g ' manifest.json
53+
working-directory: ${{ inputs.working-directory }}/${{ inputs.target-directory }}
54+
55+
56+
- name: Upload DBT Docs to subfolder
57+
uses: azure/CLI@v1
58+
if: inputs.branchBasedFolder == 'yes'
59+
with:
60+
inlineScript: |
61+
myIP=$(curl -sL http://ipinfo.io/json | jq -r '.ip')
62+
echo "Adding this IP address to rules: $myIP"
63+
echo "AGENT_IP=$(echo $myIP | tr / -)" >> $GITHUB_ENV
64+
az storage account network-rule add --resource-group ${{ inputs.resource-group }} --account-name ${{ inputs.storage-account }} --ip-address $myIP
65+
az storage account show -n ${{ inputs.storage-account }} --query networkRuleSet
66+
echo " ==> Sleep for 30 seconds to allow the firewall change to take "
67+
sleep 30
68+
az storage blob upload --account-name ${{ inputs.storage-account }} --auth-mode login -f '${{ inputs.working-directory }}/${{ inputs.target-directory }}/manifest.json' -c '${{inputs.container}}' -n '${{ env.BRANCH_NAME }}/manifest.json' --content-type 'application/json' --overwrite
69+
az storage blob upload --account-name ${{ inputs.storage-account }} --auth-mode login -f '${{ inputs.working-directory }}/${{ inputs.target-directory }}/catalog.json' -c '${{inputs.container}}' -n '${{ env.BRANCH_NAME }}/catalog.json' --content-type 'application/json' --overwrite
70+
az storage blob upload --account-name ${{ inputs.storage-account }} --auth-mode login -f '${{ inputs.working-directory }}/${{ inputs.target-directory }}/index.html' -c '${{inputs.container}}' -n '${{ env.BRANCH_NAME }}/index.html' --content-type 'text/html' --overwrite
71+
72+
- name: Upload DBT Docs
73+
uses: azure/CLI@v1
74+
if: inputs.branchBasedFolder == 'no'
75+
with:
76+
inlineScript: |
77+
myIP=$(curl -sL http://ipinfo.io/json | jq -r '.ip')
78+
echo "Adding this IP address to rules: $myIP"
79+
echo "AGENT_IP=$(echo $myIP | tr / -)" >> $GITHUB_ENV
80+
az storage account network-rule add --resource-group ${{ inputs.resource-group }} --account-name ${{ inputs.storage-account }} --ip-address $myIP
81+
echo " ==> Sleep for 30 seconds to allow the firewall change to take "
82+
sleep 30
83+
az storage blob upload --account-name ${{ inputs.storage-account }} -f '${{ inputs.working-directory }}/${{ inputs.target-directory }}/manifest.json' -c '${{inputs.container}}' -n 'manifest.json' --content-type 'application/json' --overwrite
84+
az storage blob upload --account-name ${{ inputs.storage-account }} -f '${{ inputs.working-directory }}/${{ inputs.target-directory }}/catalog.json' -c '${{inputs.container}}' -n 'catalog.json' --content-type 'application/json' --overwrite
85+
az storage blob upload --account-name ${{ inputs.storage-account }} -f '${{ inputs.working-directory }}/${{ inputs.target-directory }}/index.html' -c '${{inputs.container}}' -n 'index.html' --content-type 'text/html' --overwrite
86+
87+
- name: add the IP address of the agent to the allowed list
88+
uses: azure/CLI@v1
89+
if: always()
90+
with:
91+
inlineScript: |
92+
echo "Remove this IP address from the rules: ${{ env.AGENT_IP}}"
93+
az storage account network-rule remove --resource-group ${{ inputs.resource-group }} --account-name ${{ inputs.storage-account }} --ip-address ${{ env.AGENT_IP}}
94+
95+
# Azure logout
96+
- name: logout
97+
shell: bash
98+
run: |
99+
az logout
100+
if: always()

‎tasks/dbt-test/action.yml

+8-1
Original file line numberDiff line numberDiff line change
@@ -58,4 +58,11 @@ runs:
5858
snowflake-role: ${{ inputs.snowflake-role }}
5959
snowflake-warehouse: ${{ inputs.snowflake-warehouse }}
6060
snowflake-target-database: ${{ inputs.snowflake-target-database }}
61-
snowflake-password: ${{ inputs.snowflake-password }}
61+
snowflake-password: ${{ inputs.snowflake-password }}
62+
63+
- name: Archive Dbt Test Results
64+
uses: actions/upload-artifact@v3
65+
with:
66+
name: dbt-test-results
67+
path:
68+
${{ inputs.working-directory }}/target/run_results.json

0 commit comments

Comments
 (0)