Skip to content

Commit 15a73f1

Browse files
jonhopper-dataengineersjonhopper-dataengineers
authored
Feature/dbt apply (#6)
- split out dbt compile and dbt seed into their own tasks - updated python version to 3.12 - updated azure logins to use ODIC as an option for the authentication - Upgrade upload-artifact action to version 4 in YAML files
1 parent 77682a7 commit 15a73f1

12 files changed

Lines changed: 205 additions & 64 deletions

File tree

‎setup/aws/action.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ runs:
2020
- name: install Python
2121
uses: "actions/setup-python@v5"
2222
with:
23-
python-version: '3.10'
23+
python-version: '3.12'
2424

2525
- name: setup environment
2626
shell: bash

‎setup/dbt/action.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ runs:
1212
- name: install Python
1313
uses: "actions/setup-python@v5"
1414
with:
15-
python-version: '3.10'
15+
python-version: '3.12'
1616

1717
- name: install components
1818
shell: bash

‎setup/schemachange/action.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ runs:
1212
- name: install Python
1313
uses: "actions/setup-python@v5"
1414
with:
15-
python-version: 3.8
15+
python-version: 3.12
1616

1717
- name: install components
1818
shell: bash

‎tasks/database-permissions/action.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,7 @@ runs:
103103
$fileName = "./schemaresults.json"
104104
105105
$schemaObjects = Get-Content -Raw -Path $fileName | ConvertFrom-Json
106-
106+
107107
$schemaObjects | ForEach-Object {
108108
$schema= $_.SCHEMA_NAME.toLower()
109109
$statements.Add(-JOIN("GRANT OWNERSHIP ON SCHEMA ", $schema , " TO ROLE ",$new_role, " REVOKE CURRENT GRANTS;"))
@@ -136,7 +136,7 @@ runs:
136136
$fileName = "./maskingpolicies.json"
137137
138138
$schemaObjects = Get-Content -Raw -Path $fileName | ConvertFrom-Json
139-
139+
140140
$schemaObjects | ForEach-Object {
141141
$schema= $_.SCHEMA_NAME.toLower()
142142
$policy= $_.NAME.toLower()
@@ -152,7 +152,7 @@ runs:
152152

153153

154154
- name: Archive Permission Details
155-
uses: actions/upload-artifact@v3
155+
uses: actions/upload-artifact@v4
156156
if: ${{ inputs.archive-outputs == 'true' }}
157157
with:
158158
name: permissions

‎tasks/dbt-apply/action.yml‎

Lines changed: 31 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -35,11 +35,19 @@ inputs:
3535
description: Additional flags to pass to the dbt command
3636
required: false
3737
default: ''
38+
run_compile:
39+
description: Run dbt compile
40+
required: false
41+
default: true
42+
run_seed:
43+
description: Run dbt seed
44+
required: false
45+
default: true
3846

3947
runs:
4048
using: "composite"
4149
steps:
42-
- name: Clean target
50+
- name: Clean targets
4351
shell: pwsh
4452
run: |
4553
if(Test-Path -Path ".\target\partial_parse.msgpack") {
@@ -50,19 +58,11 @@ runs:
5058
}
5159
working-directory: ${{ inputs.working-directory }}
5260

53-
- name: Run DBT without Unit-tets
54-
if: inputs.target != 'unit-test'
61+
- name: dbt Compile
62+
if: ${{ inputs.run_compile == 'true' }}
5563
shell: pwsh
5664
run: |
57-
dbt compile --profiles-dir=${{ inputs.profiles-directory }} --target=${{ inputs.target }} --exclude tag:unit_test
58-
dbt seed --profiles-dir=${{ inputs.profiles-directory }} --target=${{ inputs.target }} --exclude path:seeds/unit_tests --full-refresh
59-
if(Test-Path -Path ".\manifest.json" -PathType Leaf) {
60-
echo " ==> Previous Manifest file found, running dbt run for release"
61-
dbt run -s state:modified --defer --state . --profiles-dir=${{ inputs.profiles-directory }} --target=${{ inputs.target }} --exclude tag:unit_test ${{ inputs.additional_flag }}
62-
} else {
63-
echo " ==> No Previous Manifest file found, running dbt run fo release"
64-
dbt run --profiles-dir=${{ inputs.profiles-directory }} --target=${{ inputs.target }} --exclude tag:unit_test ${{ inputs.additional_flag }}
65-
}
65+
dbt compile --profiles-dir=${{ inputs.profiles-directory }} --target=${{ inputs.target }}
6666
working-directory: ${{ inputs.working-directory }}
6767
env:
6868
snowflake-account: ${{ inputs.snowflake-account }}
@@ -73,20 +73,31 @@ runs:
7373
snowflake-authenticator-value: ${{ inputs.snowflake-authenticator-value }}
7474
snowflake-privatekey-passphrase: ${{ inputs.snowflake-privatekey-passphrase }}
7575

76-
- name: Run DBT with unit-tests
77-
if: inputs.target == 'unit-test'
76+
- name: dbt Seed
77+
if: ${{ inputs.run_seed == 'true' }}
7878
shell: pwsh
7979
run: |
80-
dbt compile --profiles-dir=${{ inputs.profiles-directory }} --target=${{ inputs.target }} --exclude tag:unit_test
8180
dbt seed --profiles-dir=${{ inputs.profiles-directory }} --target=${{ inputs.target }} --full-refresh
81+
working-directory: ${{ inputs.working-directory }}
82+
env:
83+
snowflake-account: ${{ inputs.snowflake-account }}
84+
snowflake-username: ${{ inputs.snowflake-username }}
85+
snowflake-role: ${{ inputs.snowflake-role }}
86+
snowflake-warehouse: ${{ inputs.snowflake-warehouse }}
87+
snowflake-target-database: ${{ inputs.snowflake-target-database }}
88+
snowflake-authenticator-value: ${{ inputs.snowflake-authenticator-value }}
89+
snowflake-privatekey-passphrase: ${{ inputs.snowflake-privatekey-passphrase }}
90+
91+
- name: dbt Run
92+
shell: pwsh
93+
run: |
8294
if(Test-Path -Path ".\manifest.json" -PathType Leaf) {
83-
echo " ==> Previous Manifest file found, running dbt run for unit tests"
95+
echo " ==> Previous Manifest file found, running dbt run for release"
8496
dbt run -s state:modified --defer --state . --profiles-dir=${{ inputs.profiles-directory }} --target=${{ inputs.target }} ${{ inputs.additional_flag }}
8597
} else {
86-
echo " ==> No Previous Manifest file found, running dbt run for unit tests"
87-
dbt run --profiles-dir=${{ inputs.profiles-directory }} --target=${{ inputs.target }} ${{ inputs.additional_flag }}
98+
echo " ==> No Previous Manifest file found, running dbt run fo release"
99+
dbt run --profiles-dir=${{ inputs.profiles-directory }} --target=${{ inputs.target }} ${{ inputs.additional_flag }}
88100
}
89-
90101
working-directory: ${{ inputs.working-directory }}
91102
env:
92103
snowflake-account: ${{ inputs.snowflake-account }}
@@ -95,4 +106,4 @@ runs:
95106
snowflake-warehouse: ${{ inputs.snowflake-warehouse }}
96107
snowflake-target-database: ${{ inputs.snowflake-target-database }}
97108
snowflake-authenticator-value: ${{ inputs.snowflake-authenticator-value }}
98-
snowflake-privatekey-passphrase: ${{ inputs.snowflake-privatekey-passphrase }}
109+
snowflake-privatekey-passphrase: ${{ inputs.snowflake-privatekey-passphrase }}

‎tasks/dbt-docs-generate/action.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ runs:
5454
snowflake-privatekey-passphrase: ${{ inputs.snowflake-privatekey-passphrase }}
5555

5656
- name: Archive Dbt Docs
57-
uses: actions/upload-artifact@v3
57+
uses: actions/upload-artifact@v4
5858
if: ${{ inputs.archive-outputs == 'true' }}
5959
with:
6060
name: dbt-docs

‎tasks/dbt-docs-publish/action.yml‎

Lines changed: 34 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
name: Publish Dbt Docs to Azure Storage
22
description: Publish the dbt docs to Azure Storage
33
inputs:
4-
azure_credentials:
5-
description: The azure azure_credentials to be used
6-
required: true
4+
use-azure-federated-login:
5+
description: Specifies if the action should use Azure Federated Login or Azure Service Principal ClientID/ClientSecret
6+
default: false
77
target-directory:
88
description: The directory where the DBT output is located
99
required: true
@@ -28,7 +28,22 @@ inputs:
2828
description: The number of seconds to wait before removing the IP address from the firewall
2929
required: false
3030
default: 30
31+
azure-credentials:
32+
description: The Azure credentials to use for authentication
33+
required: false
34+
azure-client-id:
35+
description: The Azure client ID to use for authentication
36+
required: false
37+
azure-tenant-id:
38+
description: The Azure tenant ID to use for authentication
39+
required: false
40+
azure-subscription-id:
41+
description: The Azure subscription ID to use for authentication
42+
required: false
3143

44+
permissions:
45+
id-token: write
46+
contents: read
3247
runs:
3348
using: "composite"
3449
steps:
@@ -51,9 +66,20 @@ runs:
5166
echo "CATALOG_FILE=$(echo ${{ env.BRANCH_NAME }}/catalog.json)" >> $GITHUB_ENV
5267
5368
- name: Authenticate to Azure as a Service Principal
54-
uses: azure/login@v1
69+
uses: azure/login@v2
70+
if: ${{ inputs.use-azure-federated-login == 'false' }}
71+
with:
72+
creds: ${{ inputs.azure-credentials }}
73+
74+
- name: Authenticate to Azure as a Service Principal (ODIC)
75+
uses: azure/login@v2
76+
if: ${{ inputs.use-azure-federated-login == 'true' }}
5577
with:
56-
creds: ${{ inputs.azure_credentials }}
78+
client-id: ${{ inputs.azure-client-id }}
79+
tenant-id: ${{ inputs.azure-tenant-id }}
80+
subscription-id: ${{ inputs.azure-subscription-id }}
81+
enable-AzPSSession: true
82+
5783

5884
- name: Find and replace the placeholders
5985
shell: pwsh
@@ -67,7 +93,7 @@ runs:
6793

6894

6995
- name: Upload DBT Docs to subfolder
70-
uses: azure/CLI@v1
96+
uses: azure/CLI@v2
7197
if: inputs.branchBasedFolder == 'yes'
7298
with:
7399
inlineScript: |
@@ -83,7 +109,7 @@ runs:
83109
az storage blob upload --account-name ${{ inputs.storage-account }} --auth-mode login -f '${{ inputs.working-directory }}/${{ inputs.target-directory }}/index.html' -c '${{inputs.container}}' -n '${{ env.BRANCH_NAME }}/index.html' --content-type 'text/html' --overwrite
84110
85111
- name: Upload DBT Docs
86-
uses: azure/CLI@v1
112+
uses: azure/CLI@v2
87113
if: inputs.branchBasedFolder == 'no'
88114
with:
89115
inlineScript: |
@@ -98,7 +124,7 @@ runs:
98124
az storage blob upload --account-name ${{ inputs.storage-account }} -f '${{ inputs.working-directory }}/${{ inputs.target-directory }}/index.html' -c '${{inputs.container}}' -n 'index.html' --content-type 'text/html' --overwrite
99125
100126
- name: add the IP address of the agent to the allowed list
101-
uses: azure/CLI@v1
127+
uses: azure/CLI@v2
102128
if: always()
103129
with:
104130
inlineScript: |

‎tasks/dbt-manifest-clone/action.yml‎

Lines changed: 33 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
name: Clone the Manifest for the cloned database
22
description: Clones an existing dbt manifest.json for the database
33
inputs:
4-
azure_credentials:
5-
description: The azure azure_credentials to be used
6-
required: true
4+
use-azure-federated-login:
5+
description: Specifies if the action should use Azure Federated Login or Azure Service Principal ClientID/ClientSecret
6+
default: false
77
resource-group:
88
description: The resource group where the storage account is located
99
required: true
@@ -16,17 +16,43 @@ inputs:
1616
snowflake-target-database:
1717
description: The name of the database to set the manifest for
1818
required: true
19+
azure-credentials:
20+
description: The Azure credentials to use for authentication
21+
required: false
22+
azure-client-id:
23+
description: The Azure client ID to use for authentication
24+
required: false
25+
azure-tenant-id:
26+
description: The Azure tenant ID to use for authentication
27+
required: false
28+
azure-subscription-id:
29+
description: The Azure subscription ID to use for authentication
30+
required: false
1931

32+
permissions:
33+
id-token: write
34+
contents: read
2035
runs:
2136
using: "composite"
2237
steps:
2338
- name: Authenticate to Azure as a Service Principal
24-
uses: azure/login@v1
39+
uses: azure/login@v2
40+
if: ${{ inputs.use-azure-federated-login == 'false' }}
41+
with:
42+
creds: ${{ inputs.azure-credentials }}
43+
44+
- name: Authenticate to Azure as a Service Principal (ODIC)
45+
uses: azure/login@v2
46+
if: ${{ inputs.use-azure-federated-login == 'true' }}
2547
with:
26-
creds: ${{ inputs.azure_credentials }}
48+
client-id: ${{ inputs.azure-client-id }}
49+
tenant-id: ${{ inputs.azure-tenant-id }}
50+
subscription-id: ${{ inputs.azure-subscription-id }}
51+
enable-AzPSSession: true
52+
2753

2854
- name: Clone Manifest file
29-
uses: azure/CLI@v1
55+
uses: azure/CLI@v2
3056
with:
3157
inlineScript: |
3258
if $(az storage blob exists --container-name 'manifests' --name ${{ inputs.snowflake-source-database }}_manifest.json --account-name ${{ inputs.storage-account }} --query exists --auth-mode login)
@@ -41,7 +67,7 @@ runs:
4167
--source-blob '${{ inputs.snowflake-source-database }}_manifest.json' \
4268
--auth-mode login
4369
fi
44-
70+
4571
# Azure logout
4672
- name: logout
4773
shell: bash

‎tasks/dbt-manifest-destroy/action.yml‎

Lines changed: 34 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
name: Removes the Manifest for the database
22
description: Removes an existing dbt manifest.json from the storage account
33
inputs:
4-
azure_credentials:
5-
description: The azure azure_credentials to be used
6-
required: true
4+
use-azure-federated-login:
5+
description: Specifies if the action should use Azure Federated Login or Azure Service Principal ClientID/ClientSecret
6+
default: false
77
resource-group:
88
description: The resource group where the storage account is located
99
required: true
@@ -13,17 +13,43 @@ inputs:
1313
snowflake-target-database:
1414
description: The name of the database to set the manifest for
1515
required: true
16+
azure-credentials:
17+
description: The Azure credentials to use for authentication
18+
required: false
19+
azure-client-id:
20+
description: The Azure client ID to use for authentication
21+
required: false
22+
azure-tenant-id:
23+
description: The Azure tenant ID to use for authentication
24+
required: false
25+
azure-subscription-id:
26+
description: The Azure subscription ID to use for authentication
27+
required: false
1628

29+
permissions:
30+
id-token: write
31+
contents: read
1732
runs:
1833
using: "composite"
1934
steps:
2035
- name: Authenticate to Azure as a Service Principal
21-
uses: azure/login@v1
36+
uses: azure/login@v2
37+
if: ${{ inputs.use-azure-federated-login == 'false' }}
38+
with:
39+
creds: ${{ inputs.azure-credentials }}
40+
41+
- name: Authenticate to Azure as a Service Principal (ODIC)
42+
uses: azure/login@v2
43+
if: ${{ inputs.use-azure-federated-login == 'true' }}
2244
with:
23-
creds: ${{ inputs.azure_credentials }}
45+
client-id: ${{ inputs.azure-client-id }}
46+
tenant-id: ${{ inputs.azure-tenant-id }}
47+
subscription-id: ${{ inputs.azure-subscription-id }}
48+
enable-AzPSSession: true
49+
2450

2551
- name: Clone Manifest file
26-
uses: azure/CLI@v1
52+
uses: azure/CLI@v2
2753
with:
2854
inlineScript: |
2955
if $(az storage blob exists --container-name 'manifests' --name ${{ inputs.snowflake-target-database }}_manifest.json --account-name ${{ inputs.storage-account }} --query exists --auth-mode login)
@@ -34,9 +60,9 @@ runs:
3460
--container-name 'manifests' \
3561
--name '${{ inputs.snowflake-target-database }}_manifest.json' \
3662
--delete-snapshots include \
37-
--auth-mode login
63+
--auth-mode login
3864
fi
39-
65+
4066
# Azure logout
4167
- name: logout
4268
shell: bash

0 commit comments

Comments
 (0)