From ff1086d94d2f0308026a2a1ad62f4306f702de3e Mon Sep 17 00:00:00 2001 From: fivetran-catfritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Wed, 16 Jul 2025 14:20:51 -0500 Subject: [PATCH 1/3] Q2 FY26: Apply automated update. --- .buildkite/hooks/pre-command | 3 +- .buildkite/pipeline.yml | 1 + .../maintainer_pull_request_template.md | 9 ++- .github/workflows/auto-release.yml | 13 ++-- .github/workflows/generate-docs.yml | 13 ++++ .gitignore | 74 ++++++++++++++++++- CHANGELOG.md | 19 +++++ README.md | 10 +-- integration_tests/ci/sample.profiles.yml | 4 +- integration_tests/dbt_project.yml | 3 +- 10 files changed, 127 insertions(+), 22 deletions(-) create mode 100644 .github/workflows/generate-docs.yml diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 04c85c0..0e60322 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -21,4 +21,5 @@ export CI_SNOWFLAKE_DBT_USER=$(gcloud secrets versions access latest --secret="C export CI_SNOWFLAKE_DBT_WAREHOUSE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_WAREHOUSE" --project="dbt-package-testing-363917") export CI_DATABRICKS_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HOST" --project="dbt-package-testing-363917") export CI_DATABRICKS_DBT_HTTP_PATH=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HTTP_PATH" --project="dbt-package-testing-363917") -export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917") \ No newline at end of file +export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917") +export CI_DATABRICKS_DBT_CATALOG=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_CATALOG" --project="dbt-package-testing-363917") diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 06d3b71..8314527 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -69,5 +69,6 @@ steps: - "CI_DATABRICKS_DBT_HOST" - "CI_DATABRICKS_DBT_HTTP_PATH" - "CI_DATABRICKS_DBT_TOKEN" + - "CI_DATABRICKS_DBT_CATALOG" commands: | bash .buildkite/scripts/run_models.sh databricks \ No newline at end of file diff --git a/.github/PULL_REQUEST_TEMPLATE/maintainer_pull_request_template.md b/.github/PULL_REQUEST_TEMPLATE/maintainer_pull_request_template.md index 99c450c..0899dc6 100644 --- a/.github/PULL_REQUEST_TEMPLATE/maintainer_pull_request_template.md +++ b/.github/PULL_REQUEST_TEMPLATE/maintainer_pull_request_template.md @@ -12,13 +12,15 @@ Before marking this PR as "ready for review": ## PR Overview **Package version introduced in this PR:** - +- + **This PR addresses the following Issue/Feature(s):** +- **Summary of changes:** - +- ### Submission Checklist - [ ] Alignment meeting with the reviewer (if needed) @@ -27,9 +29,10 @@ Before marking this PR as "ready for review": - [ ] **Validation Steps:** Check for unintentional effects (e.g., add/run consistency & integrity tests) - [ ] **Testing Instructions:** Confirm the change addresses the issue(s) - [ ] **Focus Areas:** Complex logic or queries that need extra attention +- [ ] Merge any relevant open PRs into this PR ### Changelog - [ ] Draft changelog for PR -- [ ] Final changelog for release review \ No newline at end of file +- [ ] Final changelog for release review diff --git a/.github/workflows/auto-release.yml b/.github/workflows/auto-release.yml index 8ed5853..76d6944 100644 --- a/.github/workflows/auto-release.yml +++ b/.github/workflows/auto-release.yml @@ -3,11 +3,12 @@ on: pull_request: types: - closed - branches: - - main + - labeled jobs: - call-workflow-passing-data: - if: github.event.pull_request.merged - uses: fivetran/dbt_package_automations/.github/workflows/auto-release.yml@main - secrets: inherit \ No newline at end of file + release: + if: | + (github.event.pull_request.merged == true && github.event.pull_request.base.ref == 'main') || + github.event.label.name == 'pre-release' + uses: fivetran/dbt_package_automations/.github/workflows/auto-release.yml@feature/pre-release-support + secrets: inherit diff --git a/.github/workflows/generate-docs.yml b/.github/workflows/generate-docs.yml new file mode 100644 index 0000000..1d8c614 --- /dev/null +++ b/.github/workflows/generate-docs.yml @@ -0,0 +1,13 @@ +name: 'generate dbt docs' +on: + pull_request: + types: + - labeled + +jobs: + generate-docs: + if: github.event.label.name == 'docs:ready' + uses: fivetran/dbt_package_automations/.github/workflows/generate-docs.yml@main + secrets: inherit + with: + schema_var_name: github_schema \ No newline at end of file diff --git a/.gitignore b/.gitignore index de963f0..06f4e77 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,73 @@ - -target/ +# dbt +**/package-lock.yml +package-lock.yml +.dbt/ dbt_modules/ +dbt_packages/ logs/ +profiles.yml +target/ +*.log + +# IDE files +.idea/ +.vscode/ +*~ +*.swp +*.swo + +# Jupyter Notebook +.ipynb_checkpoints + +# OS generated files +**/.DS_Store .DS_Store -dbt_packages/ -env/ \ No newline at end of file +.Spotlight-V100 +.Trashes +._* +Thumbs.db +ehthumbs.db + +# Python +*.egg +*.egg-info/ +*.py[cod] +*.so +*$py.class +.Python +__pycache__/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.env +.installed.cfg +lib/ +lib64/ +MANIFEST +parts/ +sdist/ +var/ +wheels/ + +# Secrets and credentials +.env.* +.secrets +credentials.json +service-account.json + +# Temporary files +.cache/ +*.temp +*.tmp + +# Virtual environments +.conda/ +.env +.venv +ENV/ +env/ +env.bak/ +venv/ +venv.bak/ diff --git a/CHANGELOG.md b/CHANGELOG.md index db7b12d..cd0872c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,22 @@ +[PR #46](https://github.com/fivetran/dbt_github_source/pull/46) includes the following updates: + +### Under the Hood - July 2025 Updates + +- Updated conditions in `.github/workflows/auto-release.yml`. +- Added `.github/workflows/generate-docs.yml`. +- Added `+docs: show: False` to `integration_tests/dbt_project.yml`. +- Migrated `flags` (e.g., `send_anonymous_usage_stats`, `use_colors`) from `sample.profiles.yml` to `integration_tests/dbt_project.yml`. +- Updated `maintainer_pull_request_template.md` with improved checklist. +- Refreshed README tag block: + - Standardized Quickstart-compatible badge set + - Left-aligned and positioned below the H1 title. +- Updated Python image version to `3.10.13` in `pipeline.yml`. +- Added `CI_DATABRICKS_DBT_CATALOG` to: + - `.buildkite/hooks/pre-command` (as an export) + - `pipeline.yml` (under the `environment` block, after `CI_DATABRICKS_DBT_TOKEN`) +- Added `certifi==2025.1.31` to `requirements.txt` (if missing). +- Updated `.gitignore` to exclude additional DBT, Python, and system artifacts. + # dbt_github_source v0.9.1 [PR #45](https://github.com/fivetran/dbt_github_source/pull/45) includes the following updates: diff --git a/README.md b/README.md index 638da0d..ddfe0aa 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ href="https://github.com/fivetran/dbt_github_source/blob/main/LICENSE"> - + @@ -87,25 +87,25 @@ models: #### Change the source table references If an individual source table has a different name than the package expects, add the table name as it appears in your destination to the respective variable: > IMPORTANT: See this project's [`dbt_project.yml`](https://github.com/fivetran/dbt_github_source/blob/main/dbt_project.yml) variable declarations to see the expected names. - + ```yml vars: github__identifier: your_table_name ``` - + ### (Optional) Step 5: Orchestrate your models with Fivetran Transformations for dbt Core™
Expand for more details Fivetran offers the ability for you to orchestrate your dbt project through [Fivetran Transformations for dbt Core™](https://fivetran.com/docs/transformations/dbt). Learn how to set up your project for orchestration through Fivetran in our [Transformations for dbt Core™ setup guides](https://fivetran.com/docs/transformations/dbt#setupguide). - +
### Does this package have dependencies? This dbt package is dependent on the following dbt packages. These dependencies are installed by default within this package. For more information on the following packages, refer to the [dbt hub](https://hub.getdbt.com/) site. > IMPORTANT: If you have any of these dependent packages in your own `packages.yml` file, we highly recommend that you remove them from your root `packages.yml` to avoid package version conflicts. - + ```yml packages: - package: fivetran/fivetran_utils diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml index cea8e1e..77f7bb5 100644 --- a/integration_tests/ci/sample.profiles.yml +++ b/integration_tests/ci/sample.profiles.yml @@ -41,10 +41,10 @@ integration_tests: schema: github_source_integration_tests threads: 8 databricks: - catalog: null + catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}" host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}" http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}" schema: github_source_integration_tests - threads: 2 + threads: 8 token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}" type: databricks \ No newline at end of file diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 301c8c5..6599008 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -63,4 +63,5 @@ dispatch: search_order: ['spark_utils', 'dbt_utils'] flags: - send_anonymous_usage_stats: False \ No newline at end of file + send_anonymous_usage_stats: False + use_colors: True \ No newline at end of file From eb8f6bfeba40b1bf5a212d933cd95562801abf93 Mon Sep 17 00:00:00 2001 From: fivetran-catfritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Wed, 16 Jul 2025 14:29:26 -0500 Subject: [PATCH 2/3] Update README.md --- README.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/README.md b/README.md index ddfe0aa..31fe319 100644 --- a/README.md +++ b/README.md @@ -10,9 +10,6 @@ - -

## What does this dbt package do? From 5111f0f74df0b2161e07d6abf8280bc0267bc5b3 Mon Sep 17 00:00:00 2001 From: fivetran-catfritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Tue, 22 Jul 2025 10:57:58 -0500 Subject: [PATCH 3/3] Q2 FY26: Update auto-release workflow only. --- .github/workflows/auto-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/auto-release.yml b/.github/workflows/auto-release.yml index 76d6944..e0a7100 100644 --- a/.github/workflows/auto-release.yml +++ b/.github/workflows/auto-release.yml @@ -10,5 +10,5 @@ jobs: if: | (github.event.pull_request.merged == true && github.event.pull_request.base.ref == 'main') || github.event.label.name == 'pre-release' - uses: fivetran/dbt_package_automations/.github/workflows/auto-release.yml@feature/pre-release-support + uses: fivetran/dbt_package_automations/.github/workflows/auto-release.yml@main secrets: inherit