From 400714945bfc325ea14774f4b3e2eaf2f3ba97e6 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Tue, 31 Mar 2026 16:46:51 +0200 Subject: [PATCH 01/17] Evaluate hardened runner for CI Co-authored-by: Isaac --- .github/workflows/push.yml | 4 ++-- Makefile | 23 ++++++++++++----------- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 78521c6d12..2efb2c41d0 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -88,8 +88,8 @@ jobs: os: - name: linux runner: - group: databricks-protected-runner-group-large - labels: linux-ubuntu-latest-large + group: databricks-eng-protected-runner-group-hardened-optin + labels: linux-ubuntu-latest-hardened - name: windows runner: diff --git a/Makefile b/Makefile index b31bb73d08..22bf8b907c 100644 --- a/Makefile +++ b/Makefile @@ -2,11 +2,16 @@ default: checks fmt lint # Default packages to test (all) -TEST_PACKAGES = ./acceptance/internal ./libs/... ./internal/... ./cmd/... ./bundle/... ./experimental/ssh/... . +# TODO: restore after runner migration +# TEST_PACKAGES = ./acceptance/internal ./libs/... ./internal/... ./cmd/... ./bundle/... ./experimental/ssh/... . +TEST_PACKAGES = . # Default acceptance test filter (all) ACCEPTANCE_TEST_FILTER = "" +# TODO: restore after runner migration +TEST_RUN_FILTER = -run TestCommandsDontUseUnderscoreInName + GO_TOOL ?= go tool -modfile=tools/go.mod GOTESTSUM_FORMAT ?= pkgname-and-test-fails GOTESTSUM_CMD ?= ${GO_TOOL} gotestsum --format ${GOTESTSUM_FORMAT} --no-summary=skipped --jsonfile test-output.json --rerun-fails @@ -84,15 +89,16 @@ test-slow: test-slow-unit test-slow-acc .PHONY: test-unit test-unit: - ${GOTESTSUM_CMD} --packages "${TEST_PACKAGES}" -- -timeout=${LOCAL_TIMEOUT} -short + ${GOTESTSUM_CMD} --packages "${TEST_PACKAGES}" -- -timeout=${LOCAL_TIMEOUT} -short ${TEST_RUN_FILTER} .PHONY: test-slow-unit test-slow-unit: - ${GOTESTSUM_CMD} --packages "${TEST_PACKAGES}" -- -timeout=${LOCAL_TIMEOUT} + ${GOTESTSUM_CMD} --packages "${TEST_PACKAGES}" -- -timeout=${LOCAL_TIMEOUT} ${TEST_RUN_FILTER} +# TODO: restore after runner migration .PHONY: test-acc test-acc: - ${GOTESTSUM_CMD} --packages ./acceptance/... -- -timeout=${LOCAL_TIMEOUT} -short -run ${ACCEPTANCE_TEST_FILTER} + @echo "Skipping acceptance tests during runner migration" .PHONY: test-slow-acc test-slow-acc: @@ -125,15 +131,10 @@ test-update-all: test-update test-update-aws slowest: ${GO_TOOL} gotestsum tool slowest --jsonfile test-output.json --threshold 1s --num 50 +# TODO: restore after runner migration .PHONY: cover cover: - rm -fr ./acceptance/build/cover/ - VERBOSE_TEST=1 ${GOTESTSUM_CMD} --packages "${TEST_PACKAGES}" -- -coverprofile=coverage.txt -timeout=${LOCAL_TIMEOUT} - VERBOSE_TEST=1 CLI_GOCOVERDIR=build/cover ${GOTESTSUM_CMD} --packages ./acceptance/... -- -timeout=${LOCAL_TIMEOUT} -run ${ACCEPTANCE_TEST_FILTER} - rm -fr ./acceptance/build/cover-merged/ - mkdir -p acceptance/build/cover-merged/ - go tool covdata merge -i $$(printf '%s,' acceptance/build/cover/* | sed 's/,$$//') -o acceptance/build/cover-merged/ - go tool covdata textfmt -i acceptance/build/cover-merged -o coverage-acceptance.txt + VERBOSE_TEST=1 ${GOTESTSUM_CMD} --packages "${TEST_PACKAGES}" -- -coverprofile=coverage.txt -timeout=${LOCAL_TIMEOUT} ${TEST_RUN_FILTER} .PHONY: showcover showcover: From 62449deb2460b432257aadbc3bfdfe226483fd00 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Tue, 31 Mar 2026 16:48:52 +0200 Subject: [PATCH 02/17] Fix runner group name Co-authored-by: Isaac --- .github/workflows/push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 2efb2c41d0..7be5b9a01f 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -88,7 +88,7 @@ jobs: os: - name: linux runner: - group: databricks-eng-protected-runner-group-hardened-optin + group: databricks-protected-runner-group-hardened-optin labels: linux-ubuntu-latest-hardened - name: windows From cc5cd36cd0606863fbb4545af043411485cd6c4c Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Tue, 31 Mar 2026 16:55:21 +0200 Subject: [PATCH 03/17] Configure JFrog proxy for Go on hardened runner The hardened runner blocks direct access to public registries. Use JFrog Artifactory as a proxy with OIDC authentication. Co-authored-by: Isaac --- .github/workflows/push.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 7be5b9a01f..47dadabf46 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -80,6 +80,10 @@ jobs: name: "make test (${{matrix.os.name}}, ${{matrix.deployment}})" runs-on: ${{ matrix.os.runner }} + permissions: + id-token: write + contents: read + strategy: fail-fast: false matrix: @@ -121,6 +125,18 @@ jobs: - name: Checkout repository and submodules uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: Setup JFrog CLI with OIDC + if: matrix.os.name == 'linux' + uses: jfrog/setup-jfrog-cli@279b1f629f43dd5bc658d8361ac4802a7ef8d2d5 # v4.9.1 + env: + JF_URL: https://databricks.jfrog.io + with: + oidc-provider-name: github-actions + + - name: Configure Go to use JFrog proxy + if: matrix.os.name == 'linux' + run: jf goc --repo-resolve=db-golang + - name: Setup build environment uses: ./.github/actions/setup-build-environment with: From 61ec386a9e301b519a05acd71261d4a009e405bf Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Tue, 31 Mar 2026 16:58:56 +0200 Subject: [PATCH 04/17] Set GOPROXY and auth for JFrog Go proxy The jf goc command only works with jf go, not native go commands. Set GOPROXY and .netrc for native go toolchain. Co-authored-by: Isaac --- .github/workflows/push.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 47dadabf46..ec1946c02b 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -135,7 +135,16 @@ jobs: - name: Configure Go to use JFrog proxy if: matrix.os.name == 'linux' - run: jf goc --repo-resolve=db-golang + run: | + # Extract access token from JFrog CLI config for use with native go commands + JFROG_TOKEN=$(jf config show --format json | jq -r '.[0].accessToken') + echo "::add-mask::${JFROG_TOKEN}" + # Configure .netrc for Go module authentication + echo "machine databricks.jfrog.io login _ password ${JFROG_TOKEN}" >> ~/.netrc + chmod 600 ~/.netrc + # Route Go module downloads through JFrog and skip sum.golang.org (blocked) + echo "GOPROXY=https://databricks.jfrog.io/artifactory/api/go/db-golang,direct" >> "$GITHUB_ENV" + echo "GONOSUMDB=*" >> "$GITHUB_ENV" - name: Setup build environment uses: ./.github/actions/setup-build-environment From 053cc5012b4f5fa6b362eb91c06acb795dde6e37 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Tue, 31 Mar 2026 17:01:32 +0200 Subject: [PATCH 05/17] Use manual OIDC token exchange for Go proxy auth Replace jf config show (no JSON format flag) with direct OIDC token exchange, matching the pattern from the hardened runner docs for uv/cargo. Co-authored-by: Isaac --- .github/workflows/push.yml | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index ec1946c02b..94fc61c088 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -125,20 +125,26 @@ jobs: - name: Checkout repository and submodules uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - name: Setup JFrog CLI with OIDC - if: matrix.os.name == 'linux' - uses: jfrog/setup-jfrog-cli@279b1f629f43dd5bc658d8361ac4802a7ef8d2d5 # v4.9.1 - env: - JF_URL: https://databricks.jfrog.io - with: - oidc-provider-name: github-actions - - name: Configure Go to use JFrog proxy if: matrix.os.name == 'linux' run: | - # Extract access token from JFrog CLI config for use with native go commands - JFROG_TOKEN=$(jf config show --format json | jq -r '.[0].accessToken') + set -euo pipefail + # Get GitHub OIDC ID token + ID_TOKEN=$(curl -sLS \ + -H "User-Agent: actions/oidc-client" \ + -H "Authorization: Bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" \ + "${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=jfrog-github" | jq -r '.value') + echo "::add-mask::${ID_TOKEN}" + # Exchange for JFrog access token + JFROG_TOKEN=$(curl -sLS -X POST \ + -H "Content-Type: application/json" \ + "https://databricks.jfrog.io/access/api/v1/oidc/token" \ + -d "{\"grant_type\": \"urn:ietf:params:oauth:grant-type:token-exchange\", \"subject_token_type\":\"urn:ietf:params:oauth:token-type:id-token\", \"subject_token\": \"${ID_TOKEN}\", \"provider_name\": \"github-actions\"}" | jq -r '.access_token') echo "::add-mask::${JFROG_TOKEN}" + if [ -z "$JFROG_TOKEN" ]; then + echo "FAIL: Could not extract JFrog access token" + exit 1 + fi # Configure .netrc for Go module authentication echo "machine databricks.jfrog.io login _ password ${JFROG_TOKEN}" >> ~/.netrc chmod 600 ~/.netrc From 0b8b171ec49da80ee76194fd1c3833baed8ae13f Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Tue, 31 Mar 2026 17:03:58 +0200 Subject: [PATCH 06/17] Use JFrog CLI directly for Go module downloads Use jf goc + jf go mod download to pre-populate the module cache via JFrog. Native go commands then work from cache. Co-authored-by: Isaac --- .github/workflows/push.yml | 38 +++++++++++++------------------------- 1 file changed, 13 insertions(+), 25 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 94fc61c088..31fb3f9de4 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -125,38 +125,26 @@ jobs: - name: Checkout repository and submodules uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - name: Configure Go to use JFrog proxy + - name: Setup JFrog CLI with OIDC if: matrix.os.name == 'linux' - run: | - set -euo pipefail - # Get GitHub OIDC ID token - ID_TOKEN=$(curl -sLS \ - -H "User-Agent: actions/oidc-client" \ - -H "Authorization: Bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" \ - "${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=jfrog-github" | jq -r '.value') - echo "::add-mask::${ID_TOKEN}" - # Exchange for JFrog access token - JFROG_TOKEN=$(curl -sLS -X POST \ - -H "Content-Type: application/json" \ - "https://databricks.jfrog.io/access/api/v1/oidc/token" \ - -d "{\"grant_type\": \"urn:ietf:params:oauth:grant-type:token-exchange\", \"subject_token_type\":\"urn:ietf:params:oauth:token-type:id-token\", \"subject_token\": \"${ID_TOKEN}\", \"provider_name\": \"github-actions\"}" | jq -r '.access_token') - echo "::add-mask::${JFROG_TOKEN}" - if [ -z "$JFROG_TOKEN" ]; then - echo "FAIL: Could not extract JFrog access token" - exit 1 - fi - # Configure .netrc for Go module authentication - echo "machine databricks.jfrog.io login _ password ${JFROG_TOKEN}" >> ~/.netrc - chmod 600 ~/.netrc - # Route Go module downloads through JFrog and skip sum.golang.org (blocked) - echo "GOPROXY=https://databricks.jfrog.io/artifactory/api/go/db-golang,direct" >> "$GITHUB_ENV" - echo "GONOSUMDB=*" >> "$GITHUB_ENV" + uses: jfrog/setup-jfrog-cli@279b1f629f43dd5bc658d8361ac4802a7ef8d2d5 # v4.9.1 + env: + JF_URL: https://databricks.jfrog.io + with: + oidc-provider-name: github-actions - name: Setup build environment uses: ./.github/actions/setup-build-environment with: cache-key: test-${{ matrix.deployment }} + - name: Download Go modules via JFrog + if: matrix.os.name == 'linux' + run: | + jf goc --repo-resolve=db-golang + jf go mod download + jf go mod download -modfile=tools/go.mod + - name: Run tests without coverage # We run tests without coverage on PR, merge_group, and schedule because we don't make use of coverage information # and would like to run the tests as fast as possible. We run it on schedule as well, because that is what From 3a065280c20599aa805a026a6c58bcb6220a503d Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Tue, 31 Mar 2026 17:13:16 +0200 Subject: [PATCH 07/17] Empty commit to test hardened runner queue time Co-authored-by: Isaac From 2d5fc49eb588673e09f7b769d9e72e914d1e1e43 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Tue, 31 Mar 2026 17:18:23 +0200 Subject: [PATCH 08/17] Revert Makefile to run full test suite Co-authored-by: Isaac --- Makefile | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index 22bf8b907c..b31bb73d08 100644 --- a/Makefile +++ b/Makefile @@ -2,16 +2,11 @@ default: checks fmt lint # Default packages to test (all) -# TODO: restore after runner migration -# TEST_PACKAGES = ./acceptance/internal ./libs/... ./internal/... ./cmd/... ./bundle/... ./experimental/ssh/... . -TEST_PACKAGES = . +TEST_PACKAGES = ./acceptance/internal ./libs/... ./internal/... ./cmd/... ./bundle/... ./experimental/ssh/... . # Default acceptance test filter (all) ACCEPTANCE_TEST_FILTER = "" -# TODO: restore after runner migration -TEST_RUN_FILTER = -run TestCommandsDontUseUnderscoreInName - GO_TOOL ?= go tool -modfile=tools/go.mod GOTESTSUM_FORMAT ?= pkgname-and-test-fails GOTESTSUM_CMD ?= ${GO_TOOL} gotestsum --format ${GOTESTSUM_FORMAT} --no-summary=skipped --jsonfile test-output.json --rerun-fails @@ -89,16 +84,15 @@ test-slow: test-slow-unit test-slow-acc .PHONY: test-unit test-unit: - ${GOTESTSUM_CMD} --packages "${TEST_PACKAGES}" -- -timeout=${LOCAL_TIMEOUT} -short ${TEST_RUN_FILTER} + ${GOTESTSUM_CMD} --packages "${TEST_PACKAGES}" -- -timeout=${LOCAL_TIMEOUT} -short .PHONY: test-slow-unit test-slow-unit: - ${GOTESTSUM_CMD} --packages "${TEST_PACKAGES}" -- -timeout=${LOCAL_TIMEOUT} ${TEST_RUN_FILTER} + ${GOTESTSUM_CMD} --packages "${TEST_PACKAGES}" -- -timeout=${LOCAL_TIMEOUT} -# TODO: restore after runner migration .PHONY: test-acc test-acc: - @echo "Skipping acceptance tests during runner migration" + ${GOTESTSUM_CMD} --packages ./acceptance/... -- -timeout=${LOCAL_TIMEOUT} -short -run ${ACCEPTANCE_TEST_FILTER} .PHONY: test-slow-acc test-slow-acc: @@ -131,10 +125,15 @@ test-update-all: test-update test-update-aws slowest: ${GO_TOOL} gotestsum tool slowest --jsonfile test-output.json --threshold 1s --num 50 -# TODO: restore after runner migration .PHONY: cover cover: - VERBOSE_TEST=1 ${GOTESTSUM_CMD} --packages "${TEST_PACKAGES}" -- -coverprofile=coverage.txt -timeout=${LOCAL_TIMEOUT} ${TEST_RUN_FILTER} + rm -fr ./acceptance/build/cover/ + VERBOSE_TEST=1 ${GOTESTSUM_CMD} --packages "${TEST_PACKAGES}" -- -coverprofile=coverage.txt -timeout=${LOCAL_TIMEOUT} + VERBOSE_TEST=1 CLI_GOCOVERDIR=build/cover ${GOTESTSUM_CMD} --packages ./acceptance/... -- -timeout=${LOCAL_TIMEOUT} -run ${ACCEPTANCE_TEST_FILTER} + rm -fr ./acceptance/build/cover-merged/ + mkdir -p acceptance/build/cover-merged/ + go tool covdata merge -i $$(printf '%s,' acceptance/build/cover/* | sed 's/,$$//') -o acceptance/build/cover-merged/ + go tool covdata textfmt -i acceptance/build/cover-merged -o coverage-acceptance.txt .PHONY: showcover showcover: From 52704131f2d26b88b9cb6e821396c0e2f7e0f7e7 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Tue, 31 Mar 2026 18:00:50 +0200 Subject: [PATCH 09/17] Add setup-node to build environment The hardened runner doesn't have Node.js pre-installed, which is needed by the run-local-node acceptance test. Co-authored-by: Isaac --- .github/actions/setup-build-environment/action.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/actions/setup-build-environment/action.yml b/.github/actions/setup-build-environment/action.yml index 761ad617bd..38503f443c 100644 --- a/.github/actions/setup-build-environment/action.yml +++ b/.github/actions/setup-build-environment/action.yml @@ -24,6 +24,11 @@ runs: go.sum cache.txt + - name: Setup Node + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: '24' + - name: Setup Python uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: From c9b641344bf0b2174e3003c3044909e653d765e3 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Wed, 1 Apr 2026 10:34:59 +0200 Subject: [PATCH 10/17] Disable run-local-node test on hardened runner The test requires npm registry access which is blocked on the hardened runner. Disable until npm is routed through JFrog. Co-authored-by: Isaac --- acceptance/cmd/workspace/apps/run-local-node/test.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/acceptance/cmd/workspace/apps/run-local-node/test.toml b/acceptance/cmd/workspace/apps/run-local-node/test.toml index 960f763230..2b6bb4bdda 100644 --- a/acceptance/cmd/workspace/apps/run-local-node/test.toml +++ b/acceptance/cmd/workspace/apps/run-local-node/test.toml @@ -1,5 +1,6 @@ +Badness = "need to enable NPM registry access" Cloud = false -Local = true +Local = false RecordRequests = false Timeout = '2m' TimeoutWindows = '10m' From 98317549d32f3d5cd3b2afc2a2bf3dab2061bac4 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Wed, 1 Apr 2026 10:36:11 +0200 Subject: [PATCH 11/17] Remove setup-node; run-local-node is the only test needing it Co-authored-by: Isaac --- .github/actions/setup-build-environment/action.yml | 5 ----- acceptance/cmd/workspace/apps/run-local-node/out.test.toml | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/actions/setup-build-environment/action.yml b/.github/actions/setup-build-environment/action.yml index 38503f443c..761ad617bd 100644 --- a/.github/actions/setup-build-environment/action.yml +++ b/.github/actions/setup-build-environment/action.yml @@ -24,11 +24,6 @@ runs: go.sum cache.txt - - name: Setup Node - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 - with: - node-version: '24' - - name: Setup Python uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: diff --git a/acceptance/cmd/workspace/apps/run-local-node/out.test.toml b/acceptance/cmd/workspace/apps/run-local-node/out.test.toml index 54146af564..8db07a290b 100644 --- a/acceptance/cmd/workspace/apps/run-local-node/out.test.toml +++ b/acceptance/cmd/workspace/apps/run-local-node/out.test.toml @@ -1,4 +1,4 @@ -Local = true +Local = false Cloud = false [EnvMatrix] From bccbb842567dfcdd8277822000ad3bfbb7e0d291 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Wed, 1 Apr 2026 10:40:14 +0200 Subject: [PATCH 12/17] Revert runner group to existing large runners The existing runners are already hardened, so no need to switch to the hardened-optin group. Co-authored-by: Isaac --- .github/workflows/push.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 31fb3f9de4..45d35db231 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -92,8 +92,8 @@ jobs: os: - name: linux runner: - group: databricks-protected-runner-group-hardened-optin - labels: linux-ubuntu-latest-hardened + group: databricks-protected-runner-group-large + labels: linux-ubuntu-latest-large - name: windows runner: From 7363747d2c42fd44dec80afca8280d1e07baf801 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Wed, 1 Apr 2026 10:46:24 +0200 Subject: [PATCH 13/17] Enable JFrog proxy for Windows runners too Windows runners are also hardened and can't reach proxy.golang.org. Co-authored-by: Isaac --- .github/workflows/push.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 45d35db231..644b975a93 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -126,7 +126,7 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Setup JFrog CLI with OIDC - if: matrix.os.name == 'linux' + if: matrix.os.name != 'macos' uses: jfrog/setup-jfrog-cli@279b1f629f43dd5bc658d8361ac4802a7ef8d2d5 # v4.9.1 env: JF_URL: https://databricks.jfrog.io @@ -139,7 +139,7 @@ jobs: cache-key: test-${{ matrix.deployment }} - name: Download Go modules via JFrog - if: matrix.os.name == 'linux' + if: matrix.os.name != 'macos' run: | jf goc --repo-resolve=db-golang jf go mod download From 9fca8f3ad0d9a309d05013ebe4d726dde876e7f0 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Wed, 1 Apr 2026 10:50:22 +0200 Subject: [PATCH 14/17] Use bash shell for JFrog module download step PowerShell splits -modfile=tools/go.mod into separate tokens. Co-authored-by: Isaac --- .github/workflows/push.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 644b975a93..76ed6ebc75 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -140,6 +140,7 @@ jobs: - name: Download Go modules via JFrog if: matrix.os.name != 'macos' + shell: bash run: | jf goc --repo-resolve=db-golang jf go mod download From eacae041130c8a2ef96521b7c7771729a360cea4 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Wed, 1 Apr 2026 11:14:16 +0200 Subject: [PATCH 15/17] Move JFrog proxy setup to shared setup-build-environment action This ensures all test jobs that use the shared action get the JFrog Go module proxy configured on hardened runners. Co-authored-by: Isaac --- .../setup-build-environment/action.yml | 16 +++++++++++ .github/workflows/push.yml | 28 ++++++++----------- 2 files changed, 28 insertions(+), 16 deletions(-) diff --git a/.github/actions/setup-build-environment/action.yml b/.github/actions/setup-build-environment/action.yml index 761ad617bd..753315a89d 100644 --- a/.github/actions/setup-build-environment/action.yml +++ b/.github/actions/setup-build-environment/action.yml @@ -12,6 +12,14 @@ runs: - name: Checkout repository and submodules uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: Setup JFrog CLI with OIDC + if: runner.os != 'macOS' + uses: jfrog/setup-jfrog-cli@279b1f629f43dd5bc658d8361ac4802a7ef8d2d5 # v4.9.1 + env: + JF_URL: https://databricks.jfrog.io + with: + oidc-provider-name: github-actions + - name: Create cache identifier run: echo "${{ inputs.cache-key }}" > cache.txt shell: bash @@ -24,6 +32,14 @@ runs: go.sum cache.txt + - name: Download Go modules via JFrog + if: runner.os != 'macOS' + shell: bash + run: | + jf goc --repo-resolve=db-golang + jf go mod download + jf go mod download -modfile=tools/go.mod + - name: Setup Python uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 76ed6ebc75..8e578094a0 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -125,27 +125,11 @@ jobs: - name: Checkout repository and submodules uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - name: Setup JFrog CLI with OIDC - if: matrix.os.name != 'macos' - uses: jfrog/setup-jfrog-cli@279b1f629f43dd5bc658d8361ac4802a7ef8d2d5 # v4.9.1 - env: - JF_URL: https://databricks.jfrog.io - with: - oidc-provider-name: github-actions - - name: Setup build environment uses: ./.github/actions/setup-build-environment with: cache-key: test-${{ matrix.deployment }} - - name: Download Go modules via JFrog - if: matrix.os.name != 'macos' - shell: bash - run: | - jf goc --repo-resolve=db-golang - jf go mod download - jf go mod download -modfile=tools/go.mod - - name: Run tests without coverage # We run tests without coverage on PR, merge_group, and schedule because we don't make use of coverage information # and would like to run the tests as fast as possible. We run it on schedule as well, because that is what @@ -183,6 +167,10 @@ jobs: name: "make test-exp-aitools" runs-on: ${{ matrix.os }} + permissions: + id-token: write + contents: read + strategy: fail-fast: false matrix: @@ -215,6 +203,10 @@ jobs: name: "make test-exp-ssh" runs-on: ${{ matrix.os }} + permissions: + id-token: write + contents: read + strategy: fail-fast: false matrix: @@ -246,6 +238,10 @@ jobs: name: "make test-pipelines" runs-on: ${{ matrix.os }} + permissions: + id-token: write + contents: read + strategy: fail-fast: false matrix: From c0cc672c508350225d0ca67bdfecb5526fa430d2 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Wed, 1 Apr 2026 11:20:16 +0200 Subject: [PATCH 16/17] Add setup-build-environment as testmask trigger and switch secondary test jobs to custom runners Co-authored-by: Isaac --- .github/workflows/push.yml | 60 +++++++++++++++++++++++++--------- tools/testmask/targets.go | 13 ++++---- tools/testmask/targets_test.go | 7 ++++ 3 files changed, 59 insertions(+), 21 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 8e578094a0..b36bc40bee 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -164,8 +164,8 @@ jobs: # Only run if the target is in the list of targets from testmask if: ${{ contains(fromJSON(needs.testmask.outputs.targets), 'test-exp-aitools') }} - name: "make test-exp-aitools" - runs-on: ${{ matrix.os }} + name: "make test-exp-aitools (${{matrix.os.name}})" + runs-on: ${{ matrix.os.runner }} permissions: id-token: write @@ -175,10 +175,20 @@ jobs: fail-fast: false matrix: os: - - macos-latest - - ubuntu-latest + - name: linux + runner: + group: databricks-protected-runner-group-large + labels: linux-ubuntu-latest-large + + - name: macos + runner: + labels: macos-latest + # The Windows tests are broken; see https://github.com/databricks/cli/pull/4024. - # - windows-latest + # - name: windows + # runner: + # group: databricks-protected-runner-group-large + # labels: windows-server-latest-large steps: - name: Checkout repository and submodules @@ -200,8 +210,8 @@ jobs: # Only run if the target is in the list of targets from testmask if: ${{ contains(fromJSON(needs.testmask.outputs.targets), 'test-exp-ssh') }} - name: "make test-exp-ssh" - runs-on: ${{ matrix.os }} + name: "make test-exp-ssh (${{matrix.os.name}})" + runs-on: ${{ matrix.os.runner }} permissions: id-token: write @@ -211,9 +221,19 @@ jobs: fail-fast: false matrix: os: - - macos-latest - - ubuntu-latest - - windows-latest + - name: linux + runner: + group: databricks-protected-runner-group-large + labels: linux-ubuntu-latest-large + + - name: windows + runner: + group: databricks-protected-runner-group-large + labels: windows-server-latest-large + + - name: macos + runner: + labels: macos-latest steps: - name: Checkout repository and submodules @@ -235,8 +255,8 @@ jobs: # Only run if the target is in the list of targets from testmask if: ${{ contains(fromJSON(needs.testmask.outputs.targets), 'test-pipelines') }} - name: "make test-pipelines" - runs-on: ${{ matrix.os }} + name: "make test-pipelines (${{matrix.os.name}})" + runs-on: ${{ matrix.os.runner }} permissions: id-token: write @@ -246,9 +266,19 @@ jobs: fail-fast: false matrix: os: - - macos-latest - - ubuntu-latest - - windows-latest + - name: linux + runner: + group: databricks-protected-runner-group-large + labels: linux-ubuntu-latest-large + + - name: windows + runner: + group: databricks-protected-runner-group-large + labels: windows-server-latest-large + + - name: macos + runner: + labels: macos-latest steps: - name: Checkout repository and submodules diff --git a/tools/testmask/targets.go b/tools/testmask/targets.go index bdacf80031..f4566ea7d4 100644 --- a/tools/testmask/targets.go +++ b/tools/testmask/targets.go @@ -11,34 +11,35 @@ type targetMapping struct { target string } -// goTriggerPatterns lists patterns that trigger all Go-dependent targets. -var goTriggerPatterns = []string{ +// commonTriggerPatterns lists patterns that trigger all test targets. +var commonTriggerPatterns = []string{ "go.mod", "go.sum", + ".github/actions/setup-build-environment/", } var fileTargetMappings = []targetMapping{ { - prefixes: slices.Concat(goTriggerPatterns, []string{ + prefixes: slices.Concat(commonTriggerPatterns, []string{ // Specify files that match targets below and should still trigger the "test" target. }), target: "test", }, { - prefixes: slices.Concat(goTriggerPatterns, []string{ + prefixes: slices.Concat(commonTriggerPatterns, []string{ "experimental/aitools/", }), target: "test-exp-aitools", }, { - prefixes: slices.Concat(goTriggerPatterns, []string{ + prefixes: slices.Concat(commonTriggerPatterns, []string{ "experimental/ssh/", "acceptance/ssh/", }), target: "test-exp-ssh", }, { - prefixes: slices.Concat(goTriggerPatterns, []string{ + prefixes: slices.Concat(commonTriggerPatterns, []string{ "cmd/pipelines/", "acceptance/pipelines/", }), diff --git a/tools/testmask/targets_test.go b/tools/testmask/targets_test.go index ab7e2aa8c3..4ca347fb37 100644 --- a/tools/testmask/targets_test.go +++ b/tools/testmask/targets_test.go @@ -61,6 +61,13 @@ func TestGetTargets(t *testing.T) { }, targets: []string{"test", "test-exp-aitools", "test-exp-ssh", "test-pipelines"}, }, + { + name: "setup_build_environment_triggers_all", + files: []string{ + ".github/actions/setup-build-environment/action.yml", + }, + targets: []string{"test", "test-exp-aitools", "test-exp-ssh", "test-pipelines"}, + }, { name: "empty_files", files: []string{}, From 789e56311b6bd7e46bac6da82e1b4d6ee96b7259 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Wed, 1 Apr 2026 11:28:13 +0200 Subject: [PATCH 17/17] Fix YAML formatting for commented-out Windows matrix entry Co-authored-by: Isaac --- .github/workflows/push.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index b36bc40bee..f6fb21d56c 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -184,11 +184,11 @@ jobs: runner: labels: macos-latest - # The Windows tests are broken; see https://github.com/databricks/cli/pull/4024. - # - name: windows - # runner: - # group: databricks-protected-runner-group-large - # labels: windows-server-latest-large + # The Windows tests are broken; see https://github.com/databricks/cli/pull/4024. + # - name: windows + # runner: + # group: databricks-protected-runner-group-large + # labels: windows-server-latest-large steps: - name: Checkout repository and submodules