From 1198a0151560aedbf57ef8fe8c1ae14f4dde9233 Mon Sep 17 00:00:00 2001 From: Mihai Mitrea Date: Wed, 1 Apr 2026 15:35:04 +0000 Subject: [PATCH 01/12] Configure JFrog Artifactory as PyPI proxy for hardened runners Hardened GitHub Actions runners block direct access to public registries. This adds OIDC-based authentication to JFrog Artifactory and routes all uv package downloads through the db-pypi proxy. A `make fix-lockfile` target prevents proxy URLs from leaking into the committed lockfile. --- .github/actions/setup-jfrog-pypi/action.yml | 31 +++++++++++++++++++++ .github/workflows/push.yml | 19 +++++++++++++ .github/workflows/release-test.yml | 6 ++++ .github/workflows/release.yml | 6 ++++ .github/workflows/tagging.yml | 10 +++++++ .github/workflows/test.yml | 12 ++++++++ Makefile | 8 ++++++ 7 files changed, 92 insertions(+) create mode 100644 .github/actions/setup-jfrog-pypi/action.yml diff --git a/.github/actions/setup-jfrog-pypi/action.yml b/.github/actions/setup-jfrog-pypi/action.yml new file mode 100644 index 000000000..bcf7d7efe --- /dev/null +++ b/.github/actions/setup-jfrog-pypi/action.yml @@ -0,0 +1,31 @@ +name: Setup JFrog PyPI proxy +description: Authenticate to JFrog via OIDC and configure uv to use the db-pypi proxy + +runs: + using: composite + steps: + - name: Get JFrog OIDC token + shell: bash + run: | + set -euo pipefail + + ID_TOKEN=$(curl -sLS \ + -H "User-Agent: actions/oidc-client" \ + -H "Authorization: Bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" \ + "${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=jfrog-github" | jq .value | tr -d '"') + echo "::add-mask::${ID_TOKEN}" + + ACCESS_TOKEN=$(curl -sLS -XPOST -H "Content-Type: application/json" \ + "https://databricks.jfrog.io/access/api/v1/oidc/token" \ + -d "{\"grant_type\": \"urn:ietf:params:oauth:grant-type:token-exchange\", \"subject_token_type\":\"urn:ietf:params:oauth:token-type:id_token\", \"subject_token\": \"${ID_TOKEN}\", \"provider_name\": \"github-actions\"}" | jq .access_token | tr -d '"') + echo "::add-mask::${ACCESS_TOKEN}" + + if [ -z "$ACCESS_TOKEN" ] || [ "$ACCESS_TOKEN" = "null" ]; then + echo "FAIL: Could not extract JFrog access token" + exit 1 + fi + + echo "JFROG_ACCESS_TOKEN=${ACCESS_TOKEN}" >> "$GITHUB_ENV" + echo "UV_INDEX_URL=https://gha-service-account:${ACCESS_TOKEN}@databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple" >> "$GITHUB_ENV" + + echo "JFrog OIDC token obtained and uv configured to use JFrog registry" diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 1b6b389eb..0567ee358 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -6,6 +6,10 @@ on: merge_group: types: [checks_requested] +permissions: + id-token: write + contents: read + jobs: tests-ubuntu: uses: ./.github/workflows/test.yml @@ -39,9 +43,18 @@ jobs: with: version: "0.6.5" + - name: Setup JFrog PyPI proxy + uses: ./.github/actions/setup-jfrog-pypi + + - name: Re-lock for JFrog + run: uv lock + - name: Format all files run: make dev fmt + - name: Fix lockfile + run: make fix-lockfile + - name: Fail on differences run: git diff --exit-code @@ -57,5 +70,11 @@ jobs: with: version: "0.6.5" + - name: Setup JFrog PyPI proxy + uses: ./.github/actions/setup-jfrog-pypi + + - name: Re-lock for JFrog + run: uv lock + - name: Check MANIFEST.in run: make dev && uv run check-manifest . diff --git a/.github/workflows/release-test.yml b/.github/workflows/release-test.yml index e8ae07507..51d7d9786 100644 --- a/.github/workflows/release-test.yml +++ b/.github/workflows/release-test.yml @@ -22,6 +22,12 @@ jobs: with: version: "0.6.5" + - name: Setup JFrog PyPI proxy + uses: ./.github/actions/setup-jfrog-pypi + + - name: Re-lock for JFrog + run: uv lock + - name: Build wheel run: uv build env: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b081afb00..1f7d2b979 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -25,6 +25,12 @@ jobs: with: version: "0.6.5" + - name: Setup JFrog PyPI proxy + uses: ./.github/actions/setup-jfrog-pypi + + - name: Re-lock for JFrog + run: uv lock + - name: Build wheel run: uv build env: diff --git a/.github/workflows/tagging.yml b/.github/workflows/tagging.yml index c3c6d2d00..c669835d8 100755 --- a/.github/workflows/tagging.yml +++ b/.github/workflows/tagging.yml @@ -34,6 +34,10 @@ jobs: runs-on: group: databricks-deco-testing-runner-group labels: ubuntu-latest-deco + + permissions: + id-token: write + contents: write steps: - name: Generate GitHub App Token id: generate-token @@ -57,6 +61,12 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a # v4.2.0 + - name: Setup JFrog PyPI proxy + uses: ./.github/actions/setup-jfrog-pypi + + - name: Re-lock tagging script for JFrog + run: uv lock --script tagging.py + - name: Run script env: GITHUB_TOKEN: ${{ steps.generate-token.outputs.token }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 57d8db779..186bdd619 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -15,6 +15,11 @@ jobs: strategy: fail-fast: false runs-on: ${{ inputs.os }} + + permissions: + id-token: write + contents: read + steps: - name: Checkout uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 @@ -28,6 +33,13 @@ jobs: version: "0.6.5" python-version: ${{ inputs.pyVersion }} + - name: Setup JFrog PyPI proxy + uses: ./.github/actions/setup-jfrog-pypi + + - name: Re-lock for JFrog + shell: bash + run: uv lock + - name: Run tests run: make dev test diff --git a/Makefile b/Makefile index b2d404851..ee0ba8307 100644 --- a/Makefile +++ b/Makefile @@ -30,5 +30,13 @@ benchmark: coverage: test open htmlcov/index.html +fix-lockfile: + @# Replace JFrog proxy URLs with public equivalents in lockfiles. + @# Prevents proxy URLs from being accidentally committed. + find . -type f -name '*.lock' -not -path './.github/*' \ + -exec sed -i 's|databricks\.jfrog\.io/artifactory/api/pypi/db-pypi/simple|pypi.org/simple|g' {} + + find . -type f -name '*.lock' -not -path './.github/*' \ + -exec sed -i 's|databricks\.jfrog\.io/artifactory/api/pypi/db-pypi/packages|files.pythonhosted.org/packages|g' {} + + clean: rm -fr dist *.egg-info .pytest_cache build htmlcov .venv From 6a83a5bb5bc77d524c21192762092e0814de2817 Mon Sep 17 00:00:00 2001 From: Mihai Mitrea Date: Wed, 1 Apr 2026 15:41:50 +0000 Subject: [PATCH 02/12] Fix lockfile sed pattern and restore lockfiles before diff check The JFrog proxy URL structure has /packages/packages/ (the proxy path plus the original PyPI path), so the sed replacement must map db-pypi/packages to files.pythonhosted.org (without /packages suffix). Also, uv lock through JFrog drops size/upload-time metadata from the lockfile. Restore committed lockfiles with git checkout before the diff check so only formatting changes and accidentally committed proxy URLs are caught. --- .github/workflows/push.yml | 6 ++++-- Makefile | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 0567ee358..a80d39885 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -52,8 +52,10 @@ jobs: - name: Format all files run: make dev fmt - - name: Fix lockfile - run: make fix-lockfile + - name: Restore lockfiles and fix proxy URLs + run: | + git checkout -- '*.lock' + make fix-lockfile - name: Fail on differences run: git diff --exit-code diff --git a/Makefile b/Makefile index ee0ba8307..c657e524c 100644 --- a/Makefile +++ b/Makefile @@ -36,7 +36,7 @@ fix-lockfile: find . -type f -name '*.lock' -not -path './.github/*' \ -exec sed -i 's|databricks\.jfrog\.io/artifactory/api/pypi/db-pypi/simple|pypi.org/simple|g' {} + find . -type f -name '*.lock' -not -path './.github/*' \ - -exec sed -i 's|databricks\.jfrog\.io/artifactory/api/pypi/db-pypi/packages|files.pythonhosted.org/packages|g' {} + + -exec sed -i 's|databricks\.jfrog\.io/artifactory/api/pypi/db-pypi/packages|files.pythonhosted.org|g' {} + clean: rm -fr dist *.egg-info .pytest_cache build htmlcov .venv From 455f6d2e96fd1049160cb28aa4a2561c84047137 Mon Sep 17 00:00:00 2001 From: Mihai Mitrea Date: Wed, 1 Apr 2026 15:59:53 +0000 Subject: [PATCH 03/12] Pin uv version in tagging workflow to match other workflows The tagging workflow used setup-uv v4.2.0 without a version pin, installing whatever uv release was latest. Align it with the rest of the workflows: setup-uv v7.3.0 pinned to uv 0.6.5. --- .github/workflows/tagging.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tagging.yml b/.github/workflows/tagging.yml index c669835d8..87d80dca2 100755 --- a/.github/workflows/tagging.yml +++ b/.github/workflows/tagging.yml @@ -59,7 +59,9 @@ jobs: git config user.email "DECO-SDK-Tagging[bot]@users.noreply.github.com" - name: Install uv - uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a # v4.2.0 + uses: astral-sh/setup-uv@eac588ad8def6316056a12d4907a9d4d84ff7a3b # v7.3.0 + with: + version: "0.6.5" - name: Setup JFrog PyPI proxy uses: ./.github/actions/setup-jfrog-pypi From 2ec2a40ca484b41bd585b3db53acb882083d8d22 Mon Sep 17 00:00:00 2001 From: Mihai Mitrea Date: Wed, 1 Apr 2026 16:06:11 +0000 Subject: [PATCH 04/12] Remove tagging.yml changes (generated file, not in P00 scope) tagging.yml is generated from the Universe codegen template and is not in the critical path for unblocking unit tests. The uv version pin and JFrog setup need to be upstreamed to the genkit sync template instead. --- .github/workflows/tagging.yml | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/.github/workflows/tagging.yml b/.github/workflows/tagging.yml index 87d80dca2..c3c6d2d00 100755 --- a/.github/workflows/tagging.yml +++ b/.github/workflows/tagging.yml @@ -34,10 +34,6 @@ jobs: runs-on: group: databricks-deco-testing-runner-group labels: ubuntu-latest-deco - - permissions: - id-token: write - contents: write steps: - name: Generate GitHub App Token id: generate-token @@ -59,15 +55,7 @@ jobs: git config user.email "DECO-SDK-Tagging[bot]@users.noreply.github.com" - name: Install uv - uses: astral-sh/setup-uv@eac588ad8def6316056a12d4907a9d4d84ff7a3b # v7.3.0 - with: - version: "0.6.5" - - - name: Setup JFrog PyPI proxy - uses: ./.github/actions/setup-jfrog-pypi - - - name: Re-lock tagging script for JFrog - run: uv lock --script tagging.py + uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a # v4.2.0 - name: Run script env: From 915a823ee307ca755fae0ab68c7c72857b7d6b2f Mon Sep 17 00:00:00 2001 From: Mihai Mitrea Date: Wed, 1 Apr 2026 16:15:09 +0000 Subject: [PATCH 05/12] Remove release workflow changes (out of P00 scope) Release workflows need special publish runners and coordination with the security team per the migration guide. Keep this PR focused on unblocking CI (tests, fmt, check-manifest). --- .github/workflows/release-test.yml | 6 ------ .github/workflows/release.yml | 6 ------ 2 files changed, 12 deletions(-) diff --git a/.github/workflows/release-test.yml b/.github/workflows/release-test.yml index 51d7d9786..e8ae07507 100644 --- a/.github/workflows/release-test.yml +++ b/.github/workflows/release-test.yml @@ -22,12 +22,6 @@ jobs: with: version: "0.6.5" - - name: Setup JFrog PyPI proxy - uses: ./.github/actions/setup-jfrog-pypi - - - name: Re-lock for JFrog - run: uv lock - - name: Build wheel run: uv build env: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1f7d2b979..b081afb00 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -25,12 +25,6 @@ jobs: with: version: "0.6.5" - - name: Setup JFrog PyPI proxy - uses: ./.github/actions/setup-jfrog-pypi - - - name: Re-lock for JFrog - run: uv lock - - name: Build wheel run: uv build env: From 2261e520543852f184208aa7be5f4db9266d2842 Mon Sep 17 00:00:00 2001 From: Mihai Mitrea Date: Tue, 7 Apr 2026 06:41:56 +0000 Subject: [PATCH 06/12] Test transparent proxy routing: remove uv lock, use --frozen Remove uv lock from CI and use uv sync --frozen instead of --locked. If the hardened runners transparently route files.pythonhosted.org through the JFrog proxy, downloads from lockfile URLs should work without rewriting them. --- .github/workflows/push.yml | 22 ++++++++++------------ .github/workflows/test.yml | 8 +++----- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index a80d39885..c3ca6bda7 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -46,16 +46,15 @@ jobs: - name: Setup JFrog PyPI proxy uses: ./.github/actions/setup-jfrog-pypi - - name: Re-lock for JFrog - run: uv lock - - name: Format all files - run: make dev fmt - - - name: Restore lockfiles and fix proxy URLs run: | - git checkout -- '*.lock' - make fix-lockfile + uv sync --frozen --extra dev + uv run black databricks tests + uv run autoflake -ri databricks tests + uv run isort databricks tests + + - name: Fix lockfile + run: make fix-lockfile - name: Fail on differences run: git diff --exit-code @@ -75,8 +74,7 @@ jobs: - name: Setup JFrog PyPI proxy uses: ./.github/actions/setup-jfrog-pypi - - name: Re-lock for JFrog - run: uv lock - - name: Check MANIFEST.in - run: make dev && uv run check-manifest . + run: | + uv sync --frozen --extra dev + uv run check-manifest . diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 186bdd619..9ed16d9d4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -36,12 +36,10 @@ jobs: - name: Setup JFrog PyPI proxy uses: ./.github/actions/setup-jfrog-pypi - - name: Re-lock for JFrog - shell: bash - run: uv lock - - name: Run tests - run: make dev test + run: | + uv sync --frozen --extra dev + uv run pytest -m 'not integration and not benchmark' --cov=databricks --cov-report html tests - name: Publish test coverage uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4.6.0 From 0cdc22a8ad043c2adb82e24f81f9edfda443eb11 Mon Sep 17 00:00:00 2001 From: Mihai Mitrea Date: Tue, 7 Apr 2026 06:45:18 +0000 Subject: [PATCH 07/12] Pass --frozen to uv run to prevent auto-sync re-locking uv run without --frozen triggers auto-sync, which re-locks the lockfile with JFrog URLs and drops size/upload-time metadata. Pass --frozen to all uv run calls since we already synced the environment explicitly. --- .github/workflows/push.yml | 8 ++++---- .github/workflows/test.yml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index c3ca6bda7..a49600c03 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -49,9 +49,9 @@ jobs: - name: Format all files run: | uv sync --frozen --extra dev - uv run black databricks tests - uv run autoflake -ri databricks tests - uv run isort databricks tests + uv run --frozen black databricks tests + uv run --frozen autoflake -ri databricks tests + uv run --frozen isort databricks tests - name: Fix lockfile run: make fix-lockfile @@ -77,4 +77,4 @@ jobs: - name: Check MANIFEST.in run: | uv sync --frozen --extra dev - uv run check-manifest . + uv run --frozen check-manifest . diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9ed16d9d4..d16ecd505 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -39,7 +39,7 @@ jobs: - name: Run tests run: | uv sync --frozen --extra dev - uv run pytest -m 'not integration and not benchmark' --cov=databricks --cov-report html tests + uv run --frozen pytest -m 'not integration and not benchmark' --cov=databricks --cov-report html tests - name: Publish test coverage uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4.6.0 From 1fa0511ce013f6712e69217752243c02d1232af3 Mon Sep 17 00:00:00 2001 From: Mihai Mitrea Date: Tue, 7 Apr 2026 06:48:40 +0000 Subject: [PATCH 08/12] Use Makefile targets with flag overrides instead of inline uv commands Add UV_SYNC_FLAGS and UV_RUN_FLAGS variables to the Makefile so CI can pass --frozen without duplicating commands. Local dev defaults remain unchanged (--locked for sync, no flags for run). --- .github/workflows/push.yml | 10 ++-------- .github/workflows/test.yml | 4 +--- Makefile | 29 ++++++++++++++++------------- 3 files changed, 19 insertions(+), 24 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index a49600c03..fe7cacfe7 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -47,11 +47,7 @@ jobs: uses: ./.github/actions/setup-jfrog-pypi - name: Format all files - run: | - uv sync --frozen --extra dev - uv run --frozen black databricks tests - uv run --frozen autoflake -ri databricks tests - uv run --frozen isort databricks tests + run: make dev fmt UV_SYNC_FLAGS=--frozen UV_RUN_FLAGS=--frozen - name: Fix lockfile run: make fix-lockfile @@ -75,6 +71,4 @@ jobs: uses: ./.github/actions/setup-jfrog-pypi - name: Check MANIFEST.in - run: | - uv sync --frozen --extra dev - uv run --frozen check-manifest . + run: make dev UV_SYNC_FLAGS=--frozen UV_RUN_FLAGS=--frozen && uv run --frozen check-manifest . diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d16ecd505..d06b17bb6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -37,9 +37,7 @@ jobs: uses: ./.github/actions/setup-jfrog-pypi - name: Run tests - run: | - uv sync --frozen --extra dev - uv run --frozen pytest -m 'not integration and not benchmark' --cov=databricks --cov-report html tests + run: make dev test UV_SYNC_FLAGS=--frozen UV_RUN_FLAGS=--frozen - name: Publish test coverage uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4.6.0 diff --git a/Makefile b/Makefile index c657e524c..1ee02ce91 100644 --- a/Makefile +++ b/Makefile @@ -1,31 +1,34 @@ +UV_SYNC_FLAGS ?= --locked +UV_RUN_FLAGS ?= + dev: - uv sync --locked --extra dev + uv sync $(UV_SYNC_FLAGS) --extra dev install: - uv sync --locked + uv sync $(UV_SYNC_FLAGS) fmt: - uv run black databricks tests - uv run autoflake -ri databricks tests - uv run isort databricks tests + uv run $(UV_RUN_FLAGS) black databricks tests + uv run $(UV_RUN_FLAGS) autoflake -ri databricks tests + uv run $(UV_RUN_FLAGS) isort databricks tests fmte: - uv run black examples - uv run autoflake -ri examples - uv run isort examples + uv run $(UV_RUN_FLAGS) black examples + uv run $(UV_RUN_FLAGS) autoflake -ri examples + uv run $(UV_RUN_FLAGS) isort examples lint: - uv run pycodestyle databricks - uv run autoflake --check-diff --quiet --recursive databricks + uv run $(UV_RUN_FLAGS) pycodestyle databricks + uv run $(UV_RUN_FLAGS) autoflake --check-diff --quiet --recursive databricks test: - uv run pytest -m 'not integration and not benchmark' --cov=databricks --cov-report html tests + uv run $(UV_RUN_FLAGS) pytest -m 'not integration and not benchmark' --cov=databricks --cov-report html tests integration: - uv run pytest -n auto -m 'integration and not benchmark' --reruns 4 --dist loadgroup --cov=databricks --cov-report html tests + uv run $(UV_RUN_FLAGS) pytest -n auto -m 'integration and not benchmark' --reruns 4 --dist loadgroup --cov=databricks --cov-report html tests benchmark: - uv run pytest -m 'benchmark' tests + uv run $(UV_RUN_FLAGS) pytest -m 'benchmark' tests coverage: test open htmlcov/index.html From c12c6f64c4199431b6c748786229cc997aff430b Mon Sep 17 00:00:00 2001 From: Mihai Mitrea Date: Tue, 7 Apr 2026 06:53:14 +0000 Subject: [PATCH 09/12] Simplify: use UV_FROZEN=1 instead of Makefile overrides Set UV_FROZEN=1 in the composite action so uv natively skips lockfile re-resolution on all commands. This lets workflows call make dev test and make dev fmt unchanged -- no Makefile modifications needed beyond the new fix-lockfile target. --- .github/actions/setup-jfrog-pypi/action.yml | 10 +++++++ .github/workflows/push.yml | 4 +-- .github/workflows/test.yml | 2 +- Makefile | 29 +++++++++------------ 4 files changed, 26 insertions(+), 19 deletions(-) diff --git a/.github/actions/setup-jfrog-pypi/action.yml b/.github/actions/setup-jfrog-pypi/action.yml index bcf7d7efe..c808d911d 100644 --- a/.github/actions/setup-jfrog-pypi/action.yml +++ b/.github/actions/setup-jfrog-pypi/action.yml @@ -26,6 +26,16 @@ runs: fi echo "JFROG_ACCESS_TOKEN=${ACCESS_TOKEN}" >> "$GITHUB_ENV" + + # Route uv package resolution through the JFrog PyPI proxy. Hardened + # runners block direct access to pypi.org, so all index queries go + # through this authenticated mirror instead. echo "UV_INDEX_URL=https://gha-service-account:${ACCESS_TOKEN}@databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple" >> "$GITHUB_ENV" + # Tell uv to use the committed lockfile as-is without re-resolving. + # Without this, uv sync --locked would fail because UV_INDEX_URL + # differs from the registry URL stored in uv.lock, and uv run would + # trigger auto-sync which rewrites the lockfile with JFrog URLs. + echo "UV_FROZEN=1" >> "$GITHUB_ENV" + echo "JFrog OIDC token obtained and uv configured to use JFrog registry" diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index fe7cacfe7..eb0bf1517 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -47,7 +47,7 @@ jobs: uses: ./.github/actions/setup-jfrog-pypi - name: Format all files - run: make dev fmt UV_SYNC_FLAGS=--frozen UV_RUN_FLAGS=--frozen + run: make dev fmt - name: Fix lockfile run: make fix-lockfile @@ -71,4 +71,4 @@ jobs: uses: ./.github/actions/setup-jfrog-pypi - name: Check MANIFEST.in - run: make dev UV_SYNC_FLAGS=--frozen UV_RUN_FLAGS=--frozen && uv run --frozen check-manifest . + run: make dev && uv run check-manifest . diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d06b17bb6..551e7441d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -37,7 +37,7 @@ jobs: uses: ./.github/actions/setup-jfrog-pypi - name: Run tests - run: make dev test UV_SYNC_FLAGS=--frozen UV_RUN_FLAGS=--frozen + run: make dev test - name: Publish test coverage uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4.6.0 diff --git a/Makefile b/Makefile index 1ee02ce91..c657e524c 100644 --- a/Makefile +++ b/Makefile @@ -1,34 +1,31 @@ -UV_SYNC_FLAGS ?= --locked -UV_RUN_FLAGS ?= - dev: - uv sync $(UV_SYNC_FLAGS) --extra dev + uv sync --locked --extra dev install: - uv sync $(UV_SYNC_FLAGS) + uv sync --locked fmt: - uv run $(UV_RUN_FLAGS) black databricks tests - uv run $(UV_RUN_FLAGS) autoflake -ri databricks tests - uv run $(UV_RUN_FLAGS) isort databricks tests + uv run black databricks tests + uv run autoflake -ri databricks tests + uv run isort databricks tests fmte: - uv run $(UV_RUN_FLAGS) black examples - uv run $(UV_RUN_FLAGS) autoflake -ri examples - uv run $(UV_RUN_FLAGS) isort examples + uv run black examples + uv run autoflake -ri examples + uv run isort examples lint: - uv run $(UV_RUN_FLAGS) pycodestyle databricks - uv run $(UV_RUN_FLAGS) autoflake --check-diff --quiet --recursive databricks + uv run pycodestyle databricks + uv run autoflake --check-diff --quiet --recursive databricks test: - uv run $(UV_RUN_FLAGS) pytest -m 'not integration and not benchmark' --cov=databricks --cov-report html tests + uv run pytest -m 'not integration and not benchmark' --cov=databricks --cov-report html tests integration: - uv run $(UV_RUN_FLAGS) pytest -n auto -m 'integration and not benchmark' --reruns 4 --dist loadgroup --cov=databricks --cov-report html tests + uv run pytest -n auto -m 'integration and not benchmark' --reruns 4 --dist loadgroup --cov=databricks --cov-report html tests benchmark: - uv run $(UV_RUN_FLAGS) pytest -m 'benchmark' tests + uv run pytest -m 'benchmark' tests coverage: test open htmlcov/index.html From 535d37465143375ccd50576894f1854a1993ba39 Mon Sep 17 00:00:00 2001 From: Mihai Mitrea Date: Tue, 7 Apr 2026 07:15:55 +0000 Subject: [PATCH 10/12] Simplify JFrog auth: use setup-jfrog-cli instead of manual curl Replace the manual OIDC token exchange (~20 lines of curl/jq) with the jfrog/setup-jfrog-cli action, which handles OIDC internally and exposes the token via step outputs. Same action + SHA as the CLI repo (PR #4875). --- .github/actions/setup-jfrog-pypi/action.yml | 34 ++++++--------------- 1 file changed, 10 insertions(+), 24 deletions(-) diff --git a/.github/actions/setup-jfrog-pypi/action.yml b/.github/actions/setup-jfrog-pypi/action.yml index c808d911d..7fdfa1cf5 100644 --- a/.github/actions/setup-jfrog-pypi/action.yml +++ b/.github/actions/setup-jfrog-pypi/action.yml @@ -4,38 +4,24 @@ description: Authenticate to JFrog via OIDC and configure uv to use the db-pypi runs: using: composite steps: - - name: Get JFrog OIDC token + - name: Setup JFrog CLI with OIDC + id: jfrog + uses: jfrog/setup-jfrog-cli@279b1f629f43dd5bc658d8361ac4802a7ef8d2d5 # v4.9.1 + env: + JF_URL: https://databricks.jfrog.io + with: + oidc-provider-name: github-actions + + - name: Configure uv for JFrog shell: bash run: | - set -euo pipefail - - ID_TOKEN=$(curl -sLS \ - -H "User-Agent: actions/oidc-client" \ - -H "Authorization: Bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" \ - "${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=jfrog-github" | jq .value | tr -d '"') - echo "::add-mask::${ID_TOKEN}" - - ACCESS_TOKEN=$(curl -sLS -XPOST -H "Content-Type: application/json" \ - "https://databricks.jfrog.io/access/api/v1/oidc/token" \ - -d "{\"grant_type\": \"urn:ietf:params:oauth:grant-type:token-exchange\", \"subject_token_type\":\"urn:ietf:params:oauth:token-type:id_token\", \"subject_token\": \"${ID_TOKEN}\", \"provider_name\": \"github-actions\"}" | jq .access_token | tr -d '"') - echo "::add-mask::${ACCESS_TOKEN}" - - if [ -z "$ACCESS_TOKEN" ] || [ "$ACCESS_TOKEN" = "null" ]; then - echo "FAIL: Could not extract JFrog access token" - exit 1 - fi - - echo "JFROG_ACCESS_TOKEN=${ACCESS_TOKEN}" >> "$GITHUB_ENV" - # Route uv package resolution through the JFrog PyPI proxy. Hardened # runners block direct access to pypi.org, so all index queries go # through this authenticated mirror instead. - echo "UV_INDEX_URL=https://gha-service-account:${ACCESS_TOKEN}@databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple" >> "$GITHUB_ENV" + echo "UV_INDEX_URL=https://${{ steps.jfrog.outputs.oidc-user }}:${{ steps.jfrog.outputs.oidc-token }}@databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple" >> "$GITHUB_ENV" # Tell uv to use the committed lockfile as-is without re-resolving. # Without this, uv sync --locked would fail because UV_INDEX_URL # differs from the registry URL stored in uv.lock, and uv run would # trigger auto-sync which rewrites the lockfile with JFrog URLs. echo "UV_FROZEN=1" >> "$GITHUB_ENV" - - echo "JFrog OIDC token obtained and uv configured to use JFrog registry" From 01ff01e18e3549ae9423d6dad3c16cde2a8e17bb Mon Sep 17 00:00:00 2001 From: Mihai Mitrea Date: Tue, 7 Apr 2026 07:21:47 +0000 Subject: [PATCH 11/12] Fix --locked/--frozen conflict: use UV_LOCK_FLAG in Makefile uv rejects --locked and --frozen together. Add UV_LOCK_FLAG variable (defaults to --locked) that the composite action clears via env var, letting UV_FROZEN=1 handle lockfile behavior in CI instead. --- .github/actions/setup-jfrog-pypi/action.yml | 5 +++++ Makefile | 8 ++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/actions/setup-jfrog-pypi/action.yml b/.github/actions/setup-jfrog-pypi/action.yml index 7fdfa1cf5..c34ae4723 100644 --- a/.github/actions/setup-jfrog-pypi/action.yml +++ b/.github/actions/setup-jfrog-pypi/action.yml @@ -25,3 +25,8 @@ runs: # differs from the registry URL stored in uv.lock, and uv run would # trigger auto-sync which rewrites the lockfile with JFrog URLs. echo "UV_FROZEN=1" >> "$GITHUB_ENV" + + # Clear the Makefile's --locked flag since it conflicts with --frozen. + # The Makefile uses UV_LOCK_FLAG ?= --locked, so setting it empty here + # prevents the conflict while UV_FROZEN=1 handles lockfile behavior. + echo "UV_LOCK_FLAG=" >> "$GITHUB_ENV" diff --git a/Makefile b/Makefile index c657e524c..bdae15593 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,12 @@ +# In CI, the JFrog composite action sets UV_LOCK_FLAG= (empty) because +# UV_FROZEN=1 handles lockfile behavior and conflicts with --locked. +UV_LOCK_FLAG ?= --locked + dev: - uv sync --locked --extra dev + uv sync $(UV_LOCK_FLAG) --extra dev install: - uv sync --locked + uv sync $(UV_LOCK_FLAG) fmt: uv run black databricks tests From 5381c9da913bf07cc5379094f3bb9476090d0f63 Mon Sep 17 00:00:00 2001 From: Mihai Mitrea Date: Tue, 7 Apr 2026 07:23:03 +0000 Subject: [PATCH 12/12] Use uv lock instead of UV_FROZEN to avoid Makefile changes Run uv lock after JFrog setup to rewrite the lockfile with JFrog URLs (same versions, different registry). This lets make dev test and make dev fmt work unchanged with --locked. In fmt, restore committed lockfiles before the diff check so only formatting issues and accidentally committed proxy URLs are caught. --- .github/actions/setup-jfrog-pypi/action.yml | 11 ----------- .github/workflows/push.yml | 12 ++++++++++-- .github/workflows/test.yml | 6 ++++++ Makefile | 8 ++------ 4 files changed, 18 insertions(+), 19 deletions(-) diff --git a/.github/actions/setup-jfrog-pypi/action.yml b/.github/actions/setup-jfrog-pypi/action.yml index c34ae4723..8855a1733 100644 --- a/.github/actions/setup-jfrog-pypi/action.yml +++ b/.github/actions/setup-jfrog-pypi/action.yml @@ -19,14 +19,3 @@ runs: # runners block direct access to pypi.org, so all index queries go # through this authenticated mirror instead. echo "UV_INDEX_URL=https://${{ steps.jfrog.outputs.oidc-user }}:${{ steps.jfrog.outputs.oidc-token }}@databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple" >> "$GITHUB_ENV" - - # Tell uv to use the committed lockfile as-is without re-resolving. - # Without this, uv sync --locked would fail because UV_INDEX_URL - # differs from the registry URL stored in uv.lock, and uv run would - # trigger auto-sync which rewrites the lockfile with JFrog URLs. - echo "UV_FROZEN=1" >> "$GITHUB_ENV" - - # Clear the Makefile's --locked flag since it conflicts with --frozen. - # The Makefile uses UV_LOCK_FLAG ?= --locked, so setting it empty here - # prevents the conflict while UV_FROZEN=1 handles lockfile behavior. - echo "UV_LOCK_FLAG=" >> "$GITHUB_ENV" diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index eb0bf1517..a80d39885 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -46,11 +46,16 @@ jobs: - name: Setup JFrog PyPI proxy uses: ./.github/actions/setup-jfrog-pypi + - name: Re-lock for JFrog + run: uv lock + - name: Format all files run: make dev fmt - - name: Fix lockfile - run: make fix-lockfile + - name: Restore lockfiles and fix proxy URLs + run: | + git checkout -- '*.lock' + make fix-lockfile - name: Fail on differences run: git diff --exit-code @@ -70,5 +75,8 @@ jobs: - name: Setup JFrog PyPI proxy uses: ./.github/actions/setup-jfrog-pypi + - name: Re-lock for JFrog + run: uv lock + - name: Check MANIFEST.in run: make dev && uv run check-manifest . diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 551e7441d..489cf2a33 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -36,6 +36,12 @@ jobs: - name: Setup JFrog PyPI proxy uses: ./.github/actions/setup-jfrog-pypi + # Re-lock so uv.lock matches the JFrog-configured UV_INDEX_URL. + # Keeps the same versions; only registry URLs change. Ephemeral (not committed). + - name: Re-lock for JFrog + shell: bash + run: uv lock + - name: Run tests run: make dev test diff --git a/Makefile b/Makefile index bdae15593..c657e524c 100644 --- a/Makefile +++ b/Makefile @@ -1,12 +1,8 @@ -# In CI, the JFrog composite action sets UV_LOCK_FLAG= (empty) because -# UV_FROZEN=1 handles lockfile behavior and conflicts with --locked. -UV_LOCK_FLAG ?= --locked - dev: - uv sync $(UV_LOCK_FLAG) --extra dev + uv sync --locked --extra dev install: - uv sync $(UV_LOCK_FLAG) + uv sync --locked fmt: uv run black databricks tests