From b8191545230f29c76813391b11a54eb44c74be58 Mon Sep 17 00:00:00 2001 From: Andrew Snare Date: Tue, 10 Mar 2026 19:24:46 +0100 Subject: [PATCH 01/19] Upgrade Hatch: 1.15.0 -> 1.16.5 Version 1.15 is broken due to the latest pip release(s). --- .github/workflows/acceptance.yml | 8 ++++++-- .github/workflows/downstreams.yml | 9 +++++++-- .github/workflows/nightly.yml | 8 ++++++-- .github/workflows/push.yml | 7 ++++--- .github/workflows/release.yml | 8 ++++++-- 5 files changed, 29 insertions(+), 11 deletions(-) diff --git a/.github/workflows/acceptance.yml b/.github/workflows/acceptance.yml index 599bbf2..b0993d7 100644 --- a/.github/workflows/acceptance.yml +++ b/.github/workflows/acceptance.yml @@ -18,6 +18,10 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +env: + HATCH_VERBOSE: "2" + HATCH_VERSION: "1.16.5" + jobs: integration: if: github.event_name == 'pull_request' && github.event.pull_request.draft == false @@ -37,7 +41,7 @@ jobs: python-version: '3.10' - name: Install hatch - run: pip install hatch==1.15.0 + run: pip install "hatch==${HATCH_VERSION}" - name: Fetch relevant branches run: | @@ -51,4 +55,4 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} ARM_CLIENT_ID: ${{ secrets.ARM_CLIENT_ID }} - ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }} + ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }} \ No newline at end of file diff --git a/.github/workflows/downstreams.yml b/.github/workflows/downstreams.yml index 97f3ef2..192b1eb 100644 --- a/.github/workflows/downstreams.yml +++ b/.github/workflows/downstreams.yml @@ -20,6 +20,10 @@ permissions: contents: read pull-requests: write +env: + HATCH_VERBOSE: "2" + HATCH_VERSION: "1.16.5" + jobs: compatibility: strategy: @@ -44,11 +48,12 @@ jobs: - name: Install toolchain run: | - pip install hatch==1.15.0 + pip install "hatch==${HATCH_VERSION}" + - name: Downstreams uses: databrickslabs/sandbox/downstreams@acceptance/v0.4.2 with: repo: ${{ matrix.downstream.name }} org: databrickslabs env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 0498aeb..6c8b25a 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -14,6 +14,10 @@ permissions: concurrency: group: single-acceptance-job-per-repo +env: + HATCH_VERBOSE: "2" + HATCH_VERSION: "1.16.5" + jobs: integration: environment: account-admin @@ -32,7 +36,7 @@ jobs: python-version: '3.10' - name: Install hatch - run: pip install hatch==1.15.0 + run: pip install "hatch==${HATCH_VERSION}" - name: Run nightly tests uses: databrickslabs/sandbox/acceptance@acceptance/v0.4.2 @@ -43,4 +47,4 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} ARM_CLIENT_ID: ${{ secrets.ARM_CLIENT_ID }} ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }} - TEST_NIGHTLY: true + TEST_NIGHTLY: true \ No newline at end of file diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 408cf04..d898745 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -15,7 +15,8 @@ on: - main env: - HATCH_VERSION: 1.15.0 + HATCH_VERBOSE: "2" + HATCH_VERSION: "1.16.5" jobs: ci: @@ -38,7 +39,7 @@ jobs: python-version: ${{ matrix.pyVersion }} - name: Install hatch - run: pip install hatch==$HATCH_VERSION + run: pip install "hatch==${HATCH_VERSION}" - name: Run unit tests run: hatch run test @@ -64,7 +65,7 @@ jobs: python-version: 3.10.x - name: Install hatch - run: pip install hatch==$HATCH_VERSION + run: pip install "hatch==${HATCH_VERSION}" - name: Reformat code run: make fmt diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8d779a0..a79be05 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -5,6 +5,10 @@ on: tags: - 'v*' +env: + HATCH_VERBOSE: "2" + HATCH_VERSION: "1.16.5" + jobs: publish: runs-on: @@ -27,7 +31,7 @@ jobs: - name: Build wheels run: | - pip install hatch==1.15.0 + pip install "hatch==${HATCH_VERSION}" hatch build - name: Draft release @@ -46,4 +50,4 @@ jobs: inputs: | dist/databricks_*.whl dist/databricks_*.tar.gz - release-signing-artifacts: true + release-signing-artifacts: true \ No newline at end of file From ab0513deeb0feea121763f16100159354ef5211d Mon Sep 17 00:00:00 2001 From: Andrew Snare Date: Tue, 10 Mar 2026 19:25:48 +0100 Subject: [PATCH 02/19] Run acceptance tests even if the PR is a draft. This reduces the requests for before integration tests have been checked. --- .github/workflows/acceptance.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/acceptance.yml b/.github/workflows/acceptance.yml index b0993d7..e68eb1e 100644 --- a/.github/workflows/acceptance.yml +++ b/.github/workflows/acceptance.yml @@ -24,7 +24,7 @@ env: jobs: integration: - if: github.event_name == 'pull_request' && github.event.pull_request.draft == false + if: github.event_name == 'pull_request' environment: account-admin runs-on: larger steps: From 03f4cd7d1a1eea1c38db3ca00315d1f42a8b9f2f Mon Sep 17 00:00:00 2001 From: Andrew Snare Date: Tue, 10 Mar 2026 19:26:50 +0100 Subject: [PATCH 03/19] Upgrade actions/setup-python: v5 -> v6 --- .github/workflows/acceptance.yml | 2 +- .github/workflows/downstreams.yml | 2 +- .github/workflows/nightly.yml | 2 +- .github/workflows/push.yml | 4 ++-- .github/workflows/release.yml | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/acceptance.yml b/.github/workflows/acceptance.yml index e68eb1e..813b2ab 100644 --- a/.github/workflows/acceptance.yml +++ b/.github/workflows/acceptance.yml @@ -34,7 +34,7 @@ jobs: fetch-depth: 0 - name: Install Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: cache: 'pip' cache-dependency-path: '**/pyproject.toml' diff --git a/.github/workflows/downstreams.yml b/.github/workflows/downstreams.yml index 192b1eb..f93eca0 100644 --- a/.github/workflows/downstreams.yml +++ b/.github/workflows/downstreams.yml @@ -40,7 +40,7 @@ jobs: fetch-depth: 0 - name: Install Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: cache: 'pip' cache-dependency-path: '**/pyproject.toml' diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 6c8b25a..0964670 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -29,7 +29,7 @@ jobs: fetch-depth: 0 - name: Install Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: cache: 'pip' cache-dependency-path: '**/pyproject.toml' diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index d898745..558d522 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -32,7 +32,7 @@ jobs: fetch-depth: 0 - name: Install Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: cache: 'pip' cache-dependency-path: '**/pyproject.toml' @@ -58,7 +58,7 @@ jobs: fetch-depth: 0 - name: Install Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: cache: 'pip' cache-dependency-path: '**/pyproject.toml' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a79be05..e6b358f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -23,7 +23,7 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v6 with: cache: 'pip' cache-dependency-path: '**/pyproject.toml' From 5d209585f6b591401110252da8abcc56bf37fc49 Mon Sep 17 00:00:00 2001 From: Andrew Snare Date: Tue, 10 Mar 2026 19:29:49 +0100 Subject: [PATCH 04/19] Upgrade actions/checkout: v4 -> v6 --- .github/workflows/acceptance.yml | 2 +- .github/workflows/downstreams.yml | 2 +- .github/workflows/nightly.yml | 2 +- .github/workflows/push.yml | 4 ++-- .github/workflows/release.yml | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/acceptance.yml b/.github/workflows/acceptance.yml index 813b2ab..7cd26f9 100644 --- a/.github/workflows/acceptance.yml +++ b/.github/workflows/acceptance.yml @@ -29,7 +29,7 @@ jobs: runs-on: larger steps: - name: Checkout Code - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: fetch-depth: 0 diff --git a/.github/workflows/downstreams.yml b/.github/workflows/downstreams.yml index f93eca0..88f3c60 100644 --- a/.github/workflows/downstreams.yml +++ b/.github/workflows/downstreams.yml @@ -35,7 +35,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4.1.7 + uses: actions/checkout@v6 with: fetch-depth: 0 diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 0964670..bdea893 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -24,7 +24,7 @@ jobs: runs-on: larger steps: - name: Checkout Code - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: fetch-depth: 0 diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 558d522..4f64233 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -27,7 +27,7 @@ jobs: pyVersion: [ '3.10', '3.11', '3.12' ] steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: fetch-depth: 0 @@ -53,7 +53,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: fetch-depth: 0 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e6b358f..bfd5ad2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -21,7 +21,7 @@ jobs: # Used to attach signing artifacts to the published release. contents: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - uses: actions/setup-python@v6 with: From cfccee9f338bfbb3cf14544fe8efd5e91774624a Mon Sep 17 00:00:00 2001 From: Andrew Snare Date: Tue, 10 Mar 2026 19:34:17 +0100 Subject: [PATCH 05/19] Remove redundant type arguments. Recent pylint flags this for fixing. --- .../labs/pytester/fixtures/baseline.py | 6 +---- .../labs/pytester/fixtures/catalog.py | 22 +++++-------------- .../labs/pytester/fixtures/compute.py | 14 +++++------- src/databricks/labs/pytester/fixtures/iam.py | 2 +- src/databricks/labs/pytester/fixtures/ml.py | 4 ++-- .../labs/pytester/fixtures/redash.py | 2 +- .../labs/pytester/fixtures/workspace.py | 10 ++++----- 7 files changed, 21 insertions(+), 39 deletions(-) diff --git a/src/databricks/labs/pytester/fixtures/baseline.py b/src/databricks/labs/pytester/fixtures/baseline.py index ead15c1..36f674f 100644 --- a/src/databricks/labs/pytester/fixtures/baseline.py +++ b/src/databricks/labs/pytester/fixtures/baseline.py @@ -60,11 +60,7 @@ def inner(k=16) -> str: T = TypeVar("T") -def factory( - name: str, - create: Callable[..., T], - remove: Callable[[T], None], -) -> Generator[Callable[..., T], None, None]: +def factory(name: str, create: Callable[..., T], remove: Callable[[T], None]) -> Generator[Callable[..., T]]: """ Factory function for creating fixtures. diff --git a/src/databricks/labs/pytester/fixtures/catalog.py b/src/databricks/labs/pytester/fixtures/catalog.py index 37656f4..c1b729b 100644 --- a/src/databricks/labs/pytester/fixtures/catalog.py +++ b/src/databricks/labs/pytester/fixtures/catalog.py @@ -52,7 +52,7 @@ def make_table( make_random, log_workspace_link, watchdog_remove_after, -) -> Generator[Callable[..., TableInfo], None, None]: +) -> Generator[Callable[..., TableInfo]]: """ Create a table and return its info. Remove it after the test. Returns instance of `databricks.sdk.service.catalog.TableInfo`. @@ -245,7 +245,7 @@ def remove(table_info: TableInfo): @fixture def make_schema( sql_backend, make_random, log_workspace_link, watchdog_remove_after -) -> Generator[Callable[..., SchemaInfo], None, None]: +) -> Generator[Callable[..., SchemaInfo]]: """ Create a schema and return its info. Remove it after the test. Returns instance of `databricks.sdk.service.catalog.SchemaInfo`. @@ -286,9 +286,7 @@ def remove(schema_info: SchemaInfo): @fixture -def make_catalog( - ws, make_random, watchdog_remove_after, log_workspace_link -) -> Generator[Callable[..., CatalogInfo], None, None]: +def make_catalog(ws, make_random, watchdog_remove_after, log_workspace_link) -> Generator[Callable[..., CatalogInfo]]: """ Create a catalog and return its info. Remove it after the test. Returns instance of `databricks.sdk.service.catalog.CatalogInfo`. @@ -322,13 +320,7 @@ def create(*, name: str | None = None) -> CatalogInfo: @fixture -def make_udf( - ws, - env_or_skip, - sql_backend, - make_schema, - make_random, -) -> Generator[Callable[..., FunctionInfo], None, None]: +def make_udf(ws, env_or_skip, sql_backend, make_schema, make_random) -> Generator[Callable[..., FunctionInfo]]: """ Create a UDF and return its info. Remove it after the test. Returns instance of `databricks.sdk.service.catalog.FunctionInfo`. @@ -394,7 +386,7 @@ def remove(udf_info: FunctionInfo): @fixture -def make_storage_credential(ws, watchdog_remove_after) -> Generator[Callable[..., StorageCredentialInfo], None, None]: +def make_storage_credential(ws, watchdog_remove_after) -> Generator[Callable[..., StorageCredentialInfo]]: """ Create a storage credential and return its info. Remove it after the test. Returns instance of `databricks.sdk.service.catalog.StorageCredentialInfo`. @@ -449,9 +441,7 @@ def remove(storage_credential: StorageCredentialInfo): @fixture -def make_volume( - ws, make_catalog, make_schema, make_random, log_workspace_link -) -> Generator[Callable[..., VolumeInfo], None, None]: +def make_volume(ws, make_catalog, make_schema, make_random, log_workspace_link) -> Generator[Callable[..., VolumeInfo]]: """ Create a volume and return its info. Remove it after the test. Returns instance of `databricks.sdk.service.catalog.VolumeInfo`. diff --git a/src/databricks/labs/pytester/fixtures/compute.py b/src/databricks/labs/pytester/fixtures/compute.py index 6ea90d6..cfefc31 100644 --- a/src/databricks/labs/pytester/fixtures/compute.py +++ b/src/databricks/labs/pytester/fixtures/compute.py @@ -31,7 +31,7 @@ def make_cluster_policy( make_random, log_workspace_link, watchdog_purge_suffix, -) -> Generator[Callable[..., CreatePolicyResponse], None, None]: +) -> Generator[Callable[..., CreatePolicyResponse]]: """ Create a Databricks cluster policy and clean it up after the test. Returns a function to create cluster policies, which returns `databricks.sdk.service.compute.CreatePolicyResponse` instance. @@ -65,7 +65,7 @@ def create(*, name: str | None = None, **kwargs) -> CreatePolicyResponse: @fixture def make_cluster( ws, make_random, log_workspace_link, watchdog_remove_after -) -> Generator[Callable[..., Wait[ClusterDetails]], None, None]: +) -> Generator[Callable[..., Wait[ClusterDetails]]]: """ Create a Databricks cluster, waits for it to start, and clean it up after the test. Returns a function to create clusters. You can get `cluster_id` attribute from the returned object. @@ -129,7 +129,7 @@ def make_instance_pool( make_random, log_workspace_link, watchdog_remove_after, -) -> Generator[Callable[..., CreateInstancePoolResponse], None, None]: +) -> Generator[Callable[..., CreateInstancePoolResponse]]: """ Create a Databricks instance pool and clean it up after the test. Returns a function to create instance pools. Use `instance_pool_id` attribute from the returned object to get an ID of the pool. @@ -171,7 +171,7 @@ def make_job( make_workspace_file, log_workspace_link, watchdog_remove_after, -) -> Generator[Callable[..., Job], None, None]: +) -> Generator[Callable[..., Job]]: """ Create a Databricks job and clean it up after the test. Returns a function to create jobs, that returns a `databricks.sdk.service.jobs.Job` instance. @@ -271,7 +271,7 @@ def make_pipeline( make_notebook, watchdog_remove_after, watchdog_purge_suffix, -) -> Generator[Callable[..., CreatePipelineResponse], None, None]: +) -> Generator[Callable[..., CreatePipelineResponse]]: """ Create Delta Live Table Pipeline and clean it up after the test. Returns a function to create pipelines. Results in a `databricks.sdk.service.pipelines.CreatePipelineResponse` instance. @@ -315,9 +315,7 @@ def create(**kwargs) -> CreatePipelineResponse: @fixture -def make_warehouse( - ws, make_random, watchdog_remove_after -) -> Generator[Callable[..., Wait[GetWarehouseResponse]], None, None]: +def make_warehouse(ws, make_random, watchdog_remove_after) -> Generator[Callable[..., Wait[GetWarehouseResponse]]]: """ Create a Databricks warehouse and clean it up after the test. Returns a function to create warehouses. diff --git a/src/databricks/labs/pytester/fixtures/iam.py b/src/databricks/labs/pytester/fixtures/iam.py index 6b8bb21..a5fd6c4 100644 --- a/src/databricks/labs/pytester/fixtures/iam.py +++ b/src/databricks/labs/pytester/fixtures/iam.py @@ -157,7 +157,7 @@ def _double_check_group_in_listing() -> None: def _make_group( name: str, cfg: Config, interface, make_random, watchdog_purge_suffix -) -> Generator[Callable[..., Group], None, None]: +) -> Generator[Callable[..., Group]]: _not_specified = object() @retried(on=[ResourceConflict], timeout=timedelta(seconds=30)) diff --git a/src/databricks/labs/pytester/fixtures/ml.py b/src/databricks/labs/pytester/fixtures/ml.py index f5c15f3..541d28a 100644 --- a/src/databricks/labs/pytester/fixtures/ml.py +++ b/src/databricks/labs/pytester/fixtures/ml.py @@ -29,7 +29,7 @@ def make_experiment( make_directory, log_workspace_link, watchdog_purge_suffix, -) -> Generator[Callable[..., CreateExperimentResponse], None, None]: +) -> Generator[Callable[..., CreateExperimentResponse]]: """ Returns a function to create Databricks Experiments and clean them up after the test. The function returns a `databricks.sdk.service.ml.CreateExperimentResponse` object. @@ -71,7 +71,7 @@ def create( @fixture -def make_model(ws, make_random, watchdog_remove_after) -> Generator[Callable[..., ModelDatabricks], None, None]: +def make_model(ws, make_random, watchdog_remove_after) -> Generator[Callable[..., ModelDatabricks]]: """ Returns a function to create Databricks Models and clean them up after the test. The function returns a `databricks.sdk.service.ml.GetModelResponse` object. diff --git a/src/databricks/labs/pytester/fixtures/redash.py b/src/databricks/labs/pytester/fixtures/redash.py index 2b56c53..71d865a 100644 --- a/src/databricks/labs/pytester/fixtures/redash.py +++ b/src/databricks/labs/pytester/fixtures/redash.py @@ -14,7 +14,7 @@ def make_query( make_random, log_workspace_link, watchdog_remove_after, -) -> Generator[Callable[..., LegacyQuery], None, None]: +) -> Generator[Callable[..., LegacyQuery]]: """ Create a query and remove it after the test is done. Returns the `databricks.sdk.service.sql.LegacyQuery` object. diff --git a/src/databricks/labs/pytester/fixtures/workspace.py b/src/databricks/labs/pytester/fixtures/workspace.py index 50704f5..ce52a11 100644 --- a/src/databricks/labs/pytester/fixtures/workspace.py +++ b/src/databricks/labs/pytester/fixtures/workspace.py @@ -18,7 +18,7 @@ @fixture -def make_notebook(ws, make_random, watchdog_purge_suffix) -> Generator[Callable[..., WorkspacePath], None, None]: +def make_notebook(ws, make_random, watchdog_purge_suffix) -> Generator[Callable[..., WorkspacePath]]: """ Returns a function to create Databricks Notebooks and clean them up after the test. The function returns [`os.PathLike` object](https://github.com/databrickslabs/blueprint?tab=readme-ov-file#python-native-pathlibpath-like-interfaces). @@ -75,7 +75,7 @@ def create( @fixture -def make_workspace_file(ws, make_random, watchdog_purge_suffix) -> Generator[Callable[..., WorkspacePath], None, None]: +def make_workspace_file(ws, make_random, watchdog_purge_suffix) -> Generator[Callable[..., WorkspacePath]]: """ Returns a function to create Databricks workspace file and clean up after the test. The function returns [`os.PathLike` object](https://github.com/databrickslabs/blueprint?tab=readme-ov-file#python-native-pathlibpath-like-interfaces). @@ -136,9 +136,7 @@ def create( @fixture -def make_directory( - ws: WorkspaceClient, make_random, watchdog_purge_suffix -) -> Generator[Callable[..., WorkspacePath], None, None]: +def make_directory(ws: WorkspaceClient, make_random, watchdog_purge_suffix) -> Generator[Callable[..., WorkspacePath]]: """ Returns a function to create Databricks Workspace Folders and clean them up after the test. The function returns [`os.PathLike` object](https://github.com/databrickslabs/blueprint?tab=readme-ov-file#python-native-pathlibpath-like-interfaces). @@ -169,7 +167,7 @@ def create(*, path: str | Path | None = None) -> WorkspacePath: @fixture -def make_repo(ws, make_random, watchdog_purge_suffix) -> Generator[Callable[..., RepoInfo], None, None]: +def make_repo(ws, make_random, watchdog_purge_suffix) -> Generator[Callable[..., RepoInfo]]: """ Returns a function to create Databricks Repos and clean them up after the test. The function returns a `databricks.sdk.service.workspace.RepoInfo` object. From 73841b720b7927b49cf12cd0aa00ad5cd43300a4 Mon Sep 17 00:00:00 2001 From: Andrew Snare Date: Tue, 10 Mar 2026 19:34:55 +0100 Subject: [PATCH 06/19] Guard against attribute that can be null. Recent mypy flags this. --- .../labs/pytester/fixtures/permissions.py | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/src/databricks/labs/pytester/fixtures/permissions.py b/src/databricks/labs/pytester/fixtures/permissions.py index a10cc6e..a875096 100644 --- a/src/databricks/labs/pytester/fixtures/permissions.py +++ b/src/databricks/labs/pytester/fixtures/permissions.py @@ -1,3 +1,5 @@ +from typing import TypeVar + import pytest from databricks.sdk.errors import InvalidParameterValue from databricks.sdk.service import iam @@ -10,6 +12,16 @@ # pylint: disable=too-complex +T = TypeVar("T") + + +# TODO: Move this somewhere more useful. +def _not_none(value: T | None, *, msg: str = "missing value") -> T: + if value is None: + raise ValueError(msg) + return value + + class _PermissionsChange: def __init__(self, object_id: str, before: list[iam.AccessControlRequest], after: list[iam.AccessControlRequest]): self.object_id = object_id @@ -24,8 +36,10 @@ def _principal(acr: iam.AccessControlRequest) -> str: return f"group_name {acr.group_name}" return f"service_principal_name {acr.service_principal_name}" - def _list(self, acl: list[iam.AccessControlRequest]): - return ", ".join(f"{self._principal(_)} {_.permission_level.value}" for _ in acl) + def _list(self, acl: list[iam.AccessControlRequest]) -> str: + return ", ".join( + f"{self._principal(_)} {_not_none(_.permission_level, msg='missing permission level').value}" for _ in acl + ) def __repr__(self): return f"{self.object_id} [{self._list(self.before)}] -> [{self._list(self.after)}]" @@ -44,7 +58,9 @@ def _principal(acr: AccessControl) -> str: return f"group_name {acr.group_name}" def _list(self, acl: list[AccessControl]): - return ", ".join(f"{self._principal(_)} {_.permission_level.value}" for _ in acl) + return ", ".join( + f"{self._principal(_)} {_not_none(_.permission_level, msg='missing permission level').value}" for _ in acl + ) def __repr__(self): return f"{self.object_id} [{self._list(self.before)}] -> [{self._list(self.after)}]" From 8a1f0ccc58a4169075504c89134883aec197086f Mon Sep 17 00:00:00 2001 From: Andrew Snare Date: Tue, 10 Mar 2026 19:35:51 +0100 Subject: [PATCH 07/19] Use the Makefile target to run the tests. --- .github/workflows/push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 4f64233..6effc9a 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -42,7 +42,7 @@ jobs: run: pip install "hatch==${HATCH_VERSION}" - name: Run unit tests - run: hatch run test + run: make test - name: Publish test coverage uses: codecov/codecov-action@v5 From 9c3bef959f4e84e1629dd128e61323ec7f0052f7 Mon Sep 17 00:00:00 2001 From: Andrew Snare Date: Tue, 10 Mar 2026 19:37:56 +0100 Subject: [PATCH 08/19] Upgrade mypy: 1.11.0 -> 1.19.1 --- pyproject.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 3d425a7..194330f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,7 +68,7 @@ path = "src/databricks/labs/pytester/__about__.py" dependencies = [ "black~=24.3.0", "coverage[toml]~=7.4.4", - "mypy~=1.11.0", + "mypy~=1.19.1", "pylint~=3.2.2", "pylint-pytest==2.0.0a0", "databricks-labs-pylint~=0.4.0", @@ -89,11 +89,11 @@ coverage = "pytest -n auto --cov src tests/unit --timeout 30 --cov-report=htm integration = "pytest -n 10 --cov src tests/integration --durations 20" fmt = ["black . --extend-exclude 'tests/unit/source_code/samples/'", "ruff check . --fix", - "mypy --disable-error-code 'annotation-unchecked' --disable-error-code import-untyped --enable-incomplete-feature=NewGenericSyntax --exclude 'tests/resources/*' --exclude dist .", + "mypy --disable-error-code 'annotation-unchecked' --disable-error-code import-untyped --exclude 'tests/resources/*' --exclude dist .", "pylint --output-format=colorized -j 0 src tests"] verify = ["black --check . --extend-exclude 'tests/unit/source_code/samples/'", "ruff check .", - "mypy --exclude 'tests/resources/*' --exclude dist . --enable-incomplete-feature=NewGenericSyntax", + "mypy --exclude 'tests/resources/*' --exclude dist .", "pylint --output-format=colorized -j 0 src tests"] lint = ["pylint --output-format=colorized -j 0 src tests"] From 1a945dd8cd6d1f47247332f8b7e7a29eecb314b0 Mon Sep 17 00:00:00 2001 From: Andrew Snare Date: Tue, 10 Mar 2026 19:40:30 +0100 Subject: [PATCH 09/19] Upgrade pylint: 3.2.2 -> 4.0.5 --- pyproject.toml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 194330f..c5aae32 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,8 +69,8 @@ dependencies = [ "black~=24.3.0", "coverage[toml]~=7.4.4", "mypy~=1.19.1", - "pylint~=3.2.2", - "pylint-pytest==2.0.0a0", + "pylint~=4.0.5", + "pylint-pytest==2.0.0a1", "databricks-labs-pylint~=0.4.0", "pytest-cov~=4.1.0", "pytest-mock~=3.14.0", @@ -267,10 +267,6 @@ py-version = "3.10" # source root. # source-roots = -# When enabled, pylint would attempt to guess common misconfiguration and emit -# user-friendly hints instead of false-positive error messages. -suggestion-mode = true - # Allow loading of arbitrary C extensions. Extensions are imported into the # active Python interpreter and may run arbitrary code. # unsafe-load-any-extension = @@ -443,6 +439,7 @@ bad-functions = ["map", "input"] # Maximum number of arguments for function / method. max-args = 9 +max-positional-arguments = 9 # Maximum number of attributes for a class (see R0902). max-attributes = 11 From 4a45cc3b1e1b5a582f10015096f1d594d6874fbd Mon Sep 17 00:00:00 2001 From: Andrew Snare Date: Tue, 10 Mar 2026 19:41:39 +0100 Subject: [PATCH 10/19] Upgrade pytest: 8.3 -> 9.0 Also upgrade all the pytest plugins. --- pyproject.toml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c5aae32..6e5a4cb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,7 +50,7 @@ classifiers = [ dependencies = [ "databricks-sdk>=0.51.0,<1.0", "databricks-labs-lsql>=0.10", - "pytest>=8.3", + "pytest>=9.0", ] [project.entry-points.pytest11] @@ -67,15 +67,15 @@ path = "src/databricks/labs/pytester/__about__.py" [tool.hatch.envs.default] dependencies = [ "black~=24.3.0", - "coverage[toml]~=7.4.4", + "coverage[toml]~=7.13.4", "mypy~=1.19.1", "pylint~=4.0.5", "pylint-pytest==2.0.0a1", "databricks-labs-pylint~=0.4.0", - "pytest-cov~=4.1.0", - "pytest-mock~=3.14.0", - "pytest-timeout~=2.3.1", - "pytest-xdist~=3.5.0", + "pytest-cov~=7.0.0", + "pytest-mock~=3.15.1", + "pytest-timeout~=2.4.0", + "pytest-xdist~=3.8.0", "ruff~=0.3.4", "databricks-connect~=15.4.3", ] From 8dcde24612d57b4409bdec7635c32978fea6caf8 Mon Sep 17 00:00:00 2001 From: Andrew Snare Date: Tue, 10 Mar 2026 19:43:13 +0100 Subject: [PATCH 11/19] Use environment variables to configure the ruff and pytest cache locations. These are easier to override via matrix interpolation. --- pyproject.toml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6e5a4cb..ae8c5bd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -83,6 +83,10 @@ dependencies = [ # store virtual env as the child of this folder. Helps VSCode (and PyCharm) to run better path = ".venv" +[tool.hatch.envs.default.env-vars] +PYTEST_ADDOPTS = "-o cache_dir=.venv/pytest-cache" +RUFF_CACHE_DIR = ".venv/ruff-cache" + [tool.hatch.envs.default.scripts] test = "pytest -n 4 --cov src --cov-report=xml --timeout 30 tests/unit --durations 20" coverage = "pytest -n auto --cov src tests/unit --timeout 30 --cov-report=html --durations 20" @@ -100,7 +104,6 @@ lint = ["pylint --output-format=colorized -j 0 src tests"] [tool.pytest.ini_options] # TODO: remove `-p no:warnings` addopts = "--no-header -p no:warnings" -cache_dir = ".venv/pytest-cache" [tool.black] target-version = ["py310"] @@ -108,7 +111,6 @@ line-length = 120 skip-string-normalization = true [tool.ruff] -cache-dir = ".venv/ruff-cache" target-version = "py310" line-length = 120 From aa27d2fd8336060cf4730102550431bfac38d8cb Mon Sep 17 00:00:00 2001 From: Andrew Snare Date: Tue, 10 Mar 2026 19:44:04 +0100 Subject: [PATCH 12/19] Ensure the mypy cache is also placed within the venv directory. --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index ae8c5bd..86d59ca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -84,6 +84,7 @@ dependencies = [ path = ".venv" [tool.hatch.envs.default.env-vars] +MYPY_CACHE_DIR = ".venv/mypy-cache" PYTEST_ADDOPTS = "-o cache_dir=.venv/pytest-cache" RUFF_CACHE_DIR = ".venv/ruff-cache" From 1b8f779e477ec2da89071d3fa1d1b7c6fbbc3527 Mon Sep 17 00:00:00 2001 From: Andrew Snare Date: Tue, 10 Mar 2026 19:47:45 +0100 Subject: [PATCH 13/19] Update project so that Hatch manages the python version in each environment instead of just using the system-provided python. --- .github/workflows/push.yml | 6 ++++-- .gitignore | 1 + Makefile | 2 +- pyproject.toml | 24 +++++++++++++++++++++++- 4 files changed, 29 insertions(+), 4 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 6effc9a..c628139 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -24,7 +24,9 @@ jobs: strategy: fail-fast: false matrix: - pyVersion: [ '3.10', '3.11', '3.12' ] + python: [ '3.10', '3.11', '3.12' ] + env: + HATCH_ENV: "test.py${{ matrix.python }}" steps: - name: Checkout uses: actions/checkout@v6 @@ -36,7 +38,7 @@ jobs: with: cache: 'pip' cache-dependency-path: '**/pyproject.toml' - python-version: ${{ matrix.pyVersion }} + python-version: '3.10' - name: Install hatch run: pip install "hatch==${HATCH_VERSION}" diff --git a/.gitignore b/.gitignore index 822e00a..b3203b6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ .idea/ .venv/ +.venv-*/ *.iml *.pyc .pytest_cache diff --git a/Makefile b/Makefile index a953697..1550724 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ all: clean lint fmt test coverage clean: - rm -fr .venv clean htmlcov .mypy_cache .pytest_cache .ruff_cache .coverage coverage.xml + rm -fr .venv* clean htmlcov .mypy_cache .pytest_cache .ruff_cache .coverage coverage.xml rm -fr **/*.pyc .venv/bin/python: diff --git a/pyproject.toml b/pyproject.toml index 86d59ca..7f12bfa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,7 +80,9 @@ dependencies = [ "databricks-connect~=15.4.3", ] -# store virtual env as the child of this folder. Helps VSCode (and PyCharm) to run better +python="3.10" + +# Fixed path expected by the `labs` tool to run acceptance tests. path = ".venv" [tool.hatch.envs.default.env-vars] @@ -102,6 +104,26 @@ verify = ["black --check . --extend-exclude 'tests/unit/source_code/samples "pylint --output-format=colorized -j 0 src tests"] lint = ["pylint --output-format=colorized -j 0 src tests"] +[tool.hatch.envs.test.env-vars] +MYPY_CACHE_DIR = ".venv-py{matrix:python}/mypy-cache" +PYTEST_ADDOPTS = "-o cache_dir=.venv-py{matrix:python}/pytest-cache" +RUFF_CACHE_DIR = ".venv-py{matrix:python}/ruff-cache" + +[tool.hatch.envs.test.overrides] +matrix.python.path = [ + # Sadly no way to interpolate this. + { value = ".venv-py3.10", if = ["3.10" ]}, + { value = ".venv-py3.11", if = ["3.11" ]}, + { value = ".venv-py3.12", if = ["3.12" ]}, +] + +[[tool.hatch.envs.test.matrix]] +python = [ + "3.10", + "3.11", + "3.12", +] + [tool.pytest.ini_options] # TODO: remove `-p no:warnings` addopts = "--no-header -p no:warnings" From 2a69068b49f4aa07e03d372255666a05273323f5 Mon Sep 17 00:00:00 2001 From: Andrew Snare Date: Tue, 10 Mar 2026 19:49:36 +0100 Subject: [PATCH 14/19] Update project and testing to also support python 3.13 and 3.14. --- .github/workflows/push.yml | 2 +- pyproject.toml | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index c628139..f936997 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -24,7 +24,7 @@ jobs: strategy: fail-fast: false matrix: - python: [ '3.10', '3.11', '3.12' ] + python: [ '3.10', '3.11', '3.12', '3.13', '3.14' ] env: HATCH_ENV: "test.py${{ matrix.python }}" steps: diff --git a/pyproject.toml b/pyproject.toml index 7f12bfa..58bb6f1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,8 @@ classifiers = [ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "Programming Language :: Python :: Implementation :: CPython", "Environment :: Console", "Framework :: Pytest", @@ -115,6 +117,8 @@ matrix.python.path = [ { value = ".venv-py3.10", if = ["3.10" ]}, { value = ".venv-py3.11", if = ["3.11" ]}, { value = ".venv-py3.12", if = ["3.12" ]}, + { value = ".venv-py3.13", if = ["3.13" ]}, + { value = ".venv-py3.14", if = ["3.14" ]} ] [[tool.hatch.envs.test.matrix]] @@ -122,6 +126,8 @@ python = [ "3.10", "3.11", "3.12", + "3.13", + "3.14", ] [tool.pytest.ini_options] From c44e40eba01b177ea0c949b57eca9d00a72ac753 Mon Sep 17 00:00:00 2001 From: Andrew Snare Date: Tue, 10 Mar 2026 19:50:49 +0100 Subject: [PATCH 15/19] Update CI/CD to run Hatch under python 3.14 where possible. This doesn't affect the python version in the environment under test. --- .github/workflows/acceptance.yml | 2 +- .github/workflows/downstreams.yml | 1 + .github/workflows/nightly.yml | 2 +- .github/workflows/push.yml | 4 ++-- .github/workflows/release.yml | 2 +- 5 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/acceptance.yml b/.github/workflows/acceptance.yml index 7cd26f9..14f9326 100644 --- a/.github/workflows/acceptance.yml +++ b/.github/workflows/acceptance.yml @@ -38,7 +38,7 @@ jobs: with: cache: 'pip' cache-dependency-path: '**/pyproject.toml' - python-version: '3.10' + python-version: '3.14' - name: Install hatch run: pip install "hatch==${HATCH_VERSION}" diff --git a/.github/workflows/downstreams.yml b/.github/workflows/downstreams.yml index 88f3c60..1da3d0d 100644 --- a/.github/workflows/downstreams.yml +++ b/.github/workflows/downstreams.yml @@ -44,6 +44,7 @@ jobs: with: cache: 'pip' cache-dependency-path: '**/pyproject.toml' + # Downstream currently just tests with system python rather than controlling the versions properly. python-version: '3.10' - name: Install toolchain diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index bdea893..6e1b185 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -33,7 +33,7 @@ jobs: with: cache: 'pip' cache-dependency-path: '**/pyproject.toml' - python-version: '3.10' + python-version: '3.14' - name: Install hatch run: pip install "hatch==${HATCH_VERSION}" diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index f936997..00ae80a 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -38,7 +38,7 @@ jobs: with: cache: 'pip' cache-dependency-path: '**/pyproject.toml' - python-version: '3.10' + python-version: '3.14' - name: Install hatch run: pip install "hatch==${HATCH_VERSION}" @@ -64,7 +64,7 @@ jobs: with: cache: 'pip' cache-dependency-path: '**/pyproject.toml' - python-version: 3.10.x + python-version: '3.14' - name: Install hatch run: pip install "hatch==${HATCH_VERSION}" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index bfd5ad2..e11f8d3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -27,7 +27,7 @@ jobs: with: cache: 'pip' cache-dependency-path: '**/pyproject.toml' - python-version: '3.10' + python-version: '3.14' - name: Build wheels run: | From 5560ef00ea0c510113548ed438329a29937347ca Mon Sep 17 00:00:00 2001 From: Andrew Snare Date: Tue, 10 Mar 2026 19:51:27 +0100 Subject: [PATCH 16/19] Update project metadata to indicate that Python 3.14 is the highest version supported. --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 58bb6f1..1467c5a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ dynamic = ["version"] description = 'Python Testing for Databricks' readme = "README.md" license-files = { paths = ["LICENSE", "NOTICE"] } -requires-python = ">=3.10" +requires-python = ">=3.10,<3.15" keywords = ["Databricks", "pytest"] maintainers = [ { name = "Serge Smertin", email = "serge.smertin@databricks.com" }, From 6c707fe1ea10d5d9513524d1023e8159d802346c Mon Sep 17 00:00:00 2001 From: Andrew Snare Date: Tue, 10 Mar 2026 19:58:30 +0100 Subject: [PATCH 17/19] EOL at EOF. --- .github/workflows/acceptance.yml | 2 +- .github/workflows/downstreams.yml | 2 +- .github/workflows/nightly.yml | 2 +- .github/workflows/release.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/acceptance.yml b/.github/workflows/acceptance.yml index 14f9326..9997c24 100644 --- a/.github/workflows/acceptance.yml +++ b/.github/workflows/acceptance.yml @@ -55,4 +55,4 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} ARM_CLIENT_ID: ${{ secrets.ARM_CLIENT_ID }} - ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }} \ No newline at end of file + ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }} diff --git a/.github/workflows/downstreams.yml b/.github/workflows/downstreams.yml index 1da3d0d..ab88940 100644 --- a/.github/workflows/downstreams.yml +++ b/.github/workflows/downstreams.yml @@ -57,4 +57,4 @@ jobs: repo: ${{ matrix.downstream.name }} org: databrickslabs env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 6e1b185..ec94f87 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -47,4 +47,4 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} ARM_CLIENT_ID: ${{ secrets.ARM_CLIENT_ID }} ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }} - TEST_NIGHTLY: true \ No newline at end of file + TEST_NIGHTLY: true diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e11f8d3..0f46af6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -50,4 +50,4 @@ jobs: inputs: | dist/databricks_*.whl dist/databricks_*.tar.gz - release-signing-artifacts: true \ No newline at end of file + release-signing-artifacts: true From 220e95eb9549b326d4c18e6a7ea3ec7d18697f70 Mon Sep 17 00:00:00 2001 From: Andrew Snare Date: Tue, 10 Mar 2026 20:11:56 +0100 Subject: [PATCH 18/19] No need to trigger on ready-for-review. Draft PRs also trigger a run. --- .github/workflows/acceptance.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/acceptance.yml b/.github/workflows/acceptance.yml index 9997c24..ee0665c 100644 --- a/.github/workflows/acceptance.yml +++ b/.github/workflows/acceptance.yml @@ -2,7 +2,7 @@ name: acceptance on: pull_request: - types: [ opened, synchronize, ready_for_review ] + types: [ opened, synchronize ] merge_group: types: [ checks_requested ] push: From 4b1fdab2d81d01251598965e88bd24a399f6d4f9 Mon Sep 17 00:00:00 2001 From: Andrew Snare Date: Wed, 11 Mar 2026 14:10:27 +0100 Subject: [PATCH 19/19] Preinstall the python version under test so that Hatch doesn't need to download it. --- .github/workflows/push.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 00ae80a..c4b6704 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -38,7 +38,10 @@ jobs: with: cache: 'pip' cache-dependency-path: '**/pyproject.toml' - python-version: '3.14' + # The version under test, as well as Python 3.14 for Hatch. + python-version: | + ${{ matrix.python }} + ${{ matrix.python != '3.14' && '3.14' || '' }} - name: Install hatch run: pip install "hatch==${HATCH_VERSION}"