diff --git a/docs/usage.md b/docs/usage.md index 6b8c593ef..91ede088d 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -540,6 +540,42 @@ pep621_dev_dependency_groups = ["test", "docs"] deptry . --pep621-dev-dependency-groups "test,docs" ``` +#### Non-dev dependency groups + +By default, _deptry_ considers that groups defined under `[dependency-groups]` contain development dependencies. Some +projects use dependency groups to define optional regular dependencies. To account for that, it is possible to specify a +list of groups that need to be treated as containing regular dependencies. + +For example, consider a project with the following `pyproject.toml`: + +```toml +[project] +... +dependencies = ["httpx"] + +[dependency-groups] +server = ["uvicorn"] +telemetry = ["opentelemetry-sdk"] +``` + +By default, `uvicorn` and `opentelemetry-sdk` are extracted as development dependencies. By specifying +`--non-dev-dependency-groups=server,telemetry`, `uvicorn` and `opentelemetry-sdk` will be treated as regular +dependencies instead. + +- Type: `list[str]` +- Default: `[]` +- `pyproject.toml` option name: `non_dev_dependency_groups` +- CLI option name: `--non-dev-dependency-groups` (short: `-nddg`) +- `pyproject.toml` example: +```toml +[tool.deptry] +non_dev_dependency_groups = ["server", "telemetry"] +``` +- CLI example: +```shell +deptry . --non-dev-dependency-groups "server,telemetry" +``` + #### Experimental namespace package !!! warning diff --git a/python/deptry/cli.py b/python/deptry/cli.py index 610be45b6..cfa2600bb 100644 --- a/python/deptry/cli.py +++ b/python/deptry/cli.py @@ -262,6 +262,14 @@ def display_deptry_version(ctx: click.Context, _param: click.Parameter, value: b default=(), show_default=False, ) +@click.option( + "--non-dev-dependency-groups", + "-nddg", + type=COMMA_SEPARATED_TUPLE, + help="Specify which groups in [dependency-groups] should be considered as groups containing regular dependencies instead of development ones", + default=(), + show_default=False, +) @click.option( "--experimental-namespace-package", is_flag=True, @@ -287,6 +295,7 @@ def cli( package_module_name_map: MutableMapping[str, tuple[str, ...]], pep621_dev_dependency_groups: tuple[str, ...], optional_dependencies_dev_groups: tuple[str, ...], + non_dev_dependency_groups: tuple[str, ...], experimental_namespace_package: bool, ) -> None: """Find dependency issues in your Python project. @@ -324,6 +333,7 @@ def cli( github_warning_errors=github_warning_errors, package_module_name_map=package_module_name_map, optional_dependencies_dev_groups=pep621_dev_dependency_groups or optional_dependencies_dev_groups, + non_dev_dependency_groups=non_dev_dependency_groups, experimental_namespace_package=experimental_namespace_package, ).run() diff --git a/python/deptry/core.py b/python/deptry/core.py index deab1804a..048e9d16b 100644 --- a/python/deptry/core.py +++ b/python/deptry/core.py @@ -39,6 +39,7 @@ class Core: json_output: str package_module_name_map: Mapping[str, tuple[str, ...]] optional_dependencies_dev_groups: tuple[str, ...] + non_dev_dependency_groups: tuple[str, ...] experimental_namespace_package: bool github_output: bool github_warning_errors: tuple[str, ...] @@ -50,6 +51,7 @@ def run(self) -> None: self.config, self.package_module_name_map, self.optional_dependencies_dev_groups, + self.non_dev_dependency_groups, self.requirements_files, self.using_default_requirements_files, self.requirements_files_dev, diff --git a/python/deptry/dependency_getter/builder.py b/python/deptry/dependency_getter/builder.py index 80add4874..e0a295d30 100644 --- a/python/deptry/dependency_getter/builder.py +++ b/python/deptry/dependency_getter/builder.py @@ -34,6 +34,7 @@ class DependencyGetterBuilder: config: Path package_module_name_map: Mapping[str, tuple[str, ...]] = field(default_factory=dict) optional_dependencies_dev_groups: tuple[str, ...] = () + non_dev_dependency_groups: tuple[str, ...] = () requirements_files: tuple[str, ...] = () using_default_requirements_files: bool = True requirements_files_dev: tuple[str, ...] = () @@ -46,7 +47,10 @@ def build(self) -> DependencyGetter: if self._project_uses_poetry(pyproject_toml): return PoetryDependencyGetter( - self.config, self.package_module_name_map, self.optional_dependencies_dev_groups + self.config, + self.package_module_name_map, + self.optional_dependencies_dev_groups, + self.non_dev_dependency_groups, ) if self._project_uses_uv(pyproject_toml): diff --git a/python/deptry/dependency_getter/pep621/base.py b/python/deptry/dependency_getter/pep621/base.py index ae91a4d25..3970a11f4 100644 --- a/python/deptry/dependency_getter/pep621/base.py +++ b/python/deptry/dependency_getter/pep621/base.py @@ -1,6 +1,5 @@ from __future__ import annotations -import itertools import logging from dataclasses import dataclass from typing import TYPE_CHECKING @@ -43,6 +42,7 @@ class PEP621DependencyGetter(DependencyGetter): """ optional_dependencies_dev_groups: tuple[str, ...] = () + non_dev_dependency_groups: tuple[str, ...] = () def get(self) -> DependenciesExtract: dependencies = self._get_dependencies() @@ -52,9 +52,12 @@ def get(self) -> DependenciesExtract: dev_dependencies_from_optional, remaining_optional_dependencies = ( self._split_development_dependencies_from_optional_dependencies(optional_dependencies) ) + dev_dependencies_from_dependency_groups, remaining_dependency_groups_dependencies = ( + self._split_development_dependencies_from_dependency_groups(dependency_groups_dependencies) + ) return DependenciesExtract( - [*dependencies, *remaining_optional_dependencies], - self._get_dev_dependencies(dependency_groups_dependencies, dev_dependencies_from_optional), + [*dependencies, *remaining_optional_dependencies, *remaining_dependency_groups_dependencies], + self._get_dev_dependencies(dev_dependencies_from_optional, dev_dependencies_from_dependency_groups), ) def _get_dependencies(self) -> list[Dependency]: @@ -109,13 +112,10 @@ def _get_dependency_groups_dependencies(self) -> dict[str, list[Dependency]]: def _get_dev_dependencies( self, - dependency_groups_dependencies: dict[str, list[Dependency]], dev_dependencies_from_optional: list[Dependency], + dev_dependencies_from_dependency_groups: list[Dependency], ) -> list[Dependency]: - return [ - *itertools.chain(*dependency_groups_dependencies.values()), - *dev_dependencies_from_optional, - ] + return [*dev_dependencies_from_optional, *dev_dependencies_from_dependency_groups] @staticmethod def _project_uses_setuptools(pyproject_toml: dict[str, Any]) -> bool: @@ -139,16 +139,6 @@ def _project_uses_setuptools(pyproject_toml: dict[str, Any]) -> bool: ) return False - def _check_for_invalid_group_names(self, optional_dependencies: dict[str, list[Dependency]]) -> None: - missing_groups = set(self.optional_dependencies_dev_groups) - set(optional_dependencies.keys()) - if missing_groups: - logging.warning( - "Warning: Trying to extract the dependencies from the optional dependency groups %s as development dependencies, " - "but the following groups were not found: %s", - list(self.optional_dependencies_dev_groups), - list(missing_groups), - ) - def _split_development_dependencies_from_optional_dependencies( self, optional_dependencies: dict[str, list[Dependency]] ) -> tuple[list[Dependency], list[Dependency]]: @@ -159,8 +149,15 @@ def _split_development_dependencies_from_optional_dependencies( dev_dependencies: list[Dependency] = [] regular_dependencies: list[Dependency] = [] - if self.optional_dependencies_dev_groups: - self._check_for_invalid_group_names(optional_dependencies) + if self.optional_dependencies_dev_groups and ( + missing_groups := set(self.optional_dependencies_dev_groups) - set(optional_dependencies.keys()) + ): + logging.warning( + "Warning: Trying to extract the dependencies from the optional dependency groups %s as development dependencies, " + "but the following groups were not found: %s", + list(self.optional_dependencies_dev_groups), + list(missing_groups), + ) for group, dependencies in optional_dependencies.items(): if group in self.optional_dependencies_dev_groups: @@ -170,6 +167,30 @@ def _split_development_dependencies_from_optional_dependencies( return dev_dependencies, regular_dependencies + def _split_development_dependencies_from_dependency_groups( + self, dependency_groups: dict[str, list[Dependency]] + ) -> tuple[list[Dependency], list[Dependency]]: + dev_dependencies: list[Dependency] = [] + regular_dependencies: list[Dependency] = [] + + if self.non_dev_dependency_groups and ( + missing_groups := set(self.non_dev_dependency_groups) - set(dependency_groups.keys()) + ): + logging.warning( + "Warning: Trying to extract the dependencies from the dependency groups %s as regular dependencies, " + "but the following groups were not found: %s", + list(self.non_dev_dependency_groups), + list(missing_groups), + ) + + for group, dependencies in dependency_groups.items(): + if group in self.non_dev_dependency_groups: + regular_dependencies.extend(dependencies) + else: + dev_dependencies.extend(dependencies) + + return dev_dependencies, regular_dependencies + def _extract_pep_508_dependencies(self, dependencies: list[str]) -> list[Dependency]: """ Given a list of dependency specifications (e.g. "django>2.1; os_name != 'nt'"), convert them to Dependency objects. diff --git a/python/deptry/dependency_getter/pep621/pdm.py b/python/deptry/dependency_getter/pep621/pdm.py index ba2497a15..f1b3b5ee7 100644 --- a/python/deptry/dependency_getter/pep621/pdm.py +++ b/python/deptry/dependency_getter/pep621/pdm.py @@ -20,8 +20,8 @@ class PDMDependencyGetter(PEP621DependencyGetter): def _get_dev_dependencies( self, - dependency_groups_dependencies: dict[str, list[Dependency]], dev_dependencies_from_optional: list[Dependency], + dev_dependencies_from_dependency_groups: list[Dependency], ) -> list[Dependency]: """ Retrieve dev dependencies from pyproject.toml, which in PDM are specified as: @@ -36,7 +36,9 @@ def _get_dev_dependencies( "tox-pdm>=0.5", ] """ - dev_dependencies = super()._get_dev_dependencies(dependency_groups_dependencies, dev_dependencies_from_optional) + dev_dependencies = super()._get_dev_dependencies( + dev_dependencies_from_optional, dev_dependencies_from_dependency_groups + ) pyproject_data = load_pyproject_toml(self.config) diff --git a/python/deptry/dependency_getter/pep621/poetry.py b/python/deptry/dependency_getter/pep621/poetry.py index 5bbf2ef02..86d294216 100644 --- a/python/deptry/dependency_getter/pep621/poetry.py +++ b/python/deptry/dependency_getter/pep621/poetry.py @@ -38,15 +38,17 @@ def _get_dependencies(self) -> list[Dependency]: def _get_dev_dependencies( self, - dependency_groups_dependencies: dict[str, list[Dependency]], dev_dependencies_from_optional: list[Dependency], + dev_dependencies_from_dependency_groups: list[Dependency], ) -> list[Dependency]: """ Poetry's development dependencies can be specified under either, or both: - [tool.poetry.dev-dependencies] - [tool.poetry.group..dependencies] """ - dev_dependencies = super()._get_dev_dependencies(dependency_groups_dependencies, dev_dependencies_from_optional) + dev_dependencies = super()._get_dev_dependencies( + dev_dependencies_from_optional, dev_dependencies_from_dependency_groups + ) pyproject_data = load_pyproject_toml(self.config) poetry_dev_dependencies: dict[str, str] = {} diff --git a/python/deptry/dependency_getter/pep621/uv.py b/python/deptry/dependency_getter/pep621/uv.py index d016eeed9..14ae689c9 100644 --- a/python/deptry/dependency_getter/pep621/uv.py +++ b/python/deptry/dependency_getter/pep621/uv.py @@ -20,8 +20,8 @@ class UvDependencyGetter(PEP621DependencyGetter): def _get_dev_dependencies( self, - dependency_groups_dependencies: dict[str, list[Dependency]], dev_dependencies_from_optional: list[Dependency], + dev_dependencies_from_dependency_groups: list[Dependency], ) -> list[Dependency]: """ Retrieve dev dependencies from pyproject.toml, which in uv are specified as: @@ -35,7 +35,9 @@ def _get_dev_dependencies( Dev dependencies marked as such from optional dependencies are also added to the list of dev dependencies found. """ - dev_dependencies = super()._get_dev_dependencies(dependency_groups_dependencies, dev_dependencies_from_optional) + dev_dependencies = super()._get_dev_dependencies( + dev_dependencies_from_optional, dev_dependencies_from_dependency_groups + ) pyproject_data = load_pyproject_toml(self.config) diff --git a/tests/unit/dependency_getter/test_pep_621.py b/tests/unit/dependency_getter/test_pep_621.py index 648118a37..e7f25ea5b 100644 --- a/tests/unit/dependency_getter/test_pep_621.py +++ b/tests/unit/dependency_getter/test_pep_621.py @@ -69,7 +69,7 @@ def test_dependency_getter(tmp_path: Path) -> None: assert "dep" in dependencies[7].top_levels -def test_dependency_getter_with_dev_dependencies(tmp_path: Path) -> None: +def test_dependency_getter_optional_dependencies_dev_groups(tmp_path: Path) -> None: fake_pyproject_toml = """[project] name = "foo" dependencies = ["qux"] @@ -100,20 +100,20 @@ def test_dependency_getter_with_dev_dependencies(tmp_path: Path) -> None: assert dependencies[1].name == "foobar" assert "foobar" in dependencies[1].top_levels - assert dev_dependencies[0].name == "foo" - assert "foo" in dev_dependencies[0].top_levels + assert dev_dependencies[0].name == "barfoo" + assert "barfoo" in dev_dependencies[0].top_levels - assert dev_dependencies[1].name == "baz" - assert "baz" in dev_dependencies[1].top_levels + assert dev_dependencies[1].name == "foo" + assert "foo" in dev_dependencies[1].top_levels - assert dev_dependencies[2].name == "foobaz" - assert "foobaz" in dev_dependencies[2].top_levels + assert dev_dependencies[2].name == "baz" + assert "baz" in dev_dependencies[2].top_levels - assert dev_dependencies[3].name == "barfoo" - assert "barfoo" in dev_dependencies[3].top_levels + assert dev_dependencies[3].name == "foobaz" + assert "foobaz" in dev_dependencies[3].top_levels -def test_dependency_getter_with_incorrect_dev_group(tmp_path: Path, caplog: LogCaptureFixture) -> None: +def test_dependency_getter_incorrect_optional_dependencies_dev_group(tmp_path: Path, caplog: LogCaptureFixture) -> None: fake_pyproject_toml = """[project] name = "foo" dependencies = ["qux"] @@ -147,6 +147,73 @@ def test_dependency_getter_with_incorrect_dev_group(tmp_path: Path, caplog: LogC assert "barfoo" in dependencies[2].top_levels +def test_dependency_getter_non_dev_dependency_groups(tmp_path: Path) -> None: + fake_pyproject_toml = """[project] +name = "foo" +dependencies = ["qux"] + +[dependency-groups] +group1 = ["foobar"] +group2 = ["barfoo"] +""" + + with run_within_dir(tmp_path): + with Path("pyproject.toml").open("w") as f: + f.write(fake_pyproject_toml) + + getter = PEP621DependencyGetter(config=Path("pyproject.toml"), non_dev_dependency_groups=("group2",)) + dependencies = getter.get().dependencies + dev_dependencies = getter.get().dev_dependencies + + assert len(dependencies) == 2 + assert len(dev_dependencies) == 1 + + assert dependencies[0].name == "qux" + assert "qux" in dependencies[0].top_levels + + assert dependencies[1].name == "barfoo" + assert "barfoo" in dependencies[1].top_levels + + assert dev_dependencies[0].name == "foobar" + assert "foobar" in dev_dependencies[0].top_levels + + +def test_dependency_getter_incorrect_dependency_groups_non_dev_group(tmp_path: Path, caplog: LogCaptureFixture) -> None: + fake_pyproject_toml = """[project] +name = "foo" +dependencies = ["qux"] + +[dependency-groups] +group1 = ["foobar"] +group2 = ["barfoo"] +""" + + with run_within_dir(tmp_path), caplog.at_level(logging.INFO): + with Path("pyproject.toml").open("w") as f: + f.write(fake_pyproject_toml) + + getter = PEP621DependencyGetter(config=Path("pyproject.toml"), non_dev_dependency_groups=("group3",)) + dependencies = getter.get().dependencies + dev_dependencies = getter.get().dev_dependencies + + assert ( + "Trying to extract the dependencies from the dependency groups ['group3'] as regular dependencies, but the following groups were not found: ['group3']" + in caplog.text + ) + + assert len(dependencies) == 1 + assert len(dev_dependencies) == 2 + + assert dependencies[0].name == "qux" + assert "qux" in dependencies[0].top_levels + + assert dev_dependencies[0].name == "foobar" + assert "foobar" in dev_dependencies[0].top_levels + + assert dev_dependencies[1].name == "barfoo" + assert "barfoo" in dev_dependencies[1].top_levels + + def test_dependency_getter_empty_dependencies(tmp_path: Path) -> None: fake_pyproject_toml = """[project] name = "foo" diff --git a/tests/unit/test_core.py b/tests/unit/test_core.py index 4f85329dc..c8c557f42 100644 --- a/tests/unit/test_core.py +++ b/tests/unit/test_core.py @@ -124,6 +124,7 @@ def test__get_local_modules( json_output="", package_module_name_map={}, optional_dependencies_dev_groups=(), + non_dev_dependency_groups=(), using_default_requirements_files=True, experimental_namespace_package=experimental_namespace_package, github_output=False,