Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 36 additions & 0 deletions docs/usage.md
Original file line number Diff line number Diff line change
Expand Up @@ -540,6 +540,42 @@ pep621_dev_dependency_groups = ["test", "docs"]
deptry . --pep621-dev-dependency-groups "test,docs"
```

#### Non-dev dependency groups

By default, _deptry_ considers that groups defined under `[dependency-groups]` contain development dependencies. Some
projects use dependency groups to define optional regular dependencies. To account for that, it is possible to specify a
list of groups that need to be treated as containing regular dependencies.

For example, consider a project with the following `pyproject.toml`:

```toml
[project]
...
dependencies = ["httpx"]

[dependency-groups]
server = ["uvicorn"]
telemetry = ["opentelemetry-sdk"]
```

By default, `uvicorn` and `opentelemetry-sdk` are extracted as development dependencies. By specifying
`--non-dev-dependency-groups=server,telemetry`, `uvicorn` and `opentelemetry-sdk` will be treated as regular
dependencies instead.

- Type: `list[str]`
- Default: `[]`
- `pyproject.toml` option name: `non_dev_dependency_groups`
- CLI option name: `--non-dev-dependency-groups` (short: `-nddg`)
- `pyproject.toml` example:
```toml
[tool.deptry]
non_dev_dependency_groups = ["server", "telemetry"]
```
- CLI example:
```shell
deptry . --non-dev-dependency-groups "server,telemetry"
```

#### Experimental namespace package

!!! warning
Expand Down
10 changes: 10 additions & 0 deletions python/deptry/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -262,6 +262,14 @@ def display_deptry_version(ctx: click.Context, _param: click.Parameter, value: b
default=(),
show_default=False,
)
@click.option(
"--non-dev-dependency-groups",
"-nddg",
type=COMMA_SEPARATED_TUPLE,
help="Specify which groups in [dependency-groups] should be considered as groups containing regular dependencies instead of development ones",
default=(),
show_default=False,
)
@click.option(
"--experimental-namespace-package",
is_flag=True,
Expand All @@ -287,6 +295,7 @@ def cli(
package_module_name_map: MutableMapping[str, tuple[str, ...]],
pep621_dev_dependency_groups: tuple[str, ...],
optional_dependencies_dev_groups: tuple[str, ...],
non_dev_dependency_groups: tuple[str, ...],
experimental_namespace_package: bool,
) -> None:
"""Find dependency issues in your Python project.
Expand Down Expand Up @@ -324,6 +333,7 @@ def cli(
github_warning_errors=github_warning_errors,
package_module_name_map=package_module_name_map,
optional_dependencies_dev_groups=pep621_dev_dependency_groups or optional_dependencies_dev_groups,
non_dev_dependency_groups=non_dev_dependency_groups,
experimental_namespace_package=experimental_namespace_package,
).run()

Expand Down
2 changes: 2 additions & 0 deletions python/deptry/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ class Core:
json_output: str
package_module_name_map: Mapping[str, tuple[str, ...]]
optional_dependencies_dev_groups: tuple[str, ...]
non_dev_dependency_groups: tuple[str, ...]
experimental_namespace_package: bool
github_output: bool
github_warning_errors: tuple[str, ...]
Expand All @@ -50,6 +51,7 @@ def run(self) -> None:
self.config,
self.package_module_name_map,
self.optional_dependencies_dev_groups,
self.non_dev_dependency_groups,
self.requirements_files,
self.using_default_requirements_files,
self.requirements_files_dev,
Expand Down
6 changes: 5 additions & 1 deletion python/deptry/dependency_getter/builder.py
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Shouldn't we pass the argument to the other DependencyGetter's as well? Now we only seem to pass it to PoetryDependencyGetter.

Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ class DependencyGetterBuilder:
config: Path
package_module_name_map: Mapping[str, tuple[str, ...]] = field(default_factory=dict)
optional_dependencies_dev_groups: tuple[str, ...] = ()
non_dev_dependency_groups: tuple[str, ...] = ()
requirements_files: tuple[str, ...] = ()
using_default_requirements_files: bool = True
requirements_files_dev: tuple[str, ...] = ()
Expand All @@ -46,7 +47,10 @@ def build(self) -> DependencyGetter:

if self._project_uses_poetry(pyproject_toml):
return PoetryDependencyGetter(
self.config, self.package_module_name_map, self.optional_dependencies_dev_groups
self.config,
self.package_module_name_map,
self.optional_dependencies_dev_groups,
self.non_dev_dependency_groups,
)

if self._project_uses_uv(pyproject_toml):
Expand Down
61 changes: 41 additions & 20 deletions python/deptry/dependency_getter/pep621/base.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
from __future__ import annotations

import itertools
import logging
from dataclasses import dataclass
from typing import TYPE_CHECKING
Expand Down Expand Up @@ -43,6 +42,7 @@ class PEP621DependencyGetter(DependencyGetter):
"""

optional_dependencies_dev_groups: tuple[str, ...] = ()
non_dev_dependency_groups: tuple[str, ...] = ()

def get(self) -> DependenciesExtract:
dependencies = self._get_dependencies()
Expand All @@ -52,9 +52,12 @@ def get(self) -> DependenciesExtract:
dev_dependencies_from_optional, remaining_optional_dependencies = (
self._split_development_dependencies_from_optional_dependencies(optional_dependencies)
)
dev_dependencies_from_dependency_groups, remaining_dependency_groups_dependencies = (
self._split_development_dependencies_from_dependency_groups(dependency_groups_dependencies)
)
return DependenciesExtract(
[*dependencies, *remaining_optional_dependencies],
self._get_dev_dependencies(dependency_groups_dependencies, dev_dependencies_from_optional),
[*dependencies, *remaining_optional_dependencies, *remaining_dependency_groups_dependencies],
self._get_dev_dependencies(dev_dependencies_from_optional, dev_dependencies_from_dependency_groups),
)

def _get_dependencies(self) -> list[Dependency]:
Expand Down Expand Up @@ -109,13 +112,10 @@ def _get_dependency_groups_dependencies(self) -> dict[str, list[Dependency]]:

def _get_dev_dependencies(
self,
dependency_groups_dependencies: dict[str, list[Dependency]],
dev_dependencies_from_optional: list[Dependency],
dev_dependencies_from_dependency_groups: list[Dependency],
) -> list[Dependency]:
return [
*itertools.chain(*dependency_groups_dependencies.values()),
*dev_dependencies_from_optional,
]
return [*dev_dependencies_from_optional, *dev_dependencies_from_dependency_groups]

@staticmethod
def _project_uses_setuptools(pyproject_toml: dict[str, Any]) -> bool:
Expand All @@ -139,16 +139,6 @@ def _project_uses_setuptools(pyproject_toml: dict[str, Any]) -> bool:
)
return False

def _check_for_invalid_group_names(self, optional_dependencies: dict[str, list[Dependency]]) -> None:
missing_groups = set(self.optional_dependencies_dev_groups) - set(optional_dependencies.keys())
if missing_groups:
logging.warning(
"Warning: Trying to extract the dependencies from the optional dependency groups %s as development dependencies, "
"but the following groups were not found: %s",
list(self.optional_dependencies_dev_groups),
list(missing_groups),
)

def _split_development_dependencies_from_optional_dependencies(
self, optional_dependencies: dict[str, list[Dependency]]
) -> tuple[list[Dependency], list[Dependency]]:
Expand All @@ -159,8 +149,15 @@ def _split_development_dependencies_from_optional_dependencies(
dev_dependencies: list[Dependency] = []
regular_dependencies: list[Dependency] = []

if self.optional_dependencies_dev_groups:
self._check_for_invalid_group_names(optional_dependencies)
if self.optional_dependencies_dev_groups and (
missing_groups := set(self.optional_dependencies_dev_groups) - set(optional_dependencies.keys())
):
logging.warning(
"Warning: Trying to extract the dependencies from the optional dependency groups %s as development dependencies, "
"but the following groups were not found: %s",
list(self.optional_dependencies_dev_groups),
list(missing_groups),
)

for group, dependencies in optional_dependencies.items():
if group in self.optional_dependencies_dev_groups:
Expand All @@ -170,6 +167,30 @@ def _split_development_dependencies_from_optional_dependencies(

return dev_dependencies, regular_dependencies

def _split_development_dependencies_from_dependency_groups(
self, dependency_groups: dict[str, list[Dependency]]
) -> tuple[list[Dependency], list[Dependency]]:
dev_dependencies: list[Dependency] = []
regular_dependencies: list[Dependency] = []

if self.non_dev_dependency_groups and (
missing_groups := set(self.non_dev_dependency_groups) - set(dependency_groups.keys())
):
logging.warning(
"Warning: Trying to extract the dependencies from the dependency groups %s as regular dependencies, "
"but the following groups were not found: %s",
list(self.non_dev_dependency_groups),
list(missing_groups),
)

for group, dependencies in dependency_groups.items():
if group in self.non_dev_dependency_groups:
regular_dependencies.extend(dependencies)
else:
dev_dependencies.extend(dependencies)

return dev_dependencies, regular_dependencies

def _extract_pep_508_dependencies(self, dependencies: list[str]) -> list[Dependency]:
"""
Given a list of dependency specifications (e.g. "django>2.1; os_name != 'nt'"), convert them to Dependency objects.
Expand Down
6 changes: 4 additions & 2 deletions python/deptry/dependency_getter/pep621/pdm.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@ class PDMDependencyGetter(PEP621DependencyGetter):

def _get_dev_dependencies(
self,
dependency_groups_dependencies: dict[str, list[Dependency]],
dev_dependencies_from_optional: list[Dependency],
dev_dependencies_from_dependency_groups: list[Dependency],
) -> list[Dependency]:
"""
Retrieve dev dependencies from pyproject.toml, which in PDM are specified as:
Expand All @@ -36,7 +36,9 @@ def _get_dev_dependencies(
"tox-pdm>=0.5",
]
"""
dev_dependencies = super()._get_dev_dependencies(dependency_groups_dependencies, dev_dependencies_from_optional)
dev_dependencies = super()._get_dev_dependencies(
dev_dependencies_from_optional, dev_dependencies_from_dependency_groups
)

pyproject_data = load_pyproject_toml(self.config)

Expand Down
6 changes: 4 additions & 2 deletions python/deptry/dependency_getter/pep621/poetry.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,15 +38,17 @@ def _get_dependencies(self) -> list[Dependency]:

def _get_dev_dependencies(
self,
dependency_groups_dependencies: dict[str, list[Dependency]],
dev_dependencies_from_optional: list[Dependency],
dev_dependencies_from_dependency_groups: list[Dependency],
) -> list[Dependency]:
"""
Poetry's development dependencies can be specified under either, or both:
- [tool.poetry.dev-dependencies]
- [tool.poetry.group.<group>.dependencies]
"""
dev_dependencies = super()._get_dev_dependencies(dependency_groups_dependencies, dev_dependencies_from_optional)
dev_dependencies = super()._get_dev_dependencies(
dev_dependencies_from_optional, dev_dependencies_from_dependency_groups
)

pyproject_data = load_pyproject_toml(self.config)
poetry_dev_dependencies: dict[str, str] = {}
Expand Down
6 changes: 4 additions & 2 deletions python/deptry/dependency_getter/pep621/uv.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@ class UvDependencyGetter(PEP621DependencyGetter):

def _get_dev_dependencies(
self,
dependency_groups_dependencies: dict[str, list[Dependency]],
dev_dependencies_from_optional: list[Dependency],
dev_dependencies_from_dependency_groups: list[Dependency],
) -> list[Dependency]:
"""
Retrieve dev dependencies from pyproject.toml, which in uv are specified as:
Expand All @@ -35,7 +35,9 @@ def _get_dev_dependencies(

Dev dependencies marked as such from optional dependencies are also added to the list of dev dependencies found.
"""
dev_dependencies = super()._get_dev_dependencies(dependency_groups_dependencies, dev_dependencies_from_optional)
dev_dependencies = super()._get_dev_dependencies(
dev_dependencies_from_optional, dev_dependencies_from_dependency_groups
)

pyproject_data = load_pyproject_toml(self.config)

Expand Down
87 changes: 77 additions & 10 deletions tests/unit/dependency_getter/test_pep_621.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ def test_dependency_getter(tmp_path: Path) -> None:
assert "dep" in dependencies[7].top_levels


def test_dependency_getter_with_dev_dependencies(tmp_path: Path) -> None:
def test_dependency_getter_optional_dependencies_dev_groups(tmp_path: Path) -> None:
fake_pyproject_toml = """[project]
name = "foo"
dependencies = ["qux"]
Expand Down Expand Up @@ -100,20 +100,20 @@ def test_dependency_getter_with_dev_dependencies(tmp_path: Path) -> None:
assert dependencies[1].name == "foobar"
assert "foobar" in dependencies[1].top_levels

assert dev_dependencies[0].name == "foo"
assert "foo" in dev_dependencies[0].top_levels
assert dev_dependencies[0].name == "barfoo"
assert "barfoo" in dev_dependencies[0].top_levels

assert dev_dependencies[1].name == "baz"
assert "baz" in dev_dependencies[1].top_levels
assert dev_dependencies[1].name == "foo"
assert "foo" in dev_dependencies[1].top_levels

assert dev_dependencies[2].name == "foobaz"
assert "foobaz" in dev_dependencies[2].top_levels
assert dev_dependencies[2].name == "baz"
assert "baz" in dev_dependencies[2].top_levels

assert dev_dependencies[3].name == "barfoo"
assert "barfoo" in dev_dependencies[3].top_levels
assert dev_dependencies[3].name == "foobaz"
assert "foobaz" in dev_dependencies[3].top_levels


def test_dependency_getter_with_incorrect_dev_group(tmp_path: Path, caplog: LogCaptureFixture) -> None:
def test_dependency_getter_incorrect_optional_dependencies_dev_group(tmp_path: Path, caplog: LogCaptureFixture) -> None:
fake_pyproject_toml = """[project]
name = "foo"
dependencies = ["qux"]
Expand Down Expand Up @@ -147,6 +147,73 @@ def test_dependency_getter_with_incorrect_dev_group(tmp_path: Path, caplog: LogC
assert "barfoo" in dependencies[2].top_levels


def test_dependency_getter_non_dev_dependency_groups(tmp_path: Path) -> None:
fake_pyproject_toml = """[project]
name = "foo"
dependencies = ["qux"]

[dependency-groups]
group1 = ["foobar"]
group2 = ["barfoo"]
"""

with run_within_dir(tmp_path):
with Path("pyproject.toml").open("w") as f:
f.write(fake_pyproject_toml)

getter = PEP621DependencyGetter(config=Path("pyproject.toml"), non_dev_dependency_groups=("group2",))
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

See comment above; now we don't seem to test if the field introduced by the PR works as expected in the DependencyGetterBuilder. Should we add/modify an existing functional test too?

dependencies = getter.get().dependencies
dev_dependencies = getter.get().dev_dependencies

assert len(dependencies) == 2
assert len(dev_dependencies) == 1

assert dependencies[0].name == "qux"
assert "qux" in dependencies[0].top_levels

assert dependencies[1].name == "barfoo"
assert "barfoo" in dependencies[1].top_levels

assert dev_dependencies[0].name == "foobar"
assert "foobar" in dev_dependencies[0].top_levels


def test_dependency_getter_incorrect_dependency_groups_non_dev_group(tmp_path: Path, caplog: LogCaptureFixture) -> None:
fake_pyproject_toml = """[project]
name = "foo"
dependencies = ["qux"]

[dependency-groups]
group1 = ["foobar"]
group2 = ["barfoo"]
"""

with run_within_dir(tmp_path), caplog.at_level(logging.INFO):
with Path("pyproject.toml").open("w") as f:
f.write(fake_pyproject_toml)

getter = PEP621DependencyGetter(config=Path("pyproject.toml"), non_dev_dependency_groups=("group3",))
dependencies = getter.get().dependencies
dev_dependencies = getter.get().dev_dependencies

assert (
"Trying to extract the dependencies from the dependency groups ['group3'] as regular dependencies, but the following groups were not found: ['group3']"
in caplog.text
)

assert len(dependencies) == 1
assert len(dev_dependencies) == 2

assert dependencies[0].name == "qux"
assert "qux" in dependencies[0].top_levels

assert dev_dependencies[0].name == "foobar"
assert "foobar" in dev_dependencies[0].top_levels

assert dev_dependencies[1].name == "barfoo"
assert "barfoo" in dev_dependencies[1].top_levels


def test_dependency_getter_empty_dependencies(tmp_path: Path) -> None:
fake_pyproject_toml = """[project]
name = "foo"
Expand Down
Loading