diff --git a/src/kapla/__init__.py b/src/kapla/__init__.py index d9f2629..2d93b0c 100644 --- a/src/kapla/__init__.py +++ b/src/kapla/__init__.py @@ -1 +1 @@ -__version__ = "0.36.0" +__version__ = "0.37.0" diff --git a/src/kapla/cli/build.py b/src/kapla/cli/build.py index c3e9e18..534962f 100644 --- a/src/kapla/cli/build.py +++ b/src/kapla/cli/build.py @@ -30,6 +30,14 @@ def set_build_parser(parser: _SubParsersAction[Any], parent: ArgumentParser) -> build_parser.add_argument( "-l", "--lock", action="store_true", default=False, dest="lock_versions" ) + build_parser.add_argument( + "--process-secondary", + action="store_true", + default=False, + dest="process_secondary_dependencies", + help="If true, all the dependencies of the dependencies will be added to the toml as well. " + "Useful to enforce a global lock to sub-projects in a mono repo .", + ) def do_build(args: Any) -> None: @@ -39,6 +47,7 @@ def do_build(args: Any) -> None: include_projects: Optional[Tuple[str]] = args.projects or None exclude_projects: Optional[Tuple[str]] = args.exclude_projects or None lock_versions: bool = args.lock_versions + process_secondary_dependencies: bool = args.process_secondary_dependencies clean: bool = not args.no_clean # Find repo @@ -51,6 +60,7 @@ def do_build(args: Any) -> None: exclude_projects=list(exclude_projects) if exclude_projects else [], lock_versions=lock_versions, clean=clean, + process_secondary_dependencies=process_secondary_dependencies, ) # Run build diff --git a/src/kapla/projects/kproject.py b/src/kapla/projects/kproject.py index 89649da..bd9ce0e 100644 --- a/src/kapla/projects/kproject.py +++ b/src/kapla/projects/kproject.py @@ -129,23 +129,29 @@ def is_already_installed(self) -> bool: return True return False + def _extract_dep_names( + self, dep: Union[str, Dict[str, DependencyMeta]] + ) -> List[str]: + """Extract dependency names from a dependency specification""" + return [dep] if isinstance(dep, str) else list(dep.keys()) + + def _collect_deps_from_list( + self, deps_list: List[Union[str, Dict[str, DependencyMeta]]] + ) -> Set[str]: + """Collect dependency names from a list of dependencies""" + names: Set[str] = set() + for dep in deps_list: + names.update(self._extract_dep_names(dep)) + return names + def get_dependencies_names(self, include_extras: bool = True) -> List[str]: """Get a list of dependencies names""" - names: Set[str] = set() - for dep in self.spec.dependencies: - if isinstance(dep, str): - names.add(dep) - else: - for name in dep: - names.add(name) + names = self._collect_deps_from_list(self.spec.dependencies) + if include_extras: for extra_deps in self.spec.extras.values(): - for dep in extra_deps: - if isinstance(dep, str): - names.add(dep) - else: - for name in dep: - names.add(name) + names.update(self._collect_deps_from_list(extra_deps)) + return list(names) def get_local_dependencies_names(self) -> List[str]: @@ -165,7 +171,7 @@ def get_local_dependencies(self) -> Dict[str, Dependency]: if name in self.repo.projects } _need_to_inspect = set(local_projects) - _inspected: Set[str] = set([self.name]) + _inspected: Set[str] = {self.name} # For each local dependency while _need_to_inspect: local_dep = local_projects[_need_to_inspect.pop()] @@ -186,6 +192,7 @@ def get_build_dependencies( include_local: bool = True, include_python: bool = True, lock_versions: bool = True, + process_secondary_dependencies: bool = False, ) -> Tuple[Dict[str, Dependency], Dict[str, List[str]], Dict[str, Group]]: """Return dependencies, extras and groups""" dependencies: Dict[str, Dependency] = {} @@ -198,9 +205,16 @@ def get_build_dependencies( else: constraints = self.repo.get_packages_constraints() - self._process_main_dependencies(dependencies, constraints, lock_versions) + self._process_main_dependencies( + dependencies, constraints, lock_versions, process_secondary_dependencies + ) self._process_extras_and_groups( - dependencies, extras, groups, constraints, lock_versions + dependencies, + extras, + groups, + constraints, + lock_versions, + process_secondary_dependencies, ) self._handle_python_dependency(dependencies, include_python) self._remove_local_dependencies(dependencies, include_local) @@ -211,21 +225,17 @@ def _process_main_dependencies( dependencies: Dict[str, Dependency], constraints: Union[DefaultDict[str, str], Dict[str, str]], lock_versions: bool, + process_secondary_dependencies: bool, ) -> None: for dep in self.spec.dependencies: self._lock_and_store_dependencies( lock_versions, dependencies, constraints, dep ) - local_dependencies = self._get_local_dependency_names(dep) - self._process_secondary_dependencies( - local_dependencies, dependencies, constraints, lock_versions - ) - - def _get_local_dependency_names( - self, dep: Union[str, Dict[str, DependencyMeta]] - ) -> List[str]: - """Extract dependency names from a dependency specification.""" - return [dep] if isinstance(dep, str) else list(dep.keys()) + local_dependencies = self._extract_dep_names(dep) + if process_secondary_dependencies: + self._process_secondary_dependencies( + local_dependencies, dependencies, constraints, lock_versions + ) def _process_secondary_dependencies( self, @@ -310,6 +320,7 @@ def _process_extras_and_groups( groups: Dict[str, Group], constraints: Union[DefaultDict[str, str], Dict[str, str]], lock_versions: bool, + process_secondary_dependencies: bool, ) -> None: for group_name, group_dependencies in self.spec.extras.items(): groups[group_name] = Group(dependencies={}) @@ -330,6 +341,7 @@ def _process_extras_and_groups( groups, constraints, lock_versions, + process_secondary_dependencies, visited, value, ) @@ -343,6 +355,7 @@ def _add_dependency_to_group( groups: Dict[str, Group], constraints: Union[DefaultDict[str, str], Dict[str, str]], lock_versions: bool, + process_secondary_dependencies: bool, visited: Optional[Set[str]] = None, value: Optional[DependencyMeta] = None, ) -> None: @@ -380,6 +393,8 @@ def _add_dependency_to_group( else {"version": locked_version, "optional": True} ) dependencies[dep_name] = Dependency.parse_obj(dep_dict) + if not process_secondary_dependencies: + return if self.repo is None: return @@ -400,6 +415,7 @@ def _add_dependency_to_group( groups, constraints, lock_versions, + process_secondary_dependencies, visited, self._create_dependency_meta(sub_dep_meta_or_version), ) @@ -463,10 +479,12 @@ def get_pyproject_spec( self, lock_versions: bool = True, build_system: BuildSystem = DEFAULT_BUILD_SYSTEM, + process_secondary_dependencies: bool = False, ) -> PyProjectSpec: """Create content of pyproject.toml file according to project.yaml""" dependencies, extras, groups = self.get_build_dependencies( - lock_versions=lock_versions + lock_versions=lock_versions, + process_secondary_dependencies=process_secondary_dependencies, ) # Gather raw tool.poetry configuration byt exclude dependencies, extras and group fields raw_poetry_config = self.spec.dict( @@ -491,13 +509,16 @@ def write_pyproject( path: Union[str, Path, None] = None, lock_versions: bool = True, build_system: BuildSystem = DEFAULT_BUILD_SYSTEM, + process_secondary_dependencies: bool = False, ) -> KPyProject: """Write auto-generated pyproject.toml file. If path argument is not specified, file is generated in the project directory by default. """ spec = self.get_pyproject_spec( - lock_versions=lock_versions, build_system=build_system + lock_versions=lock_versions, + build_system=build_system, + process_secondary_dependencies=process_secondary_dependencies, ) pyproject_path = Path(path) if path else self.pyproject_path content = spec.dict() @@ -541,10 +562,14 @@ def temporary_pyproject( lock_versions: bool = True, build_system: BuildSystem = DEFAULT_BUILD_SYSTEM, clean: bool = True, + process_secondary_dependencies: bool = False, ) -> Iterator[KPyProject]: """A context manager which ensures pyproject.toml is written to disk within context and removed out of context""" pyproject = self.write_pyproject( - path, lock_versions=lock_versions, build_system=build_system + path, + lock_versions=lock_versions, + build_system=build_system, + process_secondary_dependencies=process_secondary_dependencies, ) try: yield pyproject @@ -564,6 +589,7 @@ async def build( timeout: Optional[float] = None, deadline: Optional[float] = None, recurse: bool = True, + process_secondary_dependencies: bool = False, **kwargs: Any, ) -> Command: if recurse and self.repo: @@ -577,6 +603,7 @@ async def build( build_system=build_system, lock_versions=lock_versions, recurse=False, + process_secondary_dependencies=process_secondary_dependencies, ) ) if clear_dist: @@ -586,6 +613,7 @@ async def build( lock_versions=lock_versions, build_system=build_system, clean=clean, + process_secondary_dependencies=process_secondary_dependencies, ) as pyproject: return await pyproject.poetry_build( env=env, @@ -871,6 +899,7 @@ async def build_docker( build_dist_system, lock_versions, deadline, + True, **kwargs, ) logger.info("Invoking docker command", command=cmd.cmd) @@ -1029,6 +1058,7 @@ async def _build_dist( build_dist_system: BuildSystem, lock_versions: bool, deadline: Optional[float], + process_secondary_dependencies: bool, **kwargs: Any, ) -> None: await self.build( @@ -1037,6 +1067,7 @@ async def _build_dist( lock_versions=lock_versions, quiet=True, deadline=deadline, + process_secondary_dependencies=process_secondary_dependencies, **kwargs, ) dist_root = self.root / "dist" diff --git a/src/kapla/projects/krepo.py b/src/kapla/projects/krepo.py index 08d928a..7ea272c 100644 --- a/src/kapla/projects/krepo.py +++ b/src/kapla/projects/krepo.py @@ -333,7 +333,7 @@ def get_projects_dependencies_missing( zombie_deps: Dict[str, Dict[str, None]] = defaultdict(dict) for project_name in self.projects: - deps_summary = self.get_single_project_dependencies(project_name) + deps_summary = self.get_single_project_dependencies(project_name, False) # Iterate over groups for group_name, group_deps in deps_summary.groups.items(): @@ -448,7 +448,7 @@ async def add_missing_dependencies(self) -> None: await self.poetry_add( *packages, group=group, - lock=True, + lock=False, ) async def remove_zombie_dependencies(self) -> None: @@ -591,6 +591,7 @@ async def build_projects( timeout: Optional[float] = None, deadline: Optional[float] = None, clean: bool = True, + process_secondary_dependencies: bool = False, ) -> List[Command]: # Compute deadline to use to enforce timeouts deadline = get_deadline(timeout, deadline) @@ -617,6 +618,7 @@ async def build_project(project: KProject) -> None: raise_on_error=True, clean=clean, recurse=False, + process_secondary_dependencies=process_secondary_dependencies, ) results.append(cmd) wheels = list(Path(project.root / "dist").glob("*.whl")) diff --git a/src/kapla/specs/pyproject.py b/src/kapla/specs/pyproject.py index ca92c14..1cf163c 100644 --- a/src/kapla/specs/pyproject.py +++ b/src/kapla/specs/pyproject.py @@ -93,7 +93,7 @@ class Config(AliasedModel.Config): DEFAULT_BUILD_SYSTEM = BuildSystem( - build_backend="poetry.core.masonry.api", # pyright: ignore[reportGeneralTypeIssues] + build_backend="poetry.core.masonry.api", # pyright: ignore[reportCallIssue] requires=[ "poetry-core>=1.2.0", ],