From a62becede7eb3aa601301b39395dc40e2d891ead Mon Sep 17 00:00:00 2001 From: Zgardan Date: Tue, 29 Apr 2025 16:48:55 +0200 Subject: [PATCH 1/5] WIP --- .gitignore | 5 ++- src/kapla/projects/kproject.py | 73 +++++++++++++++++++++++++--------- src/kapla/specs/common.py | 2 +- src/kapla/specs/kproject.py | 6 +-- src/kapla/specs/pyproject.py | 4 +- 5 files changed, 64 insertions(+), 26 deletions(-) diff --git a/.gitignore b/.gitignore index cb866a2..918e42a 100644 --- a/.gitignore +++ b/.gitignore @@ -81,7 +81,6 @@ docs/_build/ .pybuilder/ target/ - # IPython profile_default/ ipython_config.py @@ -153,4 +152,6 @@ cython_debug/ # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. -#.idea/ \ No newline at end of file +#.idea/ + +WIP/ diff --git a/src/kapla/projects/kproject.py b/src/kapla/projects/kproject.py index 0e9da3a..91a6b68 100644 --- a/src/kapla/projects/kproject.py +++ b/src/kapla/projects/kproject.py @@ -8,6 +8,7 @@ from typing import ( TYPE_CHECKING, Any, + DefaultDict, Dict, Iterable, Iterator, @@ -27,6 +28,7 @@ from kapla.specs.pyproject import ( DEFAULT_BUILD_SYSTEM, Dependency, + DependencyMeta, Group, PoetryConfig, PyProjectSpec, @@ -195,26 +197,34 @@ def get_build_dependencies( constraints = defaultdict(lambda: "*") else: constraints = self.repo.get_packages_constraints() + + def _process_dep_recursive(dep_name: str, visited: Set[str]) -> None: + if dep_name in visited: + return + visited.add(dep_name) + self._lock_and_store_dependencies( + lock_versions, dependencies, constraints, dep_name + ) + if self.repo: + locked_package = self.repo.packages_lock.packages.get(dep_name) + if locked_package and locked_package.dependencies: + for sub_dep in locked_package.dependencies: + _process_dep_recursive(sub_dep, visited) + # Iterate over dependencies and replace version for dep in self.spec.dependencies: - if isinstance(dep, str): - if lock_versions: - locked_version = self.get_locked_version(dep) - else: - locked_version = constraints.get(dep, "*") - dependencies[dep] = Dependency(version=locked_version) - else: - for key, value in dep.items(): - if lock_versions: - locked_version = self.get_locked_version(key) - else: - locked_version = constraints.get(key, "*") - dependencies[key] = Dependency.parse_obj( - { - **value.dict(exclude_unset=True, by_alias=True), - "version": locked_version, - } - ) + self._lock_and_store_dependencies( + lock_versions, dependencies, constraints, dep + ) + local_dependencies = [dep] if isinstance(dep, str) else list(dep.keys()) + if self.repo: + for dep_ in local_dependencies: + locked_package = self.repo.packages_lock.packages.get(dep_) + if locked_package and locked_package.dependencies: + visited: Set[str] = set() + for secondary_dep in locked_package.dependencies: + _process_dep_recursive(secondary_dep, visited) + # Iterate over extra dependencies and replace version for group_name, group_dependencies in self.spec.extras.items(): # Let's create a group and an extra @@ -283,6 +293,33 @@ def get_build_dependencies( # Return values return dependencies, extras, groups + def _lock_and_store_dependencies( + self, + lock_versions: bool, + dependencies: Dict[str, Dependency], + constraints: Union[DefaultDict[str, str], Dict[str, str]], + dep: Union[str, Dict[str, DependencyMeta]], + ) -> None: + if isinstance(dep, str): + if lock_versions: + locked_version = self.get_locked_version(dep) + else: + locked_version = constraints.get(dep, "*") + + dependencies[dep] = Dependency(version=locked_version) + else: + for key, value in dep.items(): + if lock_versions: + locked_version = self.get_locked_version(key) + else: + locked_version = constraints.get(key, "*") + dependencies[key] = Dependency.parse_obj( + { + **value.dict(exclude_unset=True, by_alias=True), + "version": locked_version, + } + ) + def get_locked_version(self, package: str) -> str: if self.repo: return self.repo.get_locked_version(package) diff --git a/src/kapla/specs/common.py b/src/kapla/specs/common.py index 04ae418..dff9ae8 100644 --- a/src/kapla/specs/common.py +++ b/src/kapla/specs/common.py @@ -19,7 +19,7 @@ class BuildSystem(AliasedModel): class Package(AliasedModel): - """A package can be declared as a dictionnary. + """A package can be declared as a dictionary. References: * diff --git a/src/kapla/specs/kproject.py b/src/kapla/specs/kproject.py index c1f4507..75a3170 100644 --- a/src/kapla/specs/kproject.py +++ b/src/kapla/specs/kproject.py @@ -2,7 +2,7 @@ from .base import AliasedModel from .common import BasePythonConfig -from .pyproject import DepedencyMeta +from .pyproject import DependencyMeta class DockerImageSpec(AliasedModel): @@ -26,7 +26,7 @@ class DockerSpec(AliasedModel): class KProjectSpec(BasePythonConfig): # Docs: version: Optional[str] = None - dependencies: List[Union[str, Dict[str, DepedencyMeta]]] = [] + dependencies: List[Union[str, Dict[str, DependencyMeta]]] = [] docker: Optional[DockerSpec] = None # Docs: - extras: Dict[str, List[Union[str, Dict[str, DepedencyMeta]]]] = {} + extras: Dict[str, List[Union[str, Dict[str, DependencyMeta]]]] = {} diff --git a/src/kapla/specs/pyproject.py b/src/kapla/specs/pyproject.py index b8edf82..ca92c14 100644 --- a/src/kapla/specs/pyproject.py +++ b/src/kapla/specs/pyproject.py @@ -8,7 +8,7 @@ from .common import BasePythonConfig, BuildSystem -class DepedencyMeta(AliasedModel): +class DependencyMeta(AliasedModel): path: Optional[str] = None develop: Optional[bool] = None optional: Optional[bool] = None @@ -24,7 +24,7 @@ class Config(AliasedModel.Config): extra = "allow" -class Dependency(DepedencyMeta): +class Dependency(DependencyMeta): """A dependency found in a pyproject.toml file. Dependencies can be found in: From 1780bf5c800dfb98bbdd5242eb736e2cea604fc1 Mon Sep 17 00:00:00 2001 From: Zgardan Date: Tue, 29 Apr 2025 17:04:20 +0200 Subject: [PATCH 2/5] WIP --- src/kapla/projects/kproject.py | 254 +++++++++++++++++++++++---------- 1 file changed, 178 insertions(+), 76 deletions(-) diff --git a/src/kapla/projects/kproject.py b/src/kapla/projects/kproject.py index 91a6b68..a78ec75 100644 --- a/src/kapla/projects/kproject.py +++ b/src/kapla/projects/kproject.py @@ -191,107 +191,209 @@ def get_build_dependencies( dependencies: Dict[str, Dependency] = {} groups: Dict[str, Group] = {} extras: Dict[str, List[str]] = {} - # Fetch constraints + constraints: Dict[str, str] if self.repo is None: constraints = defaultdict(lambda: "*") else: constraints = self.repo.get_packages_constraints() - def _process_dep_recursive(dep_name: str, visited: Set[str]) -> None: - if dep_name in visited: - return - visited.add(dep_name) - self._lock_and_store_dependencies( - lock_versions, dependencies, constraints, dep_name - ) - if self.repo: - locked_package = self.repo.packages_lock.packages.get(dep_name) - if locked_package and locked_package.dependencies: - for sub_dep in locked_package.dependencies: - _process_dep_recursive(sub_dep, visited) + self._process_main_dependencies(dependencies, constraints, lock_versions) + self._process_extras_and_groups( + dependencies, extras, groups, constraints, lock_versions + ) + self._handle_python_dependency(dependencies, include_python) + self._remove_local_dependencies(dependencies, include_local) + return dependencies, extras, groups - # Iterate over dependencies and replace version + def _process_main_dependencies( + self, + dependencies: Dict[str, Dependency], + constraints: Union[DefaultDict[str, str], Dict[str, str]], + lock_versions: bool, + ) -> None: for dep in self.spec.dependencies: self._lock_and_store_dependencies( lock_versions, dependencies, constraints, dep ) - local_dependencies = [dep] if isinstance(dep, str) else list(dep.keys()) - if self.repo: - for dep_ in local_dependencies: - locked_package = self.repo.packages_lock.packages.get(dep_) - if locked_package and locked_package.dependencies: - visited: Set[str] = set() - for secondary_dep in locked_package.dependencies: - _process_dep_recursive(secondary_dep, visited) - - # Iterate over extra dependencies and replace version + local_dependencies = self._get_local_dependency_names(dep) + self._process_secondary_dependencies( + local_dependencies, dependencies, constraints, lock_versions + ) + + def _get_local_dependency_names( + self, dep: Union[str, Dict[str, DependencyMeta]] + ) -> List[str]: + """Extract dependency names from a dependency specification.""" + return [dep] if isinstance(dep, str) else list(dep.keys()) + + def _process_secondary_dependencies( + self, + local_dependencies: List[str], + dependencies: Dict[str, Dependency], + constraints: Union[DefaultDict[str, str], Dict[str, str]], + lock_versions: bool, + ) -> None: + """Process secondary dependencies of the given local dependencies.""" + if not self.repo: + return + + for dep_name in local_dependencies: + locked_package = self.repo.packages_lock.packages.get(dep_name) + if locked_package and locked_package.dependencies: + visited: Set[str] = set() + for secondary_dep in locked_package.dependencies: + self._process_dependency_tree( + secondary_dep, visited, dependencies, constraints, lock_versions + ) + + def _process_dependency_tree( + self, + dep_name: str, + visited: Set[str], + dependencies: Dict[str, Dependency], + constraints: Union[DefaultDict[str, str], Dict[str, str]], + lock_versions: bool, + ) -> None: + """Process a dependency and its subdependencies recursively.""" + if dep_name in visited: + return + visited.add(dep_name) + + self._lock_and_store_dependencies( + lock_versions, dependencies, constraints, dep_name + ) + + if not self.repo: + return + + locked_package = self.repo.packages_lock.packages.get(dep_name) + if locked_package and locked_package.dependencies: + for sub_dep in locked_package.dependencies: + self._process_dependency_tree( + sub_dep, visited, dependencies, constraints, lock_versions + ) + + def _process_extras_and_groups( + self, + dependencies: Dict[str, Dependency], + extras: Dict[str, List[str]], + groups: Dict[str, Group], + constraints: Union[DefaultDict[str, str], Dict[str, str]], + lock_versions: bool, + ) -> None: for group_name, group_dependencies in self.spec.extras.items(): - # Let's create a group and an extra groups[group_name] = Group(dependencies={}) extras[group_name] = [] - # Iterate over dependencies and replace version + visited: Set[str] = set() for dep in group_dependencies: if isinstance(dep, str): - if lock_versions: - locked_version = self.get_locked_version(dep) - else: - locked_version = constraints.get(dep, "*") - # Add dependency to group - groups[group_name].dependencies[dep] = Dependency( - version=locked_version + self._add_dependency_to_group( + dep, + group_name, + dependencies, + extras, + groups, + constraints, + lock_versions, + visited, ) - # Add dependency to extra - if dep not in extras[group_name]: - extras[group_name].append(dep) - # Add dependency to optional dependencies - if dep not in dependencies: - dependencies[dep] = Dependency.parse_obj( - { - "version": locked_version, - "optional": True, - } - ) else: for key, value in dep.items(): - if lock_versions: - locked_version = self.get_locked_version(key) - else: - locked_version = constraints.get(key, "*") - # Add dependency to group - groups[group_name].dependencies[key] = Dependency.parse_obj( - { - **value.dict(exclude_unset=True, by_alias=True), - "version": locked_version, - } + self._add_dependency_to_group( + key, + group_name, + dependencies, + extras, + groups, + constraints, + lock_versions, + visited, + value, ) - # Add dependency to extra - if key not in extras[group_name]: - extras[group_name].append(key) - # Add dependency to optional dependencies - if key not in dependencies: - dependencies[key] = Dependency.parse_obj( - { - **value.dict(exclude_unset=True, by_alias=True), - "version": locked_version, - "optional": True, - } - ) - # Make sure python dependency is set + + def _add_dependency_to_group( + self, + dep_name: str, + group_name: str, + dependencies: Dict[str, Dependency], + extras: Dict[str, List[str]], + groups: Dict[str, Group], + constraints: Union[DefaultDict[str, str], Dict[str, str]], + lock_versions: bool, + visited: Optional[Set[str]] = None, + value: Optional[DependencyMeta] = None, + ) -> None: + if visited is None: + visited = set() + if dep_name in visited: + return + visited.add(dep_name) + locked_version = ( + self.get_locked_version(dep_name) + if lock_versions + else constraints.get(dep_name, "*") + ) + dep_obj = ( + Dependency.parse_obj( + { + **value.dict(exclude_unset=True, by_alias=True), + "version": locked_version, + } + ) + if value + else Dependency(version=locked_version) + ) + groups[group_name].dependencies[dep_name] = dep_obj + if dep_name not in extras[group_name]: + extras[group_name].append(dep_name) + if dep_name not in dependencies: + dep_dict = ( + { + **value.dict(exclude_unset=True, by_alias=True), + "version": locked_version, + "optional": True, + } + if value + else {"version": locked_version, "optional": True} + ) + dependencies[dep_name] = Dependency.parse_obj(dep_dict) + if self.repo: + locked_package = self.repo.packages_lock.packages.get(dep_name) + if locked_package and locked_package.dependencies: + for sub_dep in locked_package.dependencies: + self._add_dependency_to_group( + sub_dep, + group_name, + dependencies, + extras, + groups, + constraints, + lock_versions, + visited, + ) + + def _handle_python_dependency( + self, + dependencies: Dict[str, Dependency], + include_python: bool, + ) -> None: if include_python: - if "python" not in dependencies: - if self.repo: - python_dep = self.repo.get_dependency("python") - if python_dep: - dependencies["python"] = python_dep.copy() + if "python" not in dependencies and self.repo: + python_dep = self.repo.get_dependency("python") + if python_dep: + dependencies["python"] = python_dep.copy() else: dependencies.pop("python", None) - # Remove local deps + + def _remove_local_dependencies( + self, + dependencies: Dict[str, Dependency], + include_local: bool, + ) -> None: if not include_local: for dep in self.get_local_dependencies_names(): - dependencies.pop(dep) - # Return values - return dependencies, extras, groups + dependencies.pop(dep, None) def _lock_and_store_dependencies( self, From b72b0fc5032170e7f254ab75b2783cf3b4aeb518 Mon Sep 17 00:00:00 2001 From: Zgardan Date: Tue, 29 Apr 2025 17:11:46 +0200 Subject: [PATCH 3/5] wip --- src/kapla/projects/krepo.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/kapla/projects/krepo.py b/src/kapla/projects/krepo.py index 45f135b..46ffeb4 100644 --- a/src/kapla/projects/krepo.py +++ b/src/kapla/projects/krepo.py @@ -154,7 +154,12 @@ def discover_projects( for workspace_directory in all_workspaces[name]: # Find files named "project.yml" or "project.yaml" starting from the workspace for filepath in find_files_using_gitignore( - ("project.yml", "project.yaml", "project_encrypted.yml", "project_encrypted.yaml"), + ( + "project.yml", + "project.yaml", + "project_encrypted.yml", + "project_encrypted.yaml", + ), root=workspace_directory, ): # Create a new instance of KProject @@ -513,7 +518,6 @@ async def install_editable_projects( # Define function to perform install async def install_project(project: KProject) -> None: - nonlocal results async with limiter: cmd = await project.install( exclude_groups=exclude_groups, @@ -529,6 +533,7 @@ async def install_project(project: KProject) -> None: clean=False, ) if cmd: + nonlocal results results.append(cmd) # Kick off install From 58c9cefd23aeb0e89505aeaaaee9cf7b663b4fd1 Mon Sep 17 00:00:00 2001 From: Zgardan Date: Tue, 29 Apr 2025 17:13:58 +0200 Subject: [PATCH 4/5] wip --- src/kapla/projects/krepo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/kapla/projects/krepo.py b/src/kapla/projects/krepo.py index 46ffeb4..9df5b5a 100644 --- a/src/kapla/projects/krepo.py +++ b/src/kapla/projects/krepo.py @@ -609,7 +609,6 @@ async def build_projects( ): # Define function to perform install async def build_project(project: KProject) -> None: - nonlocal results async with limiter: cmd = await project.build( env=env, @@ -620,6 +619,7 @@ async def build_project(project: KProject) -> None: clean=clean, recurse=False, ) + nonlocal results results.append(cmd) wheels = list(Path(project.root / "dist").glob("*.whl")) logger.info( From 30d6cf7fe6cd7e847229ab28a5cabd57cc87487c Mon Sep 17 00:00:00 2001 From: Zgardan Date: Tue, 29 Apr 2025 17:16:32 +0200 Subject: [PATCH 5/5] wip --- src/kapla/projects/krepo.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/kapla/projects/krepo.py b/src/kapla/projects/krepo.py index 9df5b5a..08d928a 100644 --- a/src/kapla/projects/krepo.py +++ b/src/kapla/projects/krepo.py @@ -533,7 +533,6 @@ async def install_project(project: KProject) -> None: clean=False, ) if cmd: - nonlocal results results.append(cmd) # Kick off install @@ -619,7 +618,6 @@ async def build_project(project: KProject) -> None: clean=clean, recurse=False, ) - nonlocal results results.append(cmd) wheels = list(Path(project.root / "dist").glob("*.whl")) logger.info(