Fetching any submodule in a subproject with submodules

Fixes #1013

Author: spoorcc

CHANGELOG.rst

@@ -1,8 +1,15 @@
+Release 0.13.0 (unreleased)
+====================================
+
+* Rename child-manifests to sub-manifests in documentation and code (#1027)
+* Fetch git submodules in git subproject at pinned revision (#1013)
+* Add nested projects in subprojects to project report (#1017)
+* Make `dfetch report` output more yaml-like (#1017)
+
 Release 0.12.1 (released 2026-02-24)
 ====================================
 
 * Fix missing unicode data in standalone binaries (#1014)
-* Rename child-manifests to sub-manifests in documentation and code (#1027)
 
 Release 0.12.0 (released 2026-02-21)
 ====================================

dfetch/log.py

@@ -62,8 +62,9 @@ def print_info_line(self, name: str, info: str) -> None:
             self.info(f"  [bold][bright_green]{name}:[/bright_green][/bold]")
             DLogger._printed_projects.add(name)
 
-        line = info.replace("\n", "\n    ")
-        self.info(f"  [bold blue]> {line}[/bold blue]")
+        if info:
+            line = info.replace("\n", "\n    ")
+            self.info(f"  [bold blue]> {line}[/bold blue]")
 
     def print_warning_line(self, name: str, info: str) -> None:
         """Print a warning line: green name, yellow value."""

dfetch/project/gitsubproject.py

@@ -7,8 +7,8 @@
 from dfetch.log import get_logger
 from dfetch.manifest.project import ProjectEntry
 from dfetch.manifest.version import Version
-from dfetch.project.subproject import SubProject
-from dfetch.util.util import safe_rmtree
+from dfetch.project.subproject import SubProject, VcsDependency
+from dfetch.util.util import safe_rm, safe_rmtree
 from dfetch.vcs.git import GitLocalRepo, GitRemote, get_git_version
 
 logger = get_logger(__name__)
@@ -57,7 +57,7 @@ def list_tool_info() -> None:
             )
             SubProject._log_tool("git", "<not found in PATH>")
 
-    def _fetch_impl(self, version: Version) -> Version:
+    def _fetch_impl(self, version: Version) -> tuple[Version, list[VcsDependency]]:
         """Get the revision of the remote and place it at the local path."""
         rev_or_branch_or_tag = self._determine_what_to_fetch(version)
 
@@ -69,17 +69,35 @@ def _fetch_impl(self, version: Version) -> Version:
         ]
 
         local_repo = GitLocalRepo(self.local_path)
-        fetched_sha = local_repo.checkout_version(
+        fetched_sha, submodules = local_repo.checkout_version(
             remote=self.remote,
             version=rev_or_branch_or_tag,
             src=self.source,
-            must_keeps=license_globs,
+            must_keeps=license_globs + [".gitmodules"],
             ignore=self.ignore,
         )
 
+        vcs_deps = []
+        for submodule in submodules:
+            self._log_project(
+                f'Found & fetched submodule "./{submodule.path}" '
+                f" ({submodule.url} @ {Version(tag=submodule.tag, branch=submodule.branch, revision=submodule.sha)})",
+            )
+            vcs_deps.append(
+                VcsDependency(
+                    remote_url=submodule.url,
+                    destination=submodule.path,
+                    branch=submodule.branch,
+                    tag=submodule.tag,
+                    revision=submodule.sha,
+                    source_type="git-submodule",
+                )
+            )
+
         safe_rmtree(os.path.join(self.local_path, local_repo.METADATA_DIR))
+        safe_rm(os.path.join(self.local_path, local_repo.GIT_MODULES_FILE))
 
-        return self._determine_fetched_version(version, fetched_sha)
+        return self._determine_fetched_version(version, fetched_sha), vcs_deps
 
     def _determine_what_to_fetch(self, version: Version) -> str:
         """Based on asked version, target to fetch."""

dfetch/project/metadata.py

@@ -16,6 +16,17 @@
 """
 
 
+class Dependency(TypedDict):
+    """Argument types for dependency class construction."""
+
+    branch: str
+    tag: str
+    revision: str
+    remote_url: str
+    destination: str
+    source_type: str
+
+
 class Options(TypedDict):  # pylint: disable=too-many-ancestors
     """Argument types for Metadata class construction."""
 
@@ -27,6 +38,7 @@ class Options(TypedDict):  # pylint: disable=too-many-ancestors
     destination: str
     hash: str
     patch: str | list[str]
+    dependencies: list["Dependency"]
 
 
 class Metadata:
@@ -54,6 +66,8 @@ def __init__(self, kwargs: Options) -> None:
         # Historically only a single patch was allowed
         self._patch: list[str] = always_str_list(kwargs.get("patch", []))
 
+        self._dependencies: list[Dependency] = kwargs.get("dependencies", [])
+
     @classmethod
     def from_project_entry(cls, project: ProjectEntry) -> "Metadata":
         """Create a metadata object from a project entry."""
@@ -66,6 +80,7 @@ def from_project_entry(cls, project: ProjectEntry) -> "Metadata":
             "last_fetch": datetime.datetime(2000, 1, 1, 0, 0, 0),
             "hash": "",
             "patch": project.patch,
+            "dependencies": [],
         }
         return cls(data)
 
@@ -77,13 +92,18 @@ def from_file(cls, path: str) -> "Metadata":
             return cls(data)
 
     def fetched(
-        self, version: Version, hash_: str = "", patch_: list[str] | None = None
+        self,
+        version: Version,
+        hash_: str = "",
+        patch_: list[str] | None = None,
+        dependencies: list[Dependency] | None = None,
     ) -> None:
         """Update metadata."""
         self._last_fetch = datetime.datetime.now()
         self._version = version
         self._hash = hash_
         self._patch = patch_ or []
+        self._dependencies = dependencies or []
 
     @property
     def version(self) -> Version:
@@ -129,6 +149,11 @@ def patch(self) -> list[str]:
         """The list of applied patches as stored in the metadata."""
         return self._patch
 
+    @property
+    def dependencies(self) -> list[Dependency]:
+        """The list of dependency projects as stored in the metadata."""
+        return self._dependencies
+
     @property
     def path(self) -> str:
         """Path to metadata file."""
@@ -152,12 +177,13 @@ def __eq__(self, other: object) -> bool:
                 other._version.revision == self._version.revision,
                 other.hash == self.hash,
                 other.patch == self.patch,
+                other.dependencies == self.dependencies,
             ]
         )
 
     def dump(self) -> None:
         """Dump metadata file to correct path."""
-        metadata = {
+        metadata: dict[str, dict[str, str | list[str] | list[Dependency]]] = {
             "dfetch": {
                 "remote_url": self.remote_url,
                 "branch": self._version.branch,
@@ -169,6 +195,9 @@ def dump(self) -> None:
             }
         }
 
+        if self.dependencies:
+            metadata["dfetch"]["dependencies"] = self.dependencies
+
         with open(self.path, "w+", encoding="utf-8") as metadata_file:
             metadata_file.write(DONT_EDIT_WARNING)
             yaml.dump(metadata, metadata_file)

dfetch/project/subproject.py

@@ -5,19 +5,42 @@
 import pathlib
 from abc import ABC, abstractmethod
 from collections.abc import Sequence
+from typing import NamedTuple
 
 from dfetch.log import get_logger
 from dfetch.manifest.project import ProjectEntry
 from dfetch.manifest.version import Version
 from dfetch.project.abstract_check_reporter import AbstractCheckReporter
-from dfetch.project.metadata import Metadata
+from dfetch.project.metadata import Dependency, Metadata
 from dfetch.util.util import hash_directory, safe_rm
 from dfetch.util.versions import latest_tag_from_list
 from dfetch.vcs.patch import Patch
 
 logger = get_logger(__name__)
 
 
+class VcsDependency(NamedTuple):
+    """Information about a vcs dependency."""
+
+    destination: str
+    remote_url: str
+    branch: str
+    tag: str
+    revision: str
+    source_type: str
+
+    def to_dependency(self) -> Dependency:
+        """Convert this vcs dependency to a Dependency object."""
+        return Dependency(
+            destination=self.destination,
+            remote_url=self.remote_url,
+            branch=self.branch,
+            tag=self.tag,
+            revision=self.revision,
+            source_type=self.source_type,
+        )
+
+
 class SubProject(ABC):
     """Abstract SubProject object.
 
@@ -125,7 +148,7 @@ def update(
             f"Fetching {to_fetch}",
             enabled=self._show_animations,
         ):
-            actually_fetched = self._fetch_impl(to_fetch)
+            actually_fetched, dependency = self._fetch_impl(to_fetch)
         self._log_project(f"Fetched {actually_fetched}")
 
         applied_patches = self._apply_patches(patch_count)
@@ -134,6 +157,7 @@ def update(
             actually_fetched,
             hash_=hash_directory(self.local_path, skiplist=[self.__metadata.FILENAME]),
             patch_=applied_patches,
+            dependencies=[dependency.to_dependency() for dependency in dependency],
         )
 
         logger.debug(f"Writing repo metadata to: {self.__metadata.path}")
@@ -381,7 +405,7 @@ def _are_there_local_changes(self, files_to_ignore: Sequence[str]) -> bool:
         )
 
     @abstractmethod
-    def _fetch_impl(self, version: Version) -> Version:
+    def _fetch_impl(self, version: Version) -> tuple[Version, list[VcsDependency]]:
         """Fetch the given version of the subproject, should be implemented by the child class."""
 
     @abstractmethod

dfetch/project/svnsubproject.py

@@ -7,7 +7,7 @@
 from dfetch.log import get_logger
 from dfetch.manifest.project import ProjectEntry
 from dfetch.manifest.version import Version
-from dfetch.project.subproject import SubProject
+from dfetch.project.subproject import SubProject, VcsDependency
 from dfetch.util.util import (
     find_matching_files,
     find_non_matching_files,
@@ -106,7 +106,7 @@ def _remove_ignored_files(self) -> None:
             if not (file_or_dir.is_file() and self.is_license_file(file_or_dir.name)):
                 safe_rm(file_or_dir)
 
-    def _fetch_impl(self, version: Version) -> Version:
+    def _fetch_impl(self, version: Version) -> tuple[Version, list[VcsDependency]]:
         """Get the revision of the remote and place it at the local path."""
         branch, branch_path, revision = self._determine_what_to_fetch(version)
         rev_arg = f"--revision {revision}" if revision else ""
@@ -147,7 +147,7 @@ def _fetch_impl(self, version: Version) -> Version:
         if self.ignore:
             self._remove_ignored_files()
 
-        return Version(tag=version.tag, branch=branch, revision=revision)
+        return Version(tag=version.tag, branch=branch, revision=revision), []
 
     @staticmethod
     def _parse_file_pattern(complete_path: str) -> tuple[str, str]:

dfetch/reporting/stdout_reporter.py

@@ -26,22 +26,36 @@ def add_project(
     ) -> None:
         """Add a project to the report."""
         del version
-        logger.print_info_field("project", project.name)
-        logger.print_info_field("    remote", project.remote)
+        logger.print_info_line(project.name, "")
+        logger.print_info_field("- remote", project.remote)
         try:
             metadata = Metadata.from_file(Metadata.from_project_entry(project).path)
-            logger.print_info_field("    remote url", metadata.remote_url)
-            logger.print_info_field("    branch", metadata.branch)
-            logger.print_info_field("    tag", metadata.tag)
-            logger.print_info_field("    last fetch", str(metadata.last_fetch))
-            logger.print_info_field("    revision", metadata.revision)
-            logger.print_info_field("    patch", ", ".join(metadata.patch))
+            logger.print_info_field("  remote url", metadata.remote_url)
+            logger.print_info_field("  branch", metadata.branch)
+            logger.print_info_field("  tag", metadata.tag)
+            logger.print_info_field("  last fetch", str(metadata.last_fetch))
+            logger.print_info_field("  revision", metadata.revision)
+            logger.print_info_field("  patch", ", ".join(metadata.patch))
             logger.print_info_field(
-                "    licenses", ",".join(license.name for license in licenses)
+                "  licenses", ",".join(license.name for license in licenses)
             )
 
+            if metadata.dependencies:
+                logger.info("")
+                logger.print_report_line("  dependencies", "")
+            for dependency in metadata.dependencies:
+                logger.print_info_field("  - path", dependency.get("destination", ""))
+                logger.print_info_field("    url", dependency.get("remote_url", ""))
+                logger.print_info_field("    branch", dependency.get("branch", ""))
+                logger.print_info_field("    tag", dependency.get("tag", ""))
+                logger.print_info_field("    revision", dependency.get("revision", ""))
+                logger.print_info_field(
+                    "    source-type", dependency.get("source_type", "")
+                )
+                logger.info("")
+
         except FileNotFoundError:
-            logger.print_info_field("    last fetch", "never")
+            logger.print_info_field("  last fetch", "never")
 
     def dump_to_file(self, outfile: str) -> bool:
         """Do nothing."""

dfetch/util/util.py

@@ -43,11 +43,12 @@ def find_matching_files(directory: str, patterns: Sequence[str]) -> Iterator[Pat
 
 
 def safe_rm(path: str | Path) -> None:
-    """Delete an file or directory safely."""
-    if os.path.isdir(path):
-        safe_rmtree(str(path))
-    else:
-        os.remove(path)
+    """Delete a file or directory safely."""
+    if os.path.lexists(path):
+        if os.path.isdir(path):
+            safe_rmtree(str(path))
+        else:
+            os.remove(path)
 
 
 def safe_rmtree(path: str) -> None: