Detect if we have already seen the dependency before downloading sources

e2e/test_bootstrap_cache.sh

@@ -60,10 +60,6 @@ $OUTDIR/wheels-repo/downloads/setuptools-*.whl
 $OUTDIR/wheels-repo/downloads/pbr-*.whl
 $OUTDIR/wheels-repo/downloads/stevedore-*.whl
 
-$OUTDIR/sdists-repo/downloads/stevedore-*.tar.gz
-$OUTDIR/sdists-repo/downloads/setuptools-*.tar.gz
-$OUTDIR/sdists-repo/downloads/pbr-*.tar.gz
-
 $OUTDIR/work-dir/pbr-*/*-requirements.txt
 $OUTDIR/work-dir/setuptools-*/*-requirements.txt
 $OUTDIR/work-dir/stevedore-*/*-requirements.txt
@@ -102,10 +98,6 @@ $OUTDIR/wheels-repo/downloads/setuptools-*.whl
 $OUTDIR/wheels-repo/downloads/pbr-*.whl
 $OUTDIR/wheels-repo/downloads/stevedore-*.whl
 
-$OUTDIR/sdists-repo/downloads/stevedore-*.tar.gz
-$OUTDIR/sdists-repo/downloads/setuptools-*.tar.gz
-$OUTDIR/sdists-repo/downloads/pbr-*.tar.gz
-
 $OUTDIR/work-dir/build-order.json
 $OUTDIR/work-dir/constraints.txt
 "

src/fromager/bootstrapper.py

@@ -71,15 +71,15 @@ def bootstrap(self, req: Requirement, req_type: RequirementType) -> Version:
 
         pbi = self.ctx.package_build_info(req)
         if pbi.pre_built:
-            resolved_version, wheel_url, wheel_filename, unpack_dir = (
-                self._resolve_and_download_prebuilt(req, req_type)
+            wheel_url, resolved_version = self._resolve_prebuilt_with_history(
+                req=req,
+                req_type=req_type,
             )
-            # Remember that this is a prebuilt wheel, and where we got it.
             source_url = wheel_url
-            source_url_type = str(SourceType.PREBUILT)
         else:
-            resolved_version, source_url, source_filename, source_url_type = (
-                self._resolve_and_download_source(req, req_type)
+            source_url, resolved_version = self._resolve_source_with_history(
+                req=req,
+                req_type=req_type,
             )
 
         self._add_to_graph(req, req_type, resolved_version, source_url)
@@ -104,11 +104,28 @@ def bootstrap(self, req: Requirement, req_type: RequirementType) -> Version:
         # for cleanup
         build_env = None
         sdist_root_dir = None
-        if not pbi.pre_built:
+        if pbi.pre_built:
+            wheel_filename, unpack_dir = self._download_prebuilt(
+                req=req,
+                req_type=req_type,
+                resolved_version=resolved_version,
+                wheel_url=source_url,
+            )
+            # Remember that this is a prebuilt wheel, and where we got it.
+            source_url_type = str(SourceType.PREBUILT)
+        else:
             unpacked_cached_wheel, cached_wheel_filename = (
                 self._download_wheel_from_cache(req, resolved_version)
             )
+            source_url_type = sources.get_source_type(self.ctx, req)
+
             if not unpacked_cached_wheel:
+                source_filename = sources.download_source(
+                    ctx=self.ctx,
+                    req=req,
+                    version=resolved_version,
+                    download_url=source_url,
+                )
                 sdist_root_dir = sources.prepare_source(
                     ctx=self.ctx,
                     req=req,
@@ -263,34 +280,15 @@ def _handle_build_requirements(
             build_environment.maybe_install(self.ctx, dep, build_type, str(resolved))
             self.progressbar.update()
 
-    def _resolve_and_download_source(
-        self, req: Requirement, req_type: RequirementType
-    ) -> tuple[Version, str, pathlib.Path, str]:
-        source_url, resolved_version = self._resolve_source_with_history(
-            req=req,
-            req_type=req_type,
-        )
-
-        source_filename = sources.download_source(
-            ctx=self.ctx,
-            req=req,
-            version=resolved_version,
-            download_url=source_url,
-        )
-        source_url_type = sources.get_source_type(self.ctx, req)
-
-        return (resolved_version, source_url, source_filename, source_url_type)
-
-    def _resolve_and_download_prebuilt(
-        self, req: Requirement, req_type: RequirementType
-    ) -> tuple[Version, str, pathlib.Path, pathlib.Path]:
+    def _download_prebuilt(
+        self,
+        req: Requirement,
+        req_type: RequirementType,
+        resolved_version: Version,
+        wheel_url: str,
+    ) -> tuple[pathlib.Path, pathlib.Path]:
         logger.info(f"{req.name}: {req_type} requirement {req} uses a pre-built wheel")
 
-        wheel_url, resolved_version = self._resolve_prebuilt_with_history(
-            req=req,
-            req_type=req_type,
-        )
-
         wheel_filename = wheels.download_wheel(req, wheel_url, self.ctx.wheels_prebuilt)
 
         # Add the wheel to the mirror so it is available to anything
@@ -304,7 +302,7 @@ def _resolve_and_download_prebuilt(
             shutil.copy(wheel_filename, dest_name)
             server.update_wheel_mirror(self.ctx)
         unpack_dir = self._create_unpack_dir(req, resolved_version)
-        return (resolved_version, wheel_url, wheel_filename, unpack_dir)
+        return (wheel_filename, unpack_dir)
 
     def _download_wheel_from_cache(
         self, req: Requirement, resolved_version: Version

src/fromager/sources.py

@@ -173,8 +173,7 @@ def _download_source_check(
             raise zipfile.BadZipFile(f"Empty zip file encountered: {source_filename}")
     elif source_filename.suffix == ".tgz" or source_filename.suffix == ".gz":
         with tarfile.open(source_filename) as tar:
-            contents = tar.getnames()
-            if not contents:
+            if not tar.next():
                 raise TypeError(f"Empty tar file encountered: {source_filename}")
     else:
         raise TypeError(