From 724e89b65cd53ec82aaf9202645eaf26e6c82904 Mon Sep 17 00:00:00 2001 From: Shubh Bapna Date: Thu, 21 Nov 2024 12:39:46 -0500 Subject: [PATCH 1/2] resolve and then detect if we have already seen the dependency before downloading the source Signed-off-by: Shubh Bapna --- e2e/test_bootstrap_cache.sh | 8 ----- src/fromager/bootstrapper.py | 66 +++++++++++++++++------------------- 2 files changed, 32 insertions(+), 42 deletions(-) diff --git a/e2e/test_bootstrap_cache.sh b/e2e/test_bootstrap_cache.sh index 1b57e6a7..92cc1761 100755 --- a/e2e/test_bootstrap_cache.sh +++ b/e2e/test_bootstrap_cache.sh @@ -60,10 +60,6 @@ $OUTDIR/wheels-repo/downloads/setuptools-*.whl $OUTDIR/wheels-repo/downloads/pbr-*.whl $OUTDIR/wheels-repo/downloads/stevedore-*.whl -$OUTDIR/sdists-repo/downloads/stevedore-*.tar.gz -$OUTDIR/sdists-repo/downloads/setuptools-*.tar.gz -$OUTDIR/sdists-repo/downloads/pbr-*.tar.gz - $OUTDIR/work-dir/pbr-*/*-requirements.txt $OUTDIR/work-dir/setuptools-*/*-requirements.txt $OUTDIR/work-dir/stevedore-*/*-requirements.txt @@ -102,10 +98,6 @@ $OUTDIR/wheels-repo/downloads/setuptools-*.whl $OUTDIR/wheels-repo/downloads/pbr-*.whl $OUTDIR/wheels-repo/downloads/stevedore-*.whl -$OUTDIR/sdists-repo/downloads/stevedore-*.tar.gz -$OUTDIR/sdists-repo/downloads/setuptools-*.tar.gz -$OUTDIR/sdists-repo/downloads/pbr-*.tar.gz - $OUTDIR/work-dir/build-order.json $OUTDIR/work-dir/constraints.txt " diff --git a/src/fromager/bootstrapper.py b/src/fromager/bootstrapper.py index 2e991eb8..e0a506cb 100644 --- a/src/fromager/bootstrapper.py +++ b/src/fromager/bootstrapper.py @@ -71,15 +71,15 @@ def bootstrap(self, req: Requirement, req_type: RequirementType) -> Version: pbi = self.ctx.package_build_info(req) if pbi.pre_built: - resolved_version, wheel_url, wheel_filename, unpack_dir = ( - self._resolve_and_download_prebuilt(req, req_type) + wheel_url, resolved_version = self._resolve_prebuilt_with_history( + req=req, + req_type=req_type, ) - # Remember that this is a prebuilt wheel, and where we got it. source_url = wheel_url - source_url_type = str(SourceType.PREBUILT) else: - resolved_version, source_url, source_filename, source_url_type = ( - self._resolve_and_download_source(req, req_type) + source_url, resolved_version = self._resolve_source_with_history( + req=req, + req_type=req_type, ) self._add_to_graph(req, req_type, resolved_version, source_url) @@ -104,11 +104,28 @@ def bootstrap(self, req: Requirement, req_type: RequirementType) -> Version: # for cleanup build_env = None sdist_root_dir = None - if not pbi.pre_built: + if pbi.pre_built: + wheel_filename, unpack_dir = self._download_prebuilt( + req=req, + req_type=req_type, + resolved_version=resolved_version, + wheel_url=source_url, + ) + # Remember that this is a prebuilt wheel, and where we got it. + source_url_type = str(SourceType.PREBUILT) + else: unpacked_cached_wheel, cached_wheel_filename = ( self._download_wheel_from_cache(req, resolved_version) ) + source_url_type = sources.get_source_type(self.ctx, req) + if not unpacked_cached_wheel: + source_filename = sources.download_source( + ctx=self.ctx, + req=req, + version=resolved_version, + download_url=source_url, + ) sdist_root_dir = sources.prepare_source( ctx=self.ctx, req=req, @@ -263,34 +280,15 @@ def _handle_build_requirements( build_environment.maybe_install(self.ctx, dep, build_type, str(resolved)) self.progressbar.update() - def _resolve_and_download_source( - self, req: Requirement, req_type: RequirementType - ) -> tuple[Version, str, pathlib.Path, str]: - source_url, resolved_version = self._resolve_source_with_history( - req=req, - req_type=req_type, - ) - - source_filename = sources.download_source( - ctx=self.ctx, - req=req, - version=resolved_version, - download_url=source_url, - ) - source_url_type = sources.get_source_type(self.ctx, req) - - return (resolved_version, source_url, source_filename, source_url_type) - - def _resolve_and_download_prebuilt( - self, req: Requirement, req_type: RequirementType - ) -> tuple[Version, str, pathlib.Path, pathlib.Path]: + def _download_prebuilt( + self, + req: Requirement, + req_type: RequirementType, + resolved_version: Version, + wheel_url: str, + ) -> tuple[pathlib.Path, pathlib.Path]: logger.info(f"{req.name}: {req_type} requirement {req} uses a pre-built wheel") - wheel_url, resolved_version = self._resolve_prebuilt_with_history( - req=req, - req_type=req_type, - ) - wheel_filename = wheels.download_wheel(req, wheel_url, self.ctx.wheels_prebuilt) # Add the wheel to the mirror so it is available to anything @@ -304,7 +302,7 @@ def _resolve_and_download_prebuilt( shutil.copy(wheel_filename, dest_name) server.update_wheel_mirror(self.ctx) unpack_dir = self._create_unpack_dir(req, resolved_version) - return (resolved_version, wheel_url, wheel_filename, unpack_dir) + return (wheel_filename, unpack_dir) def _download_wheel_from_cache( self, req: Requirement, resolved_version: Version From daba74af680d378db501927adbc77d78a3ab8ab2 Mon Sep 17 00:00:00 2001 From: Shubh Bapna Date: Thu, 21 Nov 2024 15:34:37 -0500 Subject: [PATCH 2/2] use tar.next() instead of tar.getnames() Signed-off-by: Shubh Bapna --- src/fromager/sources.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/fromager/sources.py b/src/fromager/sources.py index da8965f8..ff34b244 100644 --- a/src/fromager/sources.py +++ b/src/fromager/sources.py @@ -173,8 +173,7 @@ def _download_source_check( raise zipfile.BadZipFile(f"Empty zip file encountered: {source_filename}") elif source_filename.suffix == ".tgz" or source_filename.suffix == ".gz": with tarfile.open(source_filename) as tar: - contents = tar.getnames() - if not contents: + if not tar.next(): raise TypeError(f"Empty tar file encountered: {source_filename}") else: raise TypeError(