From 2a0d694a2a1a83a4ef8260c20ffbbe7b384be2d0 Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 15 Oct 2025 14:16:01 -0600 Subject: [PATCH 01/18] Update ensurepip bundled dependencies --- relenv/build/common.py | 60 +++++++++++++++++++++++++++++++++++++++++ relenv/build/darwin.py | 12 ++++++++- relenv/build/linux.py | 3 +++ relenv/build/windows.py | 44 +++++++++++------------------- 4 files changed, 90 insertions(+), 29 deletions(-) diff --git a/relenv/build/common.py b/relenv/build/common.py index 85f4221b..3c9f44d1 100644 --- a/relenv/build/common.py +++ b/relenv/build/common.py @@ -358,6 +358,66 @@ def build_sqlite(env, dirs, logfp): runcmd(["make", "install"], env=env, stderr=logfp, stdout=logfp) +def update_ensurepip(source_dir): + """ + Update bundled dependencies for ensurepip (pip & setuptools). + """ + # Download whl files + # pip + pip_version = "25.2" + whl = f"pip-{pip_version}-py3-none-any.whl" + whl_path = "b7/3f/945ef7ab14dc4f9d7f40288d2df998d1837ee0888ec3659c813487572faa" + url = f"https://files.pythonhosted.org/packages/{whl_path}/{whl}" + dest = source_dir / "Lib" / "ensurepip" / "_bundled" / whl + log.debug("Downloading: %s", url) + download_url(url=url, dest=dest) + + # setuptools + setuptools_version = "80.9.0" + whl = f"setuptools-{setuptools_version}-py3-none-any.whl" + whl_path = "a3/dc/17031897dae0efacfea57dfd3a82fdd2a2aeb58e0ff71b77b87e44edc772" + url = f"https://files.pythonhosted.org/packages/{whl_path}/{whl}" + dest = source_dir / "Lib" / "ensurepip" / "_bundled" / whl + log.debug("Downloading: %s", url) + download_url(url=url, dest=dest) + + # Update __init__.py + init_file = source_dir / "Lib" / "ensurepip" / "__init__.py" + # pip + old = "^_PIP_VERSION.*$" + new = f'_PIP_VERSION="{pip_version}"' + patch_file(path=init_file, old=old, new=new) + + # setuptools + old = "^_SETUPTOOLS_VERSION.*$" + new = f'_SETUPTOOLS_VERSION="{setuptools_version}"' + patch_file(path=init_file, old=old, new=new) + + +def patch_file(path, old, new): + """ + Search a file line by line for a string to replace. + + :param path: Location of the file to search + :type path: str + :param old: The value that will be replaced + :type path: str + :param new: The value that will replace the 'old' value. + :type path: str + """ + log.debug("Patching file: %s", path) + import re + + with open(path, "r") as fp: + content = fp.read() + new_content = "" + for line in content.splitlines(): + re.sub(old, new, line) + new_content += line + os.linesep + with open(path, "w") as fp: + fp.write(new_content) + + def tarball_version(href): if href.endswith("tar.gz"): try: diff --git a/relenv/build/darwin.py b/relenv/build/darwin.py index a9eca639..7d7982b9 100644 --- a/relenv/build/darwin.py +++ b/relenv/build/darwin.py @@ -6,7 +6,14 @@ import io from ..common import arches, DARWIN, MACOS_DEVELOPMENT_TARGET -from .common import runcmd, finalize, build_openssl, build_sqlite, builds +from .common import ( + runcmd, + finalize, + build_openssl, + build_sqlite, + builds, + update_ensurepip, +) ARCHES = arches[DARWIN] @@ -45,6 +52,9 @@ def build_python(env, dirs, logfp): :param logfp: A handle for the log file :type logfp: file """ + # Update ensurepip + update_ensurepip(dirs.source) + env["LDFLAGS"] = "-Wl,-rpath,{prefix}/lib {ldflags}".format( prefix=dirs.prefix, ldflags=env["LDFLAGS"] ) diff --git a/relenv/build/linux.py b/relenv/build/linux.py index c7e754e6..93423aca 100644 --- a/relenv/build/linux.py +++ b/relenv/build/linux.py @@ -353,6 +353,9 @@ def build_python(env, dirs, logfp): :param logfp: A handle for the log file :type logfp: file """ + # Update ensurepip + update_ensurepip(dirs.source) + ldflagopt = f"-Wl,--rpath={dirs.prefix}/lib" if ldflagopt not in env["LDFLAGS"]: env["LDFLAGS"] = f"{ldflagopt} {env['LDFLAGS']}" diff --git a/relenv/build/windows.py b/relenv/build/windows.py index a92425ff..34c65b1e 100644 --- a/relenv/build/windows.py +++ b/relenv/build/windows.py @@ -10,7 +10,15 @@ import pathlib import tarfile import logging -from .common import runcmd, create_archive, MODULE_DIR, builds, install_runtime +from .common import ( + runcmd, + create_archive, + MODULE_DIR, + builds, + install_runtime, + patch_file, + update_ensurepip, +) from ..common import arches, WIN32 log = logging.getLogger(__name__) @@ -36,39 +44,16 @@ def populate_env(env, dirs): env["MSBUILDDISABLENODEREUSE"] = "1" -def patch_file(path, old, new): - """ - Search a file line by line for a string to replace. - - :param path: Location of the file to search - :type path: str - :param old: The value that will be replaced - :type path: str - :param new: The value that will replace the 'old' value. - :type path: str - """ - import re - - with open(path, "r") as fp: - content = fp.read() - new_content = "" - for line in content.splitlines(): - re.sub(old, new, line) - new_content += line + os.linesep - with open(path, "w") as fp: - fp.write(new_content) - - def override_dependency(source, old, new): """ - Overwrite a dependency string for Windoes PCBuild. + Overwrite a dependency string for Windows PCBuild. :param source: Python's source directory - :type path: str + :type source: str :param old: Regular expression to search for - :type path: str + :type old: str :param new: Replacement text - :type path: str + :type new: str """ patch_file(source / "PCbuild" / "python.props", old, new) patch_file(source / "PCbuild" / "get_externals.bat", old, new) @@ -85,6 +70,9 @@ def build_python(env, dirs, logfp): :param logfp: A handle for the log file :type logfp: file """ + # Update ensurepip + update_ensurepip(dirs.source) + # Override default versions if env["RELENV_PY_MAJOR_VERSION"] in [ "3.10", From 50c309de0b83b695edc09f085f326158f870f216 Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 15 Oct 2025 14:20:55 -0600 Subject: [PATCH 02/18] set destination to directory name --- relenv/build/common.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/relenv/build/common.py b/relenv/build/common.py index 3c9f44d1..35b17ab3 100644 --- a/relenv/build/common.py +++ b/relenv/build/common.py @@ -363,23 +363,22 @@ def update_ensurepip(source_dir): Update bundled dependencies for ensurepip (pip & setuptools). """ # Download whl files + dest_dir = source_dir / "Lib" / "ensurepip" / "_bundled" # pip pip_version = "25.2" whl = f"pip-{pip_version}-py3-none-any.whl" whl_path = "b7/3f/945ef7ab14dc4f9d7f40288d2df998d1837ee0888ec3659c813487572faa" url = f"https://files.pythonhosted.org/packages/{whl_path}/{whl}" - dest = source_dir / "Lib" / "ensurepip" / "_bundled" / whl log.debug("Downloading: %s", url) - download_url(url=url, dest=dest) + download_url(url=url, dest=dest_dir) # setuptools setuptools_version = "80.9.0" whl = f"setuptools-{setuptools_version}-py3-none-any.whl" whl_path = "a3/dc/17031897dae0efacfea57dfd3a82fdd2a2aeb58e0ff71b77b87e44edc772" url = f"https://files.pythonhosted.org/packages/{whl_path}/{whl}" - dest = source_dir / "Lib" / "ensurepip" / "_bundled" / whl log.debug("Downloading: %s", url) - download_url(url=url, dest=dest) + download_url(url=url, dest=dest_dir) # Update __init__.py init_file = source_dir / "Lib" / "ensurepip" / "__init__.py" From 6eba0e148f7a3687eda638fe69ef3974a1a3af2e Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 15 Oct 2025 14:56:37 -0600 Subject: [PATCH 03/18] Put ensurepip update in finalize step --- relenv/build/common.py | 17 ++++++++++++++--- relenv/build/darwin.py | 3 --- relenv/build/linux.py | 3 --- relenv/build/windows.py | 6 +++--- 4 files changed, 17 insertions(+), 12 deletions(-) diff --git a/relenv/build/common.py b/relenv/build/common.py index 35b17ab3..25e2cfaa 100644 --- a/relenv/build/common.py +++ b/relenv/build/common.py @@ -362,15 +362,22 @@ def update_ensurepip(source_dir): """ Update bundled dependencies for ensurepip (pip & setuptools). """ + # ensurepip bundle location + bundle_dir = source_dir / "Lib" / "ensurepip" / "_bundled" + + # Remove existing whl files + for file in bundle_dir.glob("*.whl"): + if file.is_file(): + file.unlink() + # Download whl files - dest_dir = source_dir / "Lib" / "ensurepip" / "_bundled" # pip pip_version = "25.2" whl = f"pip-{pip_version}-py3-none-any.whl" whl_path = "b7/3f/945ef7ab14dc4f9d7f40288d2df998d1837ee0888ec3659c813487572faa" url = f"https://files.pythonhosted.org/packages/{whl_path}/{whl}" log.debug("Downloading: %s", url) - download_url(url=url, dest=dest_dir) + download_url(url=url, dest=bundle_dir) # setuptools setuptools_version = "80.9.0" @@ -378,7 +385,7 @@ def update_ensurepip(source_dir): whl_path = "a3/dc/17031897dae0efacfea57dfd3a82fdd2a2aeb58e0ff71b77b87e44edc772" url = f"https://files.pythonhosted.org/packages/{whl_path}/{whl}" log.debug("Downloading: %s", url) - download_url(url=url, dest=dest_dir) + download_url(url=url, dest=bundle_dir) # Update __init__.py init_file = source_dir / "Lib" / "ensurepip" / "__init__.py" @@ -1533,6 +1540,10 @@ def find_pythonlib(libdir): bindir = pathlib.Path(dirs.prefix) / "bin" sitepackages = pymodules / "site-packages" install_runtime(sitepackages) + + # update ensurepip + update_ensurepip(dirs.prefix) + # Install pip python = dirs.prefix / "bin" / "python3" if env["RELENV_HOST_ARCH"] != env["RELENV_BUILD_ARCH"]: diff --git a/relenv/build/darwin.py b/relenv/build/darwin.py index 7d7982b9..55afb610 100644 --- a/relenv/build/darwin.py +++ b/relenv/build/darwin.py @@ -52,9 +52,6 @@ def build_python(env, dirs, logfp): :param logfp: A handle for the log file :type logfp: file """ - # Update ensurepip - update_ensurepip(dirs.source) - env["LDFLAGS"] = "-Wl,-rpath,{prefix}/lib {ldflags}".format( prefix=dirs.prefix, ldflags=env["LDFLAGS"] ) diff --git a/relenv/build/linux.py b/relenv/build/linux.py index 93423aca..c7e754e6 100644 --- a/relenv/build/linux.py +++ b/relenv/build/linux.py @@ -353,9 +353,6 @@ def build_python(env, dirs, logfp): :param logfp: A handle for the log file :type logfp: file """ - # Update ensurepip - update_ensurepip(dirs.source) - ldflagopt = f"-Wl,--rpath={dirs.prefix}/lib" if ldflagopt not in env["LDFLAGS"]: env["LDFLAGS"] = f"{ldflagopt} {env['LDFLAGS']}" diff --git a/relenv/build/windows.py b/relenv/build/windows.py index 34c65b1e..c1d79eaf 100644 --- a/relenv/build/windows.py +++ b/relenv/build/windows.py @@ -70,9 +70,6 @@ def build_python(env, dirs, logfp): :param logfp: A handle for the log file :type logfp: file """ - # Update ensurepip - update_ensurepip(dirs.source) - # Override default versions if env["RELENV_PY_MAJOR_VERSION"] in [ "3.10", @@ -193,6 +190,9 @@ def finalize(env, dirs, logfp): install_runtime(sitepackages) + # update ensurepip + update_ensurepip(dirs.prefix) + # Install pip python = dirs.prefix / "Scripts" / "python.exe" runcmd([str(python), "-m", "ensurepip"], env=env, stderr=logfp, stdout=logfp) From 223ba35bc8c5bf2198afca913071a40ff786e641 Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 15 Oct 2025 15:19:28 -0600 Subject: [PATCH 04/18] Add some debugging to the logs --- relenv/build/common.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/relenv/build/common.py b/relenv/build/common.py index 25e2cfaa..b3096416 100644 --- a/relenv/build/common.py +++ b/relenv/build/common.py @@ -365,7 +365,7 @@ def update_ensurepip(source_dir): # ensurepip bundle location bundle_dir = source_dir / "Lib" / "ensurepip" / "_bundled" - # Remove existing whl files + # Remove existing whl files for file in bundle_dir.glob("*.whl"): if file.is_file(): file.unlink() @@ -376,16 +376,16 @@ def update_ensurepip(source_dir): whl = f"pip-{pip_version}-py3-none-any.whl" whl_path = "b7/3f/945ef7ab14dc4f9d7f40288d2df998d1837ee0888ec3659c813487572faa" url = f"https://files.pythonhosted.org/packages/{whl_path}/{whl}" - log.debug("Downloading: %s", url) download_url(url=url, dest=bundle_dir) + assert (bundle_dir / whl).exists() # setuptools setuptools_version = "80.9.0" whl = f"setuptools-{setuptools_version}-py3-none-any.whl" whl_path = "a3/dc/17031897dae0efacfea57dfd3a82fdd2a2aeb58e0ff71b77b87e44edc772" url = f"https://files.pythonhosted.org/packages/{whl_path}/{whl}" - log.debug("Downloading: %s", url) download_url(url=url, dest=bundle_dir) + assert (bundle_dir / whl).exists() # Update __init__.py init_file = source_dir / "Lib" / "ensurepip" / "__init__.py" @@ -399,6 +399,9 @@ def update_ensurepip(source_dir): new = f'_SETUPTOOLS_VERSION="{setuptools_version}"' patch_file(path=init_file, old=old, new=new) + log.debug("ensurepip __init__.py contents:") + log.debug(init_file.read_text()) + def patch_file(path, old, new): """ From 9a400bbe99f3aac0581d7d2a660f4a2bea3c5fcd Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 15 Oct 2025 15:23:36 -0600 Subject: [PATCH 05/18] Fix pre-commit --- relenv/build/darwin.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/relenv/build/darwin.py b/relenv/build/darwin.py index 55afb610..a9eca639 100644 --- a/relenv/build/darwin.py +++ b/relenv/build/darwin.py @@ -6,14 +6,7 @@ import io from ..common import arches, DARWIN, MACOS_DEVELOPMENT_TARGET -from .common import ( - runcmd, - finalize, - build_openssl, - build_sqlite, - builds, - update_ensurepip, -) +from .common import runcmd, finalize, build_openssl, build_sqlite, builds ARCHES = arches[DARWIN] From 5125aed90f8b83b40d311d6d8efcb2fd25b6ea31 Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 15 Oct 2025 15:28:18 -0600 Subject: [PATCH 06/18] Use \n for newline --- relenv/build/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/relenv/build/common.py b/relenv/build/common.py index b3096416..2415a59c 100644 --- a/relenv/build/common.py +++ b/relenv/build/common.py @@ -422,7 +422,7 @@ def patch_file(path, old, new): new_content = "" for line in content.splitlines(): re.sub(old, new, line) - new_content += line + os.linesep + new_content += line + "\n" with open(path, "w") as fp: fp.write(new_content) From b2bbccd7e7f51ac20942339575b55fad47584167 Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 15 Oct 2025 15:39:29 -0600 Subject: [PATCH 07/18] Fix name collision with patch_file --- relenv/build/linux.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/relenv/build/linux.py b/relenv/build/linux.py index c7e754e6..f1b636de 100644 --- a/relenv/build/linux.py +++ b/relenv/build/linux.py @@ -379,11 +379,11 @@ def build_python(env, dirs, logfp): ) if pathlib.Path("setup.py").exists(): - with tempfile.NamedTemporaryFile(mode="w", suffix="_patch") as patch_file: - patch_file.write(PATCH) - patch_file.flush() + with tempfile.NamedTemporaryFile(mode="w", suffix="_patch") as p_file: + p_file.write(PATCH) + p_file.flush() runcmd( - ["patch", "-p0", "-i", patch_file.name], + ["patch", "-p0", "-i", p_file.name], env=env, stderr=logfp, stdout=logfp, From c4633618a6934f495698a8ef55d5ee4642a05623 Mon Sep 17 00:00:00 2001 From: twangboy Date: Thu, 16 Oct 2025 14:47:34 -0600 Subject: [PATCH 08/18] Make sure bundle directory exists --- relenv/build/common.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/relenv/build/common.py b/relenv/build/common.py index 2415a59c..e99b9bf0 100644 --- a/relenv/build/common.py +++ b/relenv/build/common.py @@ -365,6 +365,9 @@ def update_ensurepip(source_dir): # ensurepip bundle location bundle_dir = source_dir / "Lib" / "ensurepip" / "_bundled" + # Make sure the destination directory exists + bundle_dir.mkdir(parents=True, exist_ok=True) + # Remove existing whl files for file in bundle_dir.glob("*.whl"): if file.is_file(): From 4cab99506cb32355858e8befa1dd2a1b71959637 Mon Sep 17 00:00:00 2001 From: twangboy Date: Thu, 16 Oct 2025 15:38:31 -0600 Subject: [PATCH 09/18] Fix patch_file and regex for ensurepip --- relenv/build/common.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/relenv/build/common.py b/relenv/build/common.py index e99b9bf0..5f4dbbfc 100644 --- a/relenv/build/common.py +++ b/relenv/build/common.py @@ -393,13 +393,13 @@ def update_ensurepip(source_dir): # Update __init__.py init_file = source_dir / "Lib" / "ensurepip" / "__init__.py" # pip - old = "^_PIP_VERSION.*$" - new = f'_PIP_VERSION="{pip_version}"' + old = "^_PIP_VERSION.*" + new = f'_PIP_VERSION = "{pip_version}"' patch_file(path=init_file, old=old, new=new) # setuptools - old = "^_SETUPTOOLS_VERSION.*$" - new = f'_SETUPTOOLS_VERSION="{setuptools_version}"' + old = "^_SETUPTOOLS_VERSION.*" + new = f'_SETUPTOOLS_VERSION = "{setuptools_version}"' patch_file(path=init_file, old=old, new=new) log.debug("ensurepip __init__.py contents:") @@ -424,8 +424,8 @@ def patch_file(path, old, new): content = fp.read() new_content = "" for line in content.splitlines(): - re.sub(old, new, line) - new_content += line + "\n" + line = re.sub(old, new, line) + new_content += line + os.linesep with open(path, "w") as fp: fp.write(new_content) From 3f1e6b8fc7327b43c2d7196738eee509cb9d75fb Mon Sep 17 00:00:00 2001 From: twangboy Date: Thu, 16 Oct 2025 16:19:39 -0600 Subject: [PATCH 10/18] Build externals on Windows --- relenv/build/windows.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/relenv/build/windows.py b/relenv/build/windows.py index c1d79eaf..3e99934a 100644 --- a/relenv/build/windows.py +++ b/relenv/build/windows.py @@ -90,7 +90,9 @@ def build_python(env, dirs, logfp): "-p", plat, "--no-tkinter", + "-e", ] + log.info("Start PCbuild") runcmd(cmd, env=env, stderr=logfp, stdout=logfp) log.info("PCbuild finished") From f5b8cc50fec464bb5faa19eea03641651f39ebd1 Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 17 Oct 2025 15:05:13 -0600 Subject: [PATCH 11/18] Pass organization to get_externals.bat --- relenv/build/windows.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/relenv/build/windows.py b/relenv/build/windows.py index 3e99934a..42e56352 100644 --- a/relenv/build/windows.py +++ b/relenv/build/windows.py @@ -77,6 +77,11 @@ def build_python(env, dirs, logfp): ]: override_dependency(dirs.source, r"sqlite-\d+.\d+.\d+.\d+", "sqlite-3.50.4.0") override_dependency(dirs.source, r"xz-\d+.\d+.\d+", "xz-5.6.2") + # We don't have a way to pass --organization to build.bat, so we need to + # patch it + old = f'.*get_externals\.bat"' + new = 'get_externals.bat --organization saltstack"' + patch_file(dirs.source / "PCbuild" / "build.bat", old=old, new=new) arch_to_plat = { "amd64": "x64", From 6075b255f043cdd9b6c209907cee97a3ca3d00ff Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 17 Oct 2025 15:29:21 -0600 Subject: [PATCH 12/18] Update ensurepip in the build step --- relenv/build/common.py | 3 --- relenv/build/darwin.py | 4 ++++ relenv/build/linux.py | 4 ++++ relenv/build/windows.py | 6 +++--- 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/relenv/build/common.py b/relenv/build/common.py index 5f4dbbfc..6a2ad2ee 100644 --- a/relenv/build/common.py +++ b/relenv/build/common.py @@ -1547,9 +1547,6 @@ def find_pythonlib(libdir): sitepackages = pymodules / "site-packages" install_runtime(sitepackages) - # update ensurepip - update_ensurepip(dirs.prefix) - # Install pip python = dirs.prefix / "bin" / "python3" if env["RELENV_HOST_ARCH"] != env["RELENV_BUILD_ARCH"]: diff --git a/relenv/build/darwin.py b/relenv/build/darwin.py index a9eca639..d6b28945 100644 --- a/relenv/build/darwin.py +++ b/relenv/build/darwin.py @@ -45,6 +45,10 @@ def build_python(env, dirs, logfp): :param logfp: A handle for the log file :type logfp: file """ + + # update ensurepip + update_ensurepip(dirs.prefix) + env["LDFLAGS"] = "-Wl,-rpath,{prefix}/lib {ldflags}".format( prefix=dirs.prefix, ldflags=env["LDFLAGS"] ) diff --git a/relenv/build/linux.py b/relenv/build/linux.py index f1b636de..e7fcc3a5 100644 --- a/relenv/build/linux.py +++ b/relenv/build/linux.py @@ -353,6 +353,10 @@ def build_python(env, dirs, logfp): :param logfp: A handle for the log file :type logfp: file """ + + # update ensurepip + update_ensurepip(dirs.prefix) + ldflagopt = f"-Wl,--rpath={dirs.prefix}/lib" if ldflagopt not in env["LDFLAGS"]: env["LDFLAGS"] = f"{ldflagopt} {env['LDFLAGS']}" diff --git a/relenv/build/windows.py b/relenv/build/windows.py index 42e56352..6d263f39 100644 --- a/relenv/build/windows.py +++ b/relenv/build/windows.py @@ -83,6 +83,9 @@ def build_python(env, dirs, logfp): new = 'get_externals.bat --organization saltstack"' patch_file(dirs.source / "PCbuild" / "build.bat", old=old, new=new) + # update ensurepip + update_ensurepip(dirs.source) + arch_to_plat = { "amd64": "x64", "x86": "win32", @@ -197,9 +200,6 @@ def finalize(env, dirs, logfp): install_runtime(sitepackages) - # update ensurepip - update_ensurepip(dirs.prefix) - # Install pip python = dirs.prefix / "Scripts" / "python.exe" runcmd([str(python), "-m", "ensurepip"], env=env, stderr=logfp, stdout=logfp) From 609784727ce25ead2d13757fff2e37dca06e11e9 Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 17 Oct 2025 15:35:58 -0600 Subject: [PATCH 13/18] Fix pre-commit errors --- relenv/build/darwin.py | 10 ++++++++-- relenv/build/linux.py | 1 - relenv/build/windows.py | 2 +- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/relenv/build/darwin.py b/relenv/build/darwin.py index d6b28945..e5723827 100644 --- a/relenv/build/darwin.py +++ b/relenv/build/darwin.py @@ -6,7 +6,14 @@ import io from ..common import arches, DARWIN, MACOS_DEVELOPMENT_TARGET -from .common import runcmd, finalize, build_openssl, build_sqlite, builds +from .common import ( + runcmd, + finalize, + build_openssl, + build_sqlite, + builds, + update_ensurepip, +) ARCHES = arches[DARWIN] @@ -45,7 +52,6 @@ def build_python(env, dirs, logfp): :param logfp: A handle for the log file :type logfp: file """ - # update ensurepip update_ensurepip(dirs.prefix) diff --git a/relenv/build/linux.py b/relenv/build/linux.py index e7fcc3a5..b479d576 100644 --- a/relenv/build/linux.py +++ b/relenv/build/linux.py @@ -353,7 +353,6 @@ def build_python(env, dirs, logfp): :param logfp: A handle for the log file :type logfp: file """ - # update ensurepip update_ensurepip(dirs.prefix) diff --git a/relenv/build/windows.py b/relenv/build/windows.py index 6d263f39..d2161186 100644 --- a/relenv/build/windows.py +++ b/relenv/build/windows.py @@ -79,7 +79,7 @@ def build_python(env, dirs, logfp): override_dependency(dirs.source, r"xz-\d+.\d+.\d+", "xz-5.6.2") # We don't have a way to pass --organization to build.bat, so we need to # patch it - old = f'.*get_externals\.bat"' + old = r'.*get_externals\.bat"' new = 'get_externals.bat --organization saltstack"' patch_file(dirs.source / "PCbuild" / "build.bat", old=old, new=new) From eb6218e6888e44e4619cc7abf110a43a1686d13f Mon Sep 17 00:00:00 2001 From: twangboy Date: Mon, 20 Oct 2025 08:47:41 -0600 Subject: [PATCH 14/18] Move ensurepip back to finalize --- relenv/build/common.py | 49 ++++++++++++++++++++++++++--------------- relenv/build/darwin.py | 12 +--------- relenv/build/linux.py | 3 --- relenv/build/windows.py | 7 +++--- 4 files changed, 35 insertions(+), 36 deletions(-) diff --git a/relenv/build/common.py b/relenv/build/common.py index 6a2ad2ee..c1d9a08a 100644 --- a/relenv/build/common.py +++ b/relenv/build/common.py @@ -358,40 +358,50 @@ def build_sqlite(env, dirs, logfp): runcmd(["make", "install"], env=env, stderr=logfp, stdout=logfp) -def update_ensurepip(source_dir): +def update_ensurepip(lib_dir): """ Update bundled dependencies for ensurepip (pip & setuptools). """ # ensurepip bundle location - bundle_dir = source_dir / "Lib" / "ensurepip" / "_bundled" + bundle_dir = lib_dir / "ensurepip" / "_bundled" # Make sure the destination directory exists bundle_dir.mkdir(parents=True, exist_ok=True) - # Remove existing whl files + # Detect existing whl files and delete them. Later versions of python don't + # include setuptools. We only want to update whl files that are a part of + # python + update_pip = False + update_setuptools = False for file in bundle_dir.glob("*.whl"): - if file.is_file(): + if "pip" in file: + update_pip = True + file.unlink() + if "setuptools" in file: + update_setuptools = True file.unlink() # Download whl files # pip - pip_version = "25.2" - whl = f"pip-{pip_version}-py3-none-any.whl" - whl_path = "b7/3f/945ef7ab14dc4f9d7f40288d2df998d1837ee0888ec3659c813487572faa" - url = f"https://files.pythonhosted.org/packages/{whl_path}/{whl}" - download_url(url=url, dest=bundle_dir) - assert (bundle_dir / whl).exists() + if update_pip: + pip_version = "25.2" + whl = f"pip-{pip_version}-py3-none-any.whl" + whl_path = "b7/3f/945ef7ab14dc4f9d7f40288d2df998d1837ee0888ec3659c813487572faa" + url = f"https://files.pythonhosted.org/packages/{whl_path}/{whl}" + download_url(url=url, dest=bundle_dir) + assert (bundle_dir / whl).exists() # setuptools - setuptools_version = "80.9.0" - whl = f"setuptools-{setuptools_version}-py3-none-any.whl" - whl_path = "a3/dc/17031897dae0efacfea57dfd3a82fdd2a2aeb58e0ff71b77b87e44edc772" - url = f"https://files.pythonhosted.org/packages/{whl_path}/{whl}" - download_url(url=url, dest=bundle_dir) - assert (bundle_dir / whl).exists() + if update_setuptools: + setuptools_version = "80.9.0" + whl = f"setuptools-{setuptools_version}-py3-none-any.whl" + whl_path = "a3/dc/17031897dae0efacfea57dfd3a82fdd2a2aeb58e0ff71b77b87e44edc772" + url = f"https://files.pythonhosted.org/packages/{whl_path}/{whl}" + download_url(url=url, dest=bundle_dir) + assert (bundle_dir / whl).exists() # Update __init__.py - init_file = source_dir / "Lib" / "ensurepip" / "__init__.py" + init_file = lib_dir / "ensurepip" / "__init__.py" # pip old = "^_PIP_VERSION.*" new = f'_PIP_VERSION = "{pip_version}"' @@ -425,7 +435,7 @@ def patch_file(path, old, new): new_content = "" for line in content.splitlines(): line = re.sub(old, new, line) - new_content += line + os.linesep + new_content += line + "\n" with open(path, "w") as fp: fp.write(new_content) @@ -1521,6 +1531,9 @@ def finalize(env, dirs, logfp): # Install relenv-sysconfigdata module libdir = pathlib.Path(dirs.prefix) / "lib" + # update ensurepip + update_ensurepip(libdir) + def find_pythonlib(libdir): for root, dirs, files in os.walk(libdir): for _ in dirs: diff --git a/relenv/build/darwin.py b/relenv/build/darwin.py index e5723827..a9eca639 100644 --- a/relenv/build/darwin.py +++ b/relenv/build/darwin.py @@ -6,14 +6,7 @@ import io from ..common import arches, DARWIN, MACOS_DEVELOPMENT_TARGET -from .common import ( - runcmd, - finalize, - build_openssl, - build_sqlite, - builds, - update_ensurepip, -) +from .common import runcmd, finalize, build_openssl, build_sqlite, builds ARCHES = arches[DARWIN] @@ -52,9 +45,6 @@ def build_python(env, dirs, logfp): :param logfp: A handle for the log file :type logfp: file """ - # update ensurepip - update_ensurepip(dirs.prefix) - env["LDFLAGS"] = "-Wl,-rpath,{prefix}/lib {ldflags}".format( prefix=dirs.prefix, ldflags=env["LDFLAGS"] ) diff --git a/relenv/build/linux.py b/relenv/build/linux.py index b479d576..f1b636de 100644 --- a/relenv/build/linux.py +++ b/relenv/build/linux.py @@ -353,9 +353,6 @@ def build_python(env, dirs, logfp): :param logfp: A handle for the log file :type logfp: file """ - # update ensurepip - update_ensurepip(dirs.prefix) - ldflagopt = f"-Wl,--rpath={dirs.prefix}/lib" if ldflagopt not in env["LDFLAGS"]: env["LDFLAGS"] = f"{ldflagopt} {env['LDFLAGS']}" diff --git a/relenv/build/windows.py b/relenv/build/windows.py index d2161186..4010588f 100644 --- a/relenv/build/windows.py +++ b/relenv/build/windows.py @@ -83,9 +83,6 @@ def build_python(env, dirs, logfp): new = 'get_externals.bat --organization saltstack"' patch_file(dirs.source / "PCbuild" / "build.bat", old=old, new=new) - # update ensurepip - update_ensurepip(dirs.source) - arch_to_plat = { "amd64": "x64", "x86": "win32", @@ -197,9 +194,11 @@ def finalize(env, dirs, logfp): """ # Lay down site customize sitepackages = dirs.prefix / "Lib" / "site-packages" - install_runtime(sitepackages) + # update ensurepip + update_ensurepip(dirs.prefix / "Lib") + # Install pip python = dirs.prefix / "Scripts" / "python.exe" runcmd([str(python), "-m", "ensurepip"], env=env, stderr=logfp, stdout=logfp) From 19dda23a7e838cebb48c63aa47e1f3eeb7ddf3c4 Mon Sep 17 00:00:00 2001 From: twangboy Date: Mon, 20 Oct 2025 11:10:46 -0600 Subject: [PATCH 15/18] Don't depend on forked cpython-source-deps repo --- relenv/build/windows.py | 76 ++++++++++++++++++++++++++++++----------- 1 file changed, 57 insertions(+), 19 deletions(-) diff --git a/relenv/build/windows.py b/relenv/build/windows.py index 4010588f..5f4f38fd 100644 --- a/relenv/build/windows.py +++ b/relenv/build/windows.py @@ -4,19 +4,21 @@ The windows build process. """ import glob -import shutil -import sys +import logging import os import pathlib +import shutil +import sys import tarfile -import logging from .common import ( - runcmd, - create_archive, - MODULE_DIR, builds, + create_archive, + download_url, + extract_archive, install_runtime, + MODULE_DIR, patch_file, + runcmd, update_ensurepip, ) from ..common import arches, WIN32 @@ -44,7 +46,7 @@ def populate_env(env, dirs): env["MSBUILDDISABLENODEREUSE"] = "1" -def override_dependency(source, old, new): +def update_props(source, old, new): """ Overwrite a dependency string for Windows PCBuild. @@ -56,7 +58,30 @@ def override_dependency(source, old, new): :type new: str """ patch_file(source / "PCbuild" / "python.props", old, new) - patch_file(source / "PCbuild" / "get_externals.bat", old, new) + + +def get_externals_source(source_root, url): + """ + Download external source code dependency. + + Download source code and extract to the "externals" directory in the root of + the python source. Only works with a tarball + """ + externals_dir = source_root / "externals" + externals_dir.mkdir(parents=True, exist_ok=True) + local_file = download_url(url, str(externals_dir)) + extract_archive(str(local_file), str(externals_dir)) + os.path.unlink(str(local_file)) + + +def get_externals_bin(source_root, url): + """ + Download external binary dependency. + + Download binaries to the "externals" directory in the root of the python + source. + """ + pass def build_python(env, dirs, logfp): @@ -71,17 +96,30 @@ def build_python(env, dirs, logfp): :type logfp: file """ # Override default versions - if env["RELENV_PY_MAJOR_VERSION"] in [ - "3.10", - "3.11", - ]: - override_dependency(dirs.source, r"sqlite-\d+.\d+.\d+.\d+", "sqlite-3.50.4.0") - override_dependency(dirs.source, r"xz-\d+.\d+.\d+", "xz-5.6.2") - # We don't have a way to pass --organization to build.bat, so we need to - # patch it - old = r'.*get_externals\.bat"' - new = 'get_externals.bat --organization saltstack"' - patch_file(dirs.source / "PCbuild" / "build.bat", old=old, new=new) + # SQLITE + if env["RELENV_PY_MAJOR_VERSION"] in ["3.10", "3.11", "3.12"]: + version = "3.50.4.0" + update_props(dirs.source, r"sqlite-\d+.\d+.\d+.\d+", "sqlite-{ver}") + url = "https://sqlite.org/2025/sqlite-autoconf-3500400.tar.gz" + get_externals_source(dirs.source, url=url) + # we need to fix the name of the extracted directory + extracted_dir = dirs.source / "externals" / "sqlite-src-3500400" + target_dir = dirs.source / "externals" / f"sqlite-{version}" + shutil.move(str(extracted_dir), str(target_dir)) + + # XZ-Utils + if env["RELENV_PY_MAJOR_VERSION"] in ["3.10", "3.11", "3.12", "3.13", "3.14"]: + update_props(dirs.source, r"xz-\d+.\d+.\d+", "xz-5.6.2") + url = "https://github.com/tukaani-project/xz/releases/download/v5.6.2/xz-5.6.2.tar.xz" + get_externals_source(dirs.source, url=url) + + # zlib (3.14 uses zlib-ng) + if env["RELENV_PY_MAJOR_VERSION"] in ["3.10", "3.11", "3.12", "3.13"]: + # already in python.props with the correct version in all the above versions + # update_props(dirs.source, r"zlib-\d+.\d+.\d+", "zlib-1.3.1") + # but it still needs to be in "externals" + url = "https://zlib.net/zlib-1.3.1.tar.gz" + get_externals_source(dirs.source, url=url) arch_to_plat = { "amd64": "x64", From c72d7a985c3196667dd621d4e10e547c648daa50 Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 21 Oct 2025 15:41:04 -0600 Subject: [PATCH 16/18] Get windows building --- relenv/_resources/xz/config.h | 148 +++++++++++++++++++++++++++++++++ relenv/_resources/xz/readme.md | 4 + relenv/build/common.py | 61 ++++++++------ relenv/build/windows.py | 63 ++++++++------ relenv/common.py | 18 +++- 5 files changed, 241 insertions(+), 53 deletions(-) create mode 100644 relenv/_resources/xz/config.h create mode 100644 relenv/_resources/xz/readme.md diff --git a/relenv/_resources/xz/config.h b/relenv/_resources/xz/config.h new file mode 100644 index 00000000..ef921e80 --- /dev/null +++ b/relenv/_resources/xz/config.h @@ -0,0 +1,148 @@ +/* config.h for compiling liblzma (*not* the whole XZ Utils) with MSVC 2019 */ + +/* Prefix for symbols exported by tuklib_*.c files */ +#define TUKLIB_SYMBOL_PREFIX lzma_ + +/* How many MiB of RAM to assume if the real amount cannot be determined. */ +#define ASSUME_RAM 128 + +/* Define to 1 if crc32 integrity check is enabled. */ +#define HAVE_CHECK_CRC32 1 + +/* Define to 1 if crc64 integrity check is enabled. */ +#define HAVE_CHECK_CRC64 1 + +/* Define to 1 if sha256 integrity check is enabled. */ +#define HAVE_CHECK_SHA256 1 + +/* Define to 1 if any of HAVE_DECODER_foo have been defined. */ +#define HAVE_DECODERS 1 + +/* Define to 1 if arm decoder is enabled. */ +#define HAVE_DECODER_ARM 1 + +/* Define to 1 if armthumb decoder is enabled. */ +#define HAVE_DECODER_ARMTHUMB 1 + +/* Define to 1 if delta decoder is enabled. */ +#define HAVE_DECODER_DELTA 1 + +/* Define to 1 if ia64 decoder is enabled. */ +#define HAVE_DECODER_IA64 1 + +/* Define to 1 if lzma1 decoder is enabled. */ +#define HAVE_DECODER_LZMA1 1 + +/* Define to 1 if lzma2 decoder is enabled. */ +#define HAVE_DECODER_LZMA2 1 + +/* Define to 1 if powerpc decoder is enabled. */ +#define HAVE_DECODER_POWERPC 1 + +/* Define to 1 if sparc decoder is enabled. */ +#define HAVE_DECODER_SPARC 1 + +/* Define to 1 if x86 decoder is enabled. */ +#define HAVE_DECODER_X86 1 + +/* Define to 1 if any of HAVE_ENCODER_foo have been defined. */ +#define HAVE_ENCODERS 1 + +/* Define to 1 if arm encoder is enabled. */ +#define HAVE_ENCODER_ARM 1 + +/* Define to 1 if armthumb encoder is enabled. */ +#define HAVE_ENCODER_ARMTHUMB 1 + +/* Define to 1 if delta encoder is enabled. */ +#define HAVE_ENCODER_DELTA 1 + +/* Define to 1 if ia64 encoder is enabled. */ +#define HAVE_ENCODER_IA64 1 + +/* Define to 1 if lzma1 encoder is enabled. */ +#define HAVE_ENCODER_LZMA1 1 + +/* Define to 1 if lzma2 encoder is enabled. */ +#define HAVE_ENCODER_LZMA2 1 + +/* Define to 1 if powerpc encoder is enabled. */ +#define HAVE_ENCODER_POWERPC 1 + +/* Define to 1 if sparc encoder is enabled. */ +#define HAVE_ENCODER_SPARC 1 + +/* Define to 1 if x86 encoder is enabled. */ +#define HAVE_ENCODER_X86 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_INTTYPES_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_LIMITS_H 1 + +/* Define to 1 to enable bt2 match finder. */ +#define HAVE_MF_BT2 1 + +/* Define to 1 to enable bt3 match finder. */ +#define HAVE_MF_BT3 1 + +/* Define to 1 to enable bt4 match finder. */ +#define HAVE_MF_BT4 1 + +/* Define to 1 to enable hc3 match finder. */ +#define HAVE_MF_HC3 1 + +/* Define to 1 to enable hc4 match finder. */ +#define HAVE_MF_HC4 1 + +/* Define to 1 if stdbool.h conforms to C99. */ +#define HAVE_STDBOOL_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STDINT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STDLIB_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STRING_H 1 + +/* Define to 1 or 0, depending whether the compiler supports simple visibility + declarations. */ +#define HAVE_VISIBILITY 0 + +/* Define to 1 if the system has the type `_Bool'. */ +#define HAVE__BOOL 1 + +#ifdef _M_IX86 +/* Define to 1 when using Windows 95 (and thus XP) compatible threads. This + avoids use of features that were added in Windows Vista. + This is used for 32-bit x86 builds for compatibility reasons since it + makes no measurable difference in performance compared to Vista threads. */ +#define MYTHREAD_WIN95 1 +#else +/* Define to 1 when using Windows Vista compatible threads. This uses features + that are not available on Windows XP. */ +#define MYTHREAD_VISTA 1 +#endif + +/* Define to 1 to disable debugging code. */ +#define NDEBUG 1 + +/* Define to the full name of this package. */ +#define PACKAGE_NAME "XZ Utils" + +/* Define to the home page for this package. */ +#define PACKAGE_URL "https://tukaani.org/xz/" + +/* The size of `size_t', as computed by sizeof. */ +#ifdef _WIN64 +#define SIZEOF_SIZE_T 8 +#else +#define SIZEOF_SIZE_T 4 +#endif + +/* Define to 1 if the system supports fast unaligned access to 16-bit and + 32-bit integers. */ +#define TUKLIB_FAST_UNALIGNED_ACCESS 1 diff --git a/relenv/_resources/xz/readme.md b/relenv/_resources/xz/readme.md new file mode 100644 index 00000000..96e13414 --- /dev/null +++ b/relenv/_resources/xz/readme.md @@ -0,0 +1,4 @@ +The config.h file was removed from XZ-Utils tarting with version 5.5.0. +XZ-Utils seems to build just fine with the config.h file from 5.4.7, so we're +including it here. This will be copied into the src/windows directory in the +extracted source for XZ-Utils. diff --git a/relenv/build/common.py b/relenv/build/common.py index c1d9a08a..d16c6802 100644 --- a/relenv/build/common.py +++ b/relenv/build/common.py @@ -368,49 +368,60 @@ def update_ensurepip(lib_dir): # Make sure the destination directory exists bundle_dir.mkdir(parents=True, exist_ok=True) - # Detect existing whl files and delete them. Later versions of python don't - # include setuptools. We only want to update whl files that are a part of - # python + # Detect existing whl. Later versions of python don't include setuptools. We + # only want to update whl files that python expects to be there + pip_version = "25.2" + setuptools_version = "80.9.0" update_pip = False update_setuptools = False for file in bundle_dir.glob("*.whl"): - if "pip" in file: - update_pip = True - file.unlink() - if "setuptools" in file: - update_setuptools = True - file.unlink() - - # Download whl files - # pip + + log.debug("Checking whl: %s", str(file)) + if file.name.startswith("pip-"): + found_version = file.name.split("-")[1] + log.debug("Found version %s", found_version) + if LooseVersion(found_version) >= LooseVersion(pip_version): + log.debug("Found correct pip version or newer: %s", found_version) + else: + file.unlink() + update_pip = True + if file.name.startswith("setuptools-"): + found_version = file.name.split("-")[1] + log.debug("Found version %s", found_version) + if LooseVersion(found_version) >= LooseVersion(setuptools_version): + log.debug( + "Found correct setuptools version or newer: %s", found_version + ) + else: + file.unlink() + update_setuptools = True + + # Download whl files and update __init__.py + init_file = lib_dir / "ensurepip" / "__init__.py" if update_pip: - pip_version = "25.2" whl = f"pip-{pip_version}-py3-none-any.whl" whl_path = "b7/3f/945ef7ab14dc4f9d7f40288d2df998d1837ee0888ec3659c813487572faa" url = f"https://files.pythonhosted.org/packages/{whl_path}/{whl}" download_url(url=url, dest=bundle_dir) assert (bundle_dir / whl).exists() + # Update __init__.py + old = "^_PIP_VERSION.*" + new = f'_PIP_VERSION = "{pip_version}"' + patch_file(path=init_file, old=old, new=new) + # setuptools if update_setuptools: - setuptools_version = "80.9.0" whl = f"setuptools-{setuptools_version}-py3-none-any.whl" whl_path = "a3/dc/17031897dae0efacfea57dfd3a82fdd2a2aeb58e0ff71b77b87e44edc772" url = f"https://files.pythonhosted.org/packages/{whl_path}/{whl}" download_url(url=url, dest=bundle_dir) assert (bundle_dir / whl).exists() - # Update __init__.py - init_file = lib_dir / "ensurepip" / "__init__.py" - # pip - old = "^_PIP_VERSION.*" - new = f'_PIP_VERSION = "{pip_version}"' - patch_file(path=init_file, old=old, new=new) - - # setuptools - old = "^_SETUPTOOLS_VERSION.*" - new = f'_SETUPTOOLS_VERSION = "{setuptools_version}"' - patch_file(path=init_file, old=old, new=new) + # setuptools + old = "^_SETUPTOOLS_VERSION.*" + new = f'_SETUPTOOLS_VERSION = "{setuptools_version}"' + patch_file(path=init_file, old=old, new=new) log.debug("ensurepip __init__.py contents:") log.debug(init_file.read_text()) diff --git a/relenv/build/windows.py b/relenv/build/windows.py index 5f4f38fd..8e7a4f27 100644 --- a/relenv/build/windows.py +++ b/relenv/build/windows.py @@ -58,20 +58,24 @@ def update_props(source, old, new): :type new: str """ patch_file(source / "PCbuild" / "python.props", old, new) + patch_file(source / "PCbuild" / "get_externals.bat", old, new) -def get_externals_source(source_root, url): +def get_externals_source(externals_dir, url): """ Download external source code dependency. Download source code and extract to the "externals" directory in the root of the python source. Only works with a tarball """ - externals_dir = source_root / "externals" - externals_dir.mkdir(parents=True, exist_ok=True) - local_file = download_url(url, str(externals_dir)) - extract_archive(str(local_file), str(externals_dir)) - os.path.unlink(str(local_file)) + zips_dir = externals_dir / "zips" + zips_dir.mkdir(parents=True, exist_ok=True) + local_file = download_url(url=url, dest=str(zips_dir)) + extract_archive(archive=str(local_file), to_dir=str(externals_dir)) + try: + os.remove(local_file) + except OSError: + log.exception("Failed to remove temporary file") def get_externals_bin(source_root, url): @@ -96,30 +100,39 @@ def build_python(env, dirs, logfp): :type logfp: file """ # Override default versions + + # Create externals directory + externals_dir = dirs.source / "externals" + externals_dir.mkdir(parents=True, exist_ok=True) + # SQLITE if env["RELENV_PY_MAJOR_VERSION"] in ["3.10", "3.11", "3.12"]: version = "3.50.4.0" - update_props(dirs.source, r"sqlite-\d+.\d+.\d+.\d+", "sqlite-{ver}") - url = "https://sqlite.org/2025/sqlite-autoconf-3500400.tar.gz" - get_externals_source(dirs.source, url=url) - # we need to fix the name of the extracted directory - extracted_dir = dirs.source / "externals" / "sqlite-src-3500400" - target_dir = dirs.source / "externals" / f"sqlite-{version}" - shutil.move(str(extracted_dir), str(target_dir)) + target_dir = externals_dir / f"sqlite-{version}" + if not target_dir.exists(): + update_props(dirs.source, r"sqlite-\d+.\d+.\d+.\d+", f"sqlite-{version}") + url = "https://sqlite.org/2025/sqlite-autoconf-3500400.tar.gz" + get_externals_source(externals_dir=externals_dir, url=url) + # # we need to fix the name of the extracted directory + extracted_dir = externals_dir / "sqlite-autoconf-3500400" + shutil.move(str(extracted_dir), str(target_dir)) # XZ-Utils if env["RELENV_PY_MAJOR_VERSION"] in ["3.10", "3.11", "3.12", "3.13", "3.14"]: - update_props(dirs.source, r"xz-\d+.\d+.\d+", "xz-5.6.2") - url = "https://github.com/tukaani-project/xz/releases/download/v5.6.2/xz-5.6.2.tar.xz" - get_externals_source(dirs.source, url=url) - - # zlib (3.14 uses zlib-ng) - if env["RELENV_PY_MAJOR_VERSION"] in ["3.10", "3.11", "3.12", "3.13"]: - # already in python.props with the correct version in all the above versions - # update_props(dirs.source, r"zlib-\d+.\d+.\d+", "zlib-1.3.1") - # but it still needs to be in "externals" - url = "https://zlib.net/zlib-1.3.1.tar.gz" - get_externals_source(dirs.source, url=url) + version = "5.6.2" + target_dir = externals_dir / f"xz-{version}" + if not target_dir.exists(): + update_props(dirs.source, r"xz-\d+.\d+.\d+", f"xz-{version}") + url = f"https://github.com/tukaani-project/xz/releases/download/v{version}/xz-{version}.tar.xz" + get_externals_source(externals_dir=externals_dir, url=url) + # Starting with version v5.5.0, XZ-Utils removed the ability to compile + # with MSBuild. We are bringing the config.h from the last version that + # had it, 5.4.7 + config_file = target_dir / "windows" / "config.h" + config_file = target_dir / "src" / "common" / "config.h" + config_file_source = dirs.root / "_resources" / "xz" / "config.h" + if not config_file.exists(): + shutil.copy(str(config_file_source), str(config_file)) arch_to_plat = { "amd64": "x64", @@ -133,7 +146,6 @@ def build_python(env, dirs, logfp): "-p", plat, "--no-tkinter", - "-e", ] log.info("Start PCbuild") @@ -275,6 +287,7 @@ def runpip(pkg): "*.pyd", "*.dll", "*.lib", + "*.whl", "/Include/*", "/Lib/site-packages/*", ] diff --git a/relenv/common.py b/relenv/common.py index 9d526ef7..2162bf8e 100644 --- a/relenv/common.py +++ b/relenv/common.py @@ -336,12 +336,19 @@ def extract_archive(to_dir, archive): :type archive: str """ if archive.endswith("tgz"): + log.debug("Found tgz archive") + read_type = "r:gz" + elif archive.endswith("tar.gz"): + log.debug("Found tar.gz archive") read_type = "r:gz" elif archive.endswith("xz"): + log.debug("Found xz archive") read_type = "r:xz" elif archive.endswith("bz2"): + log.debug("Found bz2 archive") read_type = "r:bz2" else: + log.warning("Found unknown archive type: %s", archive) read_type = "r" with tarfile.open(archive, read_type) as t: t.extractall(to_dir) @@ -411,6 +418,7 @@ def fetch_url(url, fp, backoff=3, timeout=30): n += 1 try: fin = urllib.request.urlopen(url, timeout=timeout) + break except ( urllib.error.HTTPError, urllib.error.URLError, @@ -420,6 +428,8 @@ def fetch_url(url, fp, backoff=3, timeout=30): raise RelenvException(f"Error fetching url {url} {exc}") log.debug("Unable to connect %s", url) time.sleep(n * 10) + else: + raise RelenvException(f"Error fetching url: {url}") log.info("url opened %s", url) try: total = 0 @@ -434,7 +444,6 @@ def fetch_url(url, fp, backoff=3, timeout=30): block = fin.read(10240) finally: fin.close() - # fp.close() log.info("Download complete %s", url) @@ -514,18 +523,21 @@ def download_url(url, dest, verbose=True, backoff=3, timeout=60): """ local = get_download_location(url, dest) if verbose: - print(f"Downloading {url} -> {local}") + log.debug(f"Downloading {url} -> {local}") fout = open(local, "wb") try: fetch_url(url, fout, backoff, timeout) except Exception as exc: if verbose: - print(f"Unable to download: {url} {exc}", file=sys.stderr, flush=True) + log.error(f"Unable to download: {url} {exc}", file=sys.stderr, flush=True) try: os.unlink(local) except OSError: pass raise + finally: + fout.close() + log.debug(f"Finished downloading {url} -> {local}") return local From 87dd76b571fb28dd52cd25b11886b66306404fe5 Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 22 Oct 2025 14:13:26 -0600 Subject: [PATCH 17/18] Install looseversion & packaging on Windows --- .github/workflows/build-native-action.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/build-native-action.yml b/.github/workflows/build-native-action.yml index 2b72e6b3..3e52818c 100644 --- a/.github/workflows/build-native-action.yml +++ b/.github/workflows/build-native-action.yml @@ -298,6 +298,10 @@ jobs: run: | pip3 install nox + - name: Install Dependencies + run: | + pip3 install looseversion packaging + - name: Fetch Native Build if: ${{ matrix.arch != 'amd64' }} env: From 18256036d5048520b5bde8084a207388fd3d3efb Mon Sep 17 00:00:00 2001 From: twangboy Date: Thu, 23 Oct 2025 09:19:30 -0600 Subject: [PATCH 18/18] Fix path to ensurepip on Linux/Darwin --- .github/workflows/build-native-action.yml | 4 ---- relenv/build/common.py | 17 +++++++++-------- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build-native-action.yml b/.github/workflows/build-native-action.yml index 3e52818c..2b72e6b3 100644 --- a/.github/workflows/build-native-action.yml +++ b/.github/workflows/build-native-action.yml @@ -298,10 +298,6 @@ jobs: run: | pip3 install nox - - name: Install Dependencies - run: | - pip3 install looseversion packaging - - name: Fetch Native Build if: ${{ matrix.arch != 'amd64' }} env: diff --git a/relenv/build/common.py b/relenv/build/common.py index d16c6802..7083c375 100644 --- a/relenv/build/common.py +++ b/relenv/build/common.py @@ -38,6 +38,7 @@ runcmd, work_dirs, fetch_url, + Version, ) import relenv.relocate @@ -358,12 +359,12 @@ def build_sqlite(env, dirs, logfp): runcmd(["make", "install"], env=env, stderr=logfp, stdout=logfp) -def update_ensurepip(lib_dir): +def update_ensurepip(directory): """ Update bundled dependencies for ensurepip (pip & setuptools). """ # ensurepip bundle location - bundle_dir = lib_dir / "ensurepip" / "_bundled" + bundle_dir = directory / "ensurepip" / "_bundled" # Make sure the destination directory exists bundle_dir.mkdir(parents=True, exist_ok=True) @@ -380,7 +381,7 @@ def update_ensurepip(lib_dir): if file.name.startswith("pip-"): found_version = file.name.split("-")[1] log.debug("Found version %s", found_version) - if LooseVersion(found_version) >= LooseVersion(pip_version): + if Version(found_version) >= Version(pip_version): log.debug("Found correct pip version or newer: %s", found_version) else: file.unlink() @@ -388,7 +389,7 @@ def update_ensurepip(lib_dir): if file.name.startswith("setuptools-"): found_version = file.name.split("-")[1] log.debug("Found version %s", found_version) - if LooseVersion(found_version) >= LooseVersion(setuptools_version): + if Version(found_version) >= Version(setuptools_version): log.debug( "Found correct setuptools version or newer: %s", found_version ) @@ -397,7 +398,7 @@ def update_ensurepip(lib_dir): update_setuptools = True # Download whl files and update __init__.py - init_file = lib_dir / "ensurepip" / "__init__.py" + init_file = directory / "ensurepip" / "__init__.py" if update_pip: whl = f"pip-{pip_version}-py3-none-any.whl" whl_path = "b7/3f/945ef7ab14dc4f9d7f40288d2df998d1837ee0888ec3659c813487572faa" @@ -1542,9 +1543,6 @@ def finalize(env, dirs, logfp): # Install relenv-sysconfigdata module libdir = pathlib.Path(dirs.prefix) / "lib" - # update ensurepip - update_ensurepip(libdir) - def find_pythonlib(libdir): for root, dirs, files in os.walk(libdir): for _ in dirs: @@ -1553,6 +1551,9 @@ def find_pythonlib(libdir): pymodules = libdir / find_pythonlib(libdir) + # update ensurepip + update_ensurepip(pymodules) + cwd = os.getcwd() modname = find_sysconfigdata(pymodules) path = sys.path