From 68989a34eab7143ec8ea317621d1fb4b1156688f Mon Sep 17 00:00:00 2001 From: Mattias Ellert Date: Fri, 14 Mar 2025 19:59:49 +0100 Subject: [PATCH 01/21] Add a dummy non-empty environment when using assert_python_{failure,ok} If the env_vars argument to assert_python_failure or assert_python_ok is empty the test will run in isolated mode (-I) which means that the PYTHONPATH environment variable will be ignored and the test fails because the isal module can not be found, or the test is run usung the system installed version of the module instead of the newly built module that should be tested. By adding a dummy entry to the env_vars argument the isolated mode is not used and the PYTHONPATH environment variable is not ignored and the test works as expected. --- CHANGELOG.rst | 4 ++++ tests/test_gzip_compliance.py | 26 +++++++++++++++++++++----- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 805a953a..59757cb1 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -8,6 +8,10 @@ Changelog .. that users understand how the changes affect the new version. +version 1.7.3 +----------------- ++ Fix an issue where some tests failed because they ignored PYTHONPATH. + version 1.7.2 ----------------- + Use upstream ISA-L version 2.31.1 which includes patches to make diff --git a/tests/test_gzip_compliance.py b/tests/test_gzip_compliance.py index cc979317..bd11451d 100644 --- a/tests/test_gzip_compliance.py +++ b/tests/test_gzip_compliance.py @@ -845,9 +845,23 @@ def test_decompress_infile_outfile(self): self.assertTrue(os.path.exists(igzipname)) + # The following tests use assert_python_failure or assert_python_ok. + # + # If the env_vars argument to assert_python_failure or assert_python_ok + # is empty the test will run in isolated mode (-I) which means that the + # PYTHONPATH environment variable will be ignored and the test fails + # because the isal module can not be found, or the test is run usung the + # system installed version of the module instead of the newly built + # module that should be tested. + # + # By adding a dummy entry to the env_vars argument the isolated mode is + # not used and the PYTHONPATH environment variable is not ignored and + # the test works as expected. + def test_decompress_infile_outfile_error(self): rc, out, err = assert_python_failure('-m', 'isal.igzip', '-d', - 'thisisatest.out') + 'thisisatest.out', + **{'_dummy': '1'}) self.assertEqual(b"filename doesn't end in .gz: 'thisisatest.out'. " b"Cannot determine output filename.", err.strip()) @@ -872,7 +886,8 @@ def test_compress_infile_outfile_default(self): with open(local_testigzip, 'wb') as fp: fp.write(self.data) - rc, out, err = assert_python_ok('-m', 'isal.igzip', local_testigzip) + rc, out, err = assert_python_ok('-m', 'isal.igzip', local_testigzip, + **{'_dummy': '1'}) self.assertTrue(os.path.exists(igzipname)) self.assertEqual(out, b'') @@ -891,7 +906,8 @@ def test_compress_infile_outfile(self): rc, out, err = assert_python_ok('-m', 'isal.igzip', compress_level, - local_testigzip) + local_testigzip, + **{'_dummy': '1'}) self.assertTrue(os.path.exists(igzipname)) self.assertEqual(out, b'') @@ -901,7 +917,7 @@ def test_compress_infile_outfile(self): def test_compress_fast_best_are_exclusive(self): rc, out, err = assert_python_failure('-m', 'isal.igzip', '--fast', - '--best') + '--best', **{'_dummy': '1'}) self.assertIn( b"error: argument -3/--best: not allowed with argument -0/--fast", err) @@ -909,7 +925,7 @@ def test_compress_fast_best_are_exclusive(self): def test_decompress_cannot_have_flags_compression(self): rc, out, err = assert_python_failure('-m', 'isal.igzip', '--fast', - '-d') + '-d', **{'_dummy': '1'}) self.assertIn( b'error: argument -d/--decompress: not allowed with argument ' b'-0/--fast', From d08488ba37e068703ddff6168bc578d2dcdf8463 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 18 Mar 2025 11:32:50 +0100 Subject: [PATCH 02/21] Update changelog version --- CHANGELOG.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 59757cb1..f9a053b7 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -8,7 +8,7 @@ Changelog .. that users understand how the changes affect the new version. -version 1.7.3 +version 1.8.0-dev ----------------- + Fix an issue where some tests failed because they ignored PYTHONPATH. From 0215519b557f574352620a06255921d8025ecef1 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 27 May 2025 13:08:28 +0200 Subject: [PATCH 03/21] Move to setuptools-scm --- MANIFEST.in | 2 ++ pyproject.toml | 62 +++++++++++++++++++++++++++++++++++++++++++++----- setup.cfg | 2 -- setup.py | 44 +---------------------------------- 4 files changed, 59 insertions(+), 51 deletions(-) delete mode 100644 setup.cfg diff --git a/MANIFEST.in b/MANIFEST.in index 8cc2b029..23ea5299 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,6 +2,8 @@ graft src/isal/isa-l include src/isal/*.h prune tests prune docs +prune .github +exclude .git* prune benchmark_scripts exclude requirements-docs.txt exclude codecov.yml diff --git a/pyproject.toml b/pyproject.toml index bf5fe99c..425f0f9d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,10 +1,60 @@ [build-system] -requires = ["setuptools>=64", "versioningit>=1.1.0"] +requires = ["setuptools>=77", "setuptools-scm>=8"] build-backend = "setuptools.build_meta" -[tool.versioningit.vcs] -method="git" -default-tag = "v0.0.0" +[project] +name = "isal" +dynamic = ["version"] +description = """ +Faster zlib and gzip compatible compression and decompression by providing \ +python bindings for the ISA-L ibrary.""" +license="PSF-2.0" +keywords=["isal", "isa-l", "compression", "deflate", "gzip", "igzip"] +authors = [{name = "Leiden University Medical Center"}, + {email = "r.h.p.vorderman@lumc.nl"}] +readme = "README.rst" +requires-python = ">=3.8" # BadGzipFile imported +classifiers = [ + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", + "Programming Language :: C", + "Development Status :: 5 - Production/Stable", + "Topic :: System :: Archiving :: Compression", + "Operating System :: POSIX :: Linux", + "Operating System :: MacOS", + "Operating System :: Microsoft :: Windows", +] +urls.homepage = "https://github.com/pycompression/python-isal" +urls.documentation = "python-isal.readthedocs.io" -[tool.versioningit.write] -file = "src/isal/_version.py" +[tool.setuptools_scm] +version_file = "src/isal/_version.py" + +[tool.setuptools.packages.find] +where = ["src"] +include = ["isal"] + +[tool.setuptools.package-data] +isal = ['*.pyi', 'py.typed', 'isa-l/LICENSE', 'isa-l/README.md', 'isa-l/Release_notes.txt'] +[tool.setuptools.exclude-package-data] +isal = [ + "*.c", + "*.h", + "isa-l/*/*", + "isa-l/Mak*", + "isa-l/.*", + "isa-l/autogen.sh", + "isa-l/Doxyfile", + "isa-l/CONTRIBUTING.md", + "isa-l/SECURITY.md", + "isa-l/configure.ac", + "isa-l/isa-l.*", + "isa-l/libisal.pc.in", + "isa-l/make.inc", +] diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 256134e1..00000000 --- a/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[metadata] -license_files=LICENSE \ No newline at end of file diff --git a/setup.py b/setup.py index 55c89496..1ed64c2d 100644 --- a/setup.py +++ b/setup.py @@ -15,11 +15,9 @@ import tempfile from pathlib import Path -from setuptools import Extension, find_packages, setup +from setuptools import Extension, setup from setuptools.command.build_ext import build_ext -import versioningit - ISA_L_SOURCE = os.path.join("src", "isal", "isa-l") SYSTEM_IS_BSD = (sys.platform.startswith("freebsd") or @@ -138,46 +136,6 @@ def build_isa_l(): setup( - name="isal", - version=versioningit.get_version(), - description="Faster zlib and gzip compatible compression and " - "decompression by providing python bindings for the ISA-L " - "library.", - author="Leiden University Medical Center", - author_email="r.h.p.vorderman@lumc.nl", # A placeholder for now - long_description=Path("README.rst").read_text(), - long_description_content_type="text/x-rst", cmdclass={"build_ext": BuildIsalExt}, - license="PSF-2.0", - keywords="isal isa-l compression deflate gzip igzip threads", - zip_safe=False, - packages=find_packages('src'), - package_dir={'': 'src'}, - package_data={'isal': ['*.pyi', 'py.typed', - # Include isa-l LICENSE and other relevant files - # with the binary distribution. - 'isa-l/LICENSE', 'isa-l/README.md', - 'isa-l/Release_notes.txt']}, - url="https://github.com/pycompression/python-isal", - classifiers=[ - "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "Programming Language :: Python :: 3.13", - "Programming Language :: Python :: Implementation :: CPython", - "Programming Language :: Python :: Implementation :: PyPy", - "Programming Language :: C", - "Development Status :: 5 - Production/Stable", - "Topic :: System :: Archiving :: Compression", - "License :: OSI Approved :: Python Software Foundation License", - "Operating System :: POSIX :: Linux", - "Operating System :: MacOS", - "Operating System :: Microsoft :: Windows", - ], - python_requires=">=3.8", # BadGzipFile imported ext_modules=EXTENSIONS ) From 1da70a5bb2c4c3e7853064afe91b79098428bc35 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 27 May 2025 13:08:44 +0200 Subject: [PATCH 04/21] Include tests in source distribution --- MANIFEST.in | 1 - 1 file changed, 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index 23ea5299..4a48e873 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,6 +1,5 @@ graft src/isal/isa-l include src/isal/*.h -prune tests prune docs prune .github exclude .git* From 26c2b45de6b9e9d540e924e8b8acbbb4fcd847b7 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 27 May 2025 13:13:24 +0200 Subject: [PATCH 05/21] Fix CFLAGS on default behavior for later setuptools versions --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6b55ef76..17dd7f83 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -243,14 +243,14 @@ jobs: CIBW_ENVIRONMENT_LINUX: >- PYTHON_ISAL_BUILD_CACHE=True PYTHON_ISAL_BUILD_CACHE_FILE=/tmp/build_cache - CFLAGS="-g0 -DNDEBUG" + CFLAGS="-O3 -DNDEBUG" CIBW_ENVIRONMENT_WINDOWS: >- PYTHON_ISAL_BUILD_CACHE=True PYTHON_ISAL_BUILD_CACHE_FILE=${{ runner.temp }}\build_cache CIBW_ENVIRONMENT_MACOS: >- PYTHON_ISAL_BUILD_CACHE=True PYTHON_ISAL_BUILD_CACHE_FILE=${{ runner.temp }}/build_cache - CFLAGS="-g0 -DNDEBUG" + CFLAGS="-O3 -DNDEBUG" - name: Build sdist if: ${{runner.os == 'Linux' && matrix.cibw_archs_linux == 'x86_64'}} run: | From 736644860839a957fa6b47e2aea2f2727c7067ea Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 27 May 2025 13:18:59 +0200 Subject: [PATCH 06/21] Update changelog with build changes --- CHANGELOG.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index f9a053b7..b79300d2 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -10,6 +10,10 @@ Changelog version 1.8.0-dev ----------------- ++ Change build backend to setuptools-scm which is more commonly used and + supported. ++ Include test packages in the source distribution, so source distribution + installations can be verified. + Fix an issue where some tests failed because they ignored PYTHONPATH. version 1.7.2 From fc7d5a56c5d8fb072e1b7a549ecaf1c339046622 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 27 May 2025 13:31:35 +0200 Subject: [PATCH 07/21] Update supported python versions --- .github/workflows/ci.yml | 15 +++++++-------- CHANGELOG.rst | 1 + pyproject.toml | 7 ++++--- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 17dd7f83..b886a730 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,10 +16,10 @@ jobs: - uses: actions/checkout@v4 with: submodules: recursive - - name: Set up Python 3.8 + - name: Set up Python 3.9 uses: actions/setup-python@v5 with: - python-version: 3.8 + python-version: "3.9" - name: Install tox run: pip install tox - name: Lint @@ -36,10 +36,10 @@ jobs: - uses: actions/checkout@v4 with: submodules: recursive - - name: Set up Python 3.8 + - name: Set up Python 3.9 uses: actions/setup-python@v5 with: - python-version: 3.8 + python-version: "3.9" - name: Install isal run: sudo apt-get install libisal-dev - name: Install tox and upgrade setuptools and pip @@ -54,7 +54,6 @@ jobs: strategy: matrix: python-version: - - "3.8" - "3.9" - "3.10" - "3.11" @@ -65,11 +64,11 @@ jobs: os: ["ubuntu-latest"] include: - os: "macos-13" - python-version: "3.8" + python-version: "3.9" - os: "macos-14" python-version: "3.10" - os: "windows-latest" - python-version: "3.8" + python-version: "3.9" steps: - uses: actions/checkout@v4 with: @@ -105,7 +104,7 @@ jobs: strategy: matrix: python_version: - - "3.8" + - "3.9" steps: - uses: actions/checkout@v4 with: diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b79300d2..c49acba7 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -10,6 +10,7 @@ Changelog version 1.8.0-dev ----------------- ++ Python 3.8 is no longer supported. + Change build backend to setuptools-scm which is more commonly used and supported. + Include test packages in the source distribution, so source distribution diff --git a/pyproject.toml b/pyproject.toml index 425f0f9d..8405984e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,14 +13,15 @@ keywords=["isal", "isa-l", "compression", "deflate", "gzip", "igzip"] authors = [{name = "Leiden University Medical Center"}, {email = "r.h.p.vorderman@lumc.nl"}] readme = "README.rst" -requires-python = ">=3.8" # BadGzipFile imported +requires-python = ">=3.9" # Because of setuptools version classifiers = [ "Programming Language :: Python :: 3 :: Only", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Programming Language :: C", From b749782cf5253bc1dccb8cdaef32d26c2e96c6ac Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Wed, 28 May 2025 13:30:12 +0200 Subject: [PATCH 08/21] Mark extensions as free-threading-compatible and build wheels --- .github/workflows/ci.yml | 2 ++ CHANGELOG.rst | 2 ++ src/isal/_isalmodule.c | 5 +++++ src/isal/igzip_libmodule.c | 5 +++++ src/isal/isal_zlibmodule.c | 4 ++++ 5 files changed, 18 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b886a730..104b11db 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,6 +59,7 @@ jobs: - "3.11" - "3.12" - "3.13" + - "3.13t" - "pypy-3.9" - "pypy-3.10" os: ["ubuntu-latest"] @@ -215,6 +216,7 @@ jobs: run: cibuildwheel --output-dir dist env: CIBW_SKIP: "*-win32 *-manylinux_i686 cp38-macosx_*arm64 cp39-macosx_*arm64" # Skip 32 bit and problematic mac builds. + CIBW_ENABLE: "cpython-freethreading" CIBW_ARCHS_LINUX: ${{ matrix.cibw_archs_linux }} CIBW_BEFORE_ALL_LINUX: ${{ matrix.cibw_before_all_linux }} # Fully test the build wheels again. diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c49acba7..c20a0a5f 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -16,6 +16,8 @@ version 1.8.0-dev + Include test packages in the source distribution, so source distribution installations can be verified. + Fix an issue where some tests failed because they ignored PYTHONPATH. ++ Enable support for free-threading and build free-threaded wheels for + CPython 3.13. version 1.7.2 ----------------- diff --git a/src/isal/_isalmodule.c b/src/isal/_isalmodule.c index a429d3a7..55f43917 100644 --- a/src/isal/_isalmodule.c +++ b/src/isal/_isalmodule.c @@ -31,6 +31,11 @@ PyInit__isal(void) if (m == NULL) { return NULL; } + +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED); +#endif + PyModule_AddIntMacro(m, ISAL_MAJOR_VERSION); PyModule_AddIntMacro(m, ISAL_MINOR_VERSION); PyModule_AddIntMacro(m, ISAL_PATCH_VERSION); diff --git a/src/isal/igzip_libmodule.c b/src/isal/igzip_libmodule.c index de94ab01..d498145f 100644 --- a/src/isal/igzip_libmodule.c +++ b/src/isal/igzip_libmodule.c @@ -617,6 +617,11 @@ PyInit_igzip_lib(void) if (m == NULL) return NULL; +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED); +#endif + + IsalError = PyErr_NewException("igzip_lib.IsalError", NULL, NULL); if (IsalError == NULL) { return NULL; diff --git a/src/isal/isal_zlibmodule.c b/src/isal/isal_zlibmodule.c index 67547be9..3a280ec8 100644 --- a/src/isal/isal_zlibmodule.c +++ b/src/isal/isal_zlibmodule.c @@ -2183,6 +2183,10 @@ PyInit_isal_zlib(void) return NULL; } +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED); +#endif + PyObject *igzip_lib_module = PyImport_ImportModule("isal.igzip_lib"); if (igzip_lib_module == NULL) { return NULL; From 5ef868bd29559dd1647230622bae3c8224f27738 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 24 Jun 2025 11:40:33 +0200 Subject: [PATCH 09/21] Fix flush implementation --- src/isal/igzip_threaded.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/src/isal/igzip_threaded.py b/src/isal/igzip_threaded.py index 7f1c94fc..355b7f72 100644 --- a/src/isal/igzip_threaded.py +++ b/src/isal/igzip_threaded.py @@ -316,7 +316,7 @@ def write(self, b) -> int: self.input_queues[worker_index].put((data, zdict)) return len(data) - def _end_gzip_stream(self): + def flush(self): self._check_closed() # Wait for all data to be compressed for in_q in self.input_queues: @@ -324,22 +324,18 @@ def _end_gzip_stream(self): # Wait for all data to be written for out_q in self.output_queues: out_q.join() - # Write an empty deflate block with a lost block marker. + self.raw.flush() + + def close(self) -> None: + if self._closed: + return + self.flush() self.raw.write(isal_zlib.compress(b"", wbits=-15)) trailer = struct.pack(" None: - if self._closed: - return - self._end_gzip_stream() self.stop() if self.exception: self.raw.close() From 43983269f56a911e84fce09b56cfb95765f6334f Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 24 Jun 2025 14:22:54 +0200 Subject: [PATCH 10/21] Add a test for correctly flushing data to disk --- tests/test_igzip_threaded.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/tests/test_igzip_threaded.py b/tests/test_igzip_threaded.py index 41c61bfe..6331182b 100644 --- a/tests/test_igzip_threaded.py +++ b/tests/test_igzip_threaded.py @@ -12,6 +12,7 @@ import subprocess import sys import tempfile +import zlib from pathlib import Path from isal import igzip_threaded @@ -243,15 +244,28 @@ def test_threaded_program_can_exit_on_error(tmp_path, mode, threads): @pytest.mark.parametrize("threads", [1, 2]) def test_flush(tmp_path, threads): + empty_block_end = b"\x00\x00\xff\xff" + deflate_last_block = zlib.compress(b"", wbits=-15) test_file = tmp_path / "output.gz" with igzip_threaded.open(test_file, "wb", threads=threads) as f: f.write(b"1") f.flush() - assert gzip.decompress(test_file.read_bytes()) == b"1" + data = test_file.read_bytes() + assert data[-4:] == empty_block_end + # Cut off gzip header and end data with an explicit last block to + # test if the data was compressed correctly. + deflate_block = data[10:] + deflate_last_block + assert zlib.decompress(deflate_block, wbits=-15) == b"1" f.write(b"2") f.flush() - assert gzip.decompress(test_file.read_bytes()) == b"12" + data = test_file.read_bytes() + assert data[-4:] == empty_block_end + deflate_block = data[10:] + deflate_last_block + assert zlib.decompress(deflate_block, wbits=-15) == b"12" f.write(b"3") f.flush() - assert gzip.decompress(test_file.read_bytes()) == b"123" + data = test_file.read_bytes() + assert data[-4:] == empty_block_end + deflate_block = data[10:] + deflate_last_block + assert zlib.decompress(deflate_block, wbits=-15) == b"123" assert gzip.decompress(test_file.read_bytes()) == b"123" From 85e9d4853ca9d67d7b206c71428fc597b31549c6 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 24 Jun 2025 14:26:24 +0200 Subject: [PATCH 11/21] Add multithreaded flushing fix to the CHANGELOG --- CHANGELOG.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c49acba7..508abc46 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -11,6 +11,10 @@ Changelog version 1.8.0-dev ----------------- + Python 3.8 is no longer supported. ++ Fix an issue where flushing using igzip_threaded caused a gzip end of stream + and started a new gzip stream. In essence creating a concatenated gzip + stream. Now it is in concordance with how single threaded gzip streams + are flushed using Z_SYNC_FLUSH. + Change build backend to setuptools-scm which is more commonly used and supported. + Include test packages in the source distribution, so source distribution From 0fef0a9bd44689a51d0e32fd51f07708e96e3012 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 24 Jun 2025 14:31:31 +0200 Subject: [PATCH 12/21] Fix test for older versions of python --- tests/test_igzip_threaded.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_igzip_threaded.py b/tests/test_igzip_threaded.py index 6331182b..d2aee567 100644 --- a/tests/test_igzip_threaded.py +++ b/tests/test_igzip_threaded.py @@ -245,7 +245,8 @@ def test_threaded_program_can_exit_on_error(tmp_path, mode, threads): @pytest.mark.parametrize("threads", [1, 2]) def test_flush(tmp_path, threads): empty_block_end = b"\x00\x00\xff\xff" - deflate_last_block = zlib.compress(b"", wbits=-15) + compressobj = zlib.compressobj(wbits=-15) + deflate_last_block = compressobj.compress(b"") + compressobj.flush() test_file = tmp_path / "output.gz" with igzip_threaded.open(test_file, "wb", threads=threads) as f: f.write(b"1") From eb3e5c23bbea2e1d91e0d2cb50114c2522870d7f Mon Sep 17 00:00:00 2001 From: Mattias Ellert Date: Wed, 2 Jul 2025 09:49:25 +0200 Subject: [PATCH 13/21] Fix a typo in a comment --- tests/test_gzip_compliance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_gzip_compliance.py b/tests/test_gzip_compliance.py index bd11451d..5d71577c 100644 --- a/tests/test_gzip_compliance.py +++ b/tests/test_gzip_compliance.py @@ -850,7 +850,7 @@ def test_decompress_infile_outfile(self): # If the env_vars argument to assert_python_failure or assert_python_ok # is empty the test will run in isolated mode (-I) which means that the # PYTHONPATH environment variable will be ignored and the test fails - # because the isal module can not be found, or the test is run usung the + # because the isal module can not be found, or the test is run using the # system installed version of the module instead of the newly built # module that should be tested. # From ffd757b260106ae2d3a27894b0bfed98622f1267 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Mon, 8 Sep 2025 15:04:22 +0300 Subject: [PATCH 14/21] Add test for free threading that compresses and decompresses bytes --- tests/test_freethreading.py | 110 ++++++++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 tests/test_freethreading.py diff --git a/tests/test_freethreading.py b/tests/test_freethreading.py new file mode 100644 index 00000000..a338bcca --- /dev/null +++ b/tests/test_freethreading.py @@ -0,0 +1,110 @@ +import concurrent.futures +import random +import string +import threading + +import pytest + +from isal import isal_zlib, igzip_lib + + +HAMLET_SCENE = b""" +LAERTES + + O, fear me not. + I stay too long: but here my father comes. + + Enter POLONIUS + + A double blessing is a double grace, + Occasion smiles upon a second leave. + +LORD POLONIUS + + Yet here, Laertes! aboard, aboard, for shame! + The wind sits in the shoulder of your sail, + And you are stay'd for. There; my blessing with thee! + And these few precepts in thy memory + See thou character. Give thy thoughts no tongue, + Nor any unproportioned thought his act. + Be thou familiar, but by no means vulgar. + Those friends thou hast, and their adoption tried, + Grapple them to thy soul with hoops of steel; + But do not dull thy palm with entertainment + Of each new-hatch'd, unfledged comrade. Beware + Of entrance to a quarrel, but being in, + Bear't that the opposed may beware of thee. + Give every man thy ear, but few thy voice; + Take each man's censure, but reserve thy judgment. + Costly thy habit as thy purse can buy, + But not express'd in fancy; rich, not gaudy; + For the apparel oft proclaims the man, + And they in France of the best rank and station + Are of a most select and generous chief in that. + Neither a borrower nor a lender be; + For loan oft loses both itself and friend, + And borrowing dulls the edge of husbandry. + This above all: to thine ownself be true, + And it must follow, as the night the day, + Thou canst not then be false to any man. + Farewell: my blessing season this in thee! + +LAERTES + + Most humbly do I take my leave, my lord. + +LORD POLONIUS + + The time invites you; go; your servants tend. + +LAERTES + + Farewell, Ophelia; and remember well + What I have said to you. + +OPHELIA + + 'Tis in my memory lock'd, + And you yourself shall keep the key of it. + +LAERTES + + Farewell. +""" + +NUM_THREADS = 10 +NUM_ITERATIONS = 20 +NUM_JOBS = 50 # To simulate 50 jobs running in 10 threads +barrier = threading.Barrier(parties=NUM_THREADS) + + +def isal_compress_decompress(compress, decompress): + for _ in range(NUM_ITERATIONS): + barrier.wait() + x = compress(HAMLET_SCENE) + assert decompress(x) == HAMLET_SCENE + + length = len(HAMLET_SCENE) + hamlet_random = HAMLET_SCENE + b"".join( + [s.encode() for s in random.choices(string.printable, k=length)] + ) + barrier.wait() + x = compress(hamlet_random) + assert decompress(x) == hamlet_random + + +@pytest.mark.parametrize( + "compress,decompress", + [ + pytest.param(isal_zlib.compress, isal_zlib.decompress, id="zlib"), + pytest.param(igzip_lib.compress, igzip_lib.decompress, id="igzip"), + ] +) +def test_isal_compress_decompress_threaded(compress, decompress): + with concurrent.futures.ThreadPoolExecutor(NUM_THREADS) as executor: + futures = [ + executor.submit(isal_compress_decompress, compress, decompress) + for _ in range(NUM_JOBS) + ] + for future in concurrent.futures.as_completed(futures): + future.result() # To fire assertion error if there is one From a3a3a1597795a8b95a7900694976e9c1b9d4aab8 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Mon, 8 Sep 2025 15:07:30 +0300 Subject: [PATCH 15/21] Test 3.14 on CI --- .github/workflows/ci.yml | 3 ++- pyproject.toml | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 104b11db..76e58fbf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -60,6 +60,8 @@ jobs: - "3.12" - "3.13" - "3.13t" + - "3.14-dev" + - "3.14t-dev" - "pypy-3.9" - "pypy-3.10" os: ["ubuntu-latest"] @@ -216,7 +218,6 @@ jobs: run: cibuildwheel --output-dir dist env: CIBW_SKIP: "*-win32 *-manylinux_i686 cp38-macosx_*arm64 cp39-macosx_*arm64" # Skip 32 bit and problematic mac builds. - CIBW_ENABLE: "cpython-freethreading" CIBW_ARCHS_LINUX: ${{ matrix.cibw_archs_linux }} CIBW_BEFORE_ALL_LINUX: ${{ matrix.cibw_before_all_linux }} # Fully test the build wheels again. diff --git a/pyproject.toml b/pyproject.toml index 8405984e..0a95142a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,7 @@ classifiers = [ "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Programming Language :: C", From b23507830a8061e855e32bf179befcbd1a6ea653 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 9 Sep 2025 14:32:38 +0200 Subject: [PATCH 16/21] Remove python 3.9 support and add 3.14 support --- .github/workflows/ci.yml | 21 +++++++++++---------- CHANGELOG.rst | 3 ++- pyproject.toml | 2 +- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b886a730..9e95e301 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,10 +16,10 @@ jobs: - uses: actions/checkout@v4 with: submodules: recursive - - name: Set up Python 3.9 + - name: Set up Python 3.10 uses: actions/setup-python@v5 with: - python-version: "3.9" + python-version: "3.10" - name: Install tox run: pip install tox - name: Lint @@ -36,10 +36,10 @@ jobs: - uses: actions/checkout@v4 with: submodules: recursive - - name: Set up Python 3.9 + - name: Set up Python 3.10 uses: actions/setup-python@v5 with: - python-version: "3.9" + python-version: "3.10" - name: Install isal run: sudo apt-get install libisal-dev - name: Install tox and upgrade setuptools and pip @@ -54,21 +54,21 @@ jobs: strategy: matrix: python-version: - - "3.9" - "3.10" - "3.11" - "3.12" - "3.13" - - "pypy-3.9" + - "3.14" - "pypy-3.10" + - "pypy-3.11" os: ["ubuntu-latest"] include: - os: "macos-13" - python-version: "3.9" - - os: "macos-14" + python-version: "3.10" + - os: "macos-latest" python-version: "3.10" - os: "windows-latest" - python-version: "3.9" + python-version: "3.10" steps: - uses: actions/checkout@v4 with: @@ -77,6 +77,7 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + allow-prereleases: true - name: Install tox and upgrade setuptools run: pip install --upgrade tox setuptools - name: Install build dependencies (Linux) # Yasm in pypa/manylinux images. @@ -104,7 +105,7 @@ jobs: strategy: matrix: python_version: - - "3.9" + - "3.10" steps: - uses: actions/checkout@v4 with: diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 508abc46..9f430fa1 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -10,7 +10,8 @@ Changelog version 1.8.0-dev ----------------- -+ Python 3.8 is no longer supported. ++ Python 3.14 is supported. ++ Python 3.8 and 3.9 are no longer supported. + Fix an issue where flushing using igzip_threaded caused a gzip end of stream and started a new gzip stream. In essence creating a concatenated gzip stream. Now it is in concordance with how single threaded gzip streams diff --git a/pyproject.toml b/pyproject.toml index 8405984e..9fb66063 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,11 +17,11 @@ requires-python = ">=3.9" # Because of setuptools version classifiers = [ "Programming Language :: Python :: 3 :: Only", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Programming Language :: C", From 474783e98241eed2433b5194a5219ebf7a5670f7 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 9 Sep 2025 14:52:35 +0200 Subject: [PATCH 17/21] Only support 3.14 for threaded builds. --- CHANGELOG.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 5d9d1581..e0d37375 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -22,7 +22,7 @@ version 1.8.0-dev installations can be verified. + Fix an issue where some tests failed because they ignored PYTHONPATH. + Enable support for free-threading and build free-threaded wheels for - CPython 3.13. + CPython 3.14. version 1.7.2 ----------------- From 0b9574a16d443ce99de8307215e3c02aa585a16c Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 9 Sep 2025 14:58:56 +0200 Subject: [PATCH 18/21] Fix linting issues --- tests/test_freethreading.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_freethreading.py b/tests/test_freethreading.py index a338bcca..8e0ae092 100644 --- a/tests/test_freethreading.py +++ b/tests/test_freethreading.py @@ -3,9 +3,9 @@ import string import threading -import pytest +from isal import igzip_lib, isal_zlib -from isal import isal_zlib, igzip_lib +import pytest HAMLET_SCENE = b""" From 9492937925f55b7def66d434f73190bf2dcb024b Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 9 Sep 2025 15:00:48 +0200 Subject: [PATCH 19/21] Update test_freethreading.py Remove trailing whitespace --- tests/test_freethreading.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_freethreading.py b/tests/test_freethreading.py index 8e0ae092..10d1d53b 100644 --- a/tests/test_freethreading.py +++ b/tests/test_freethreading.py @@ -3,7 +3,7 @@ import string import threading -from isal import igzip_lib, isal_zlib +from isal import igzip_lib, isal_zlib import pytest From 8e9681d3674b0bd3ffcaf8cff8081edb5231f02a Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 9 Sep 2025 15:03:40 +0200 Subject: [PATCH 20/21] Credit where it's due --- CHANGELOG.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e0d37375..0cd0f65c 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -22,7 +22,7 @@ version 1.8.0-dev installations can be verified. + Fix an issue where some tests failed because they ignored PYTHONPATH. + Enable support for free-threading and build free-threaded wheels for - CPython 3.14. + CPython 3.14. Thanks to @lysnikolaou and @ngoldbaum. version 1.7.2 ----------------- From ddcc9667c2d618df31352c3021959895a2fc1db0 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Wed, 10 Sep 2025 09:13:43 +0200 Subject: [PATCH 21/21] Prepare release 1.8.0 --- CHANGELOG.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 0cd0f65c..36e20a2d 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -8,7 +8,7 @@ Changelog .. that users understand how the changes affect the new version. -version 1.8.0-dev +version 1.8.0 ----------------- + Python 3.14 is supported. + Python 3.8 and 3.9 are no longer supported.