From 2ae078911e86a38ff1acbd06417cbf1ff3e0f4c3 Mon Sep 17 00:00:00 2001 From: Max Date: Sun, 7 Apr 2024 16:38:20 +0100 Subject: [PATCH 01/19] value_to_buffer supports mode --- tarn/utils.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tarn/utils.py b/tarn/utils.py index ecf98b1..8eda415 100644 --- a/tarn/utils.py +++ b/tarn/utils.py @@ -8,6 +8,7 @@ from .compat import set_path_attrs from .interface import Value + # TODO: legacy PathLike = Union[Path, str] @@ -58,12 +59,12 @@ def match_buffers(first: BinaryIO, second: BinaryIO, context: str): @contextmanager -def value_to_buffer(value: Union[Value, bytes]): +def value_to_buffer(value: Union[Value, bytes], mode: str = 'rb'): if isinstance(value, bytes): yield BytesIO(value) elif isinstance(value, (str, os.PathLike)): - with open(value, 'rb') as file: + with open(value, mode) as file: yield file else: From cb1846e7c8f21694b41f03360dd3c94fa9986ce2 Mon Sep 17 00:00:00 2001 From: talgat Date: Tue, 13 Aug 2024 18:46:23 +0300 Subject: [PATCH 02/19] s3 tags cap fix --- tarn/__version__.py | 2 +- tarn/location/s3.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/tarn/__version__.py b/tarn/__version__.py index ef91994..092052c 100644 --- a/tarn/__version__.py +++ b/tarn/__version__.py @@ -1 +1 @@ -__version__ = '0.14.0' +__version__ = '0.14.1' diff --git a/tarn/location/s3.py b/tarn/location/s3.py index 9e8e35f..944482c 100644 --- a/tarn/location/s3.py +++ b/tarn/location/s3.py @@ -101,7 +101,10 @@ def _update_labels(self, path: str, labels: MaybeLabels): if labels is not None: tags_dict = self.s3.get_tags(path) tags_dict.update({f'_{label}': f'_{label}' for label in labels}) - self.s3.put_tags(path, tags_dict) + if len(tags_dict) <= 10: + self.s3.put_tags(path, tags_dict) + else: + warnings.warn('S3 tags are capped at 10. New labels were ignored.') def _get_labels(self, path: str) -> MaybeLabels: tags_dict = self.s3.get_tags(path) From 0466e0a1c2c9a19aafa0851bc8ae72b469dfd3ce Mon Sep 17 00:00:00 2001 From: talgat Date: Tue, 13 Aug 2024 18:56:30 +0300 Subject: [PATCH 03/19] more info in warning --- tarn/location/s3.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tarn/location/s3.py b/tarn/location/s3.py index 944482c..a692e6b 100644 --- a/tarn/location/s3.py +++ b/tarn/location/s3.py @@ -104,7 +104,8 @@ def _update_labels(self, path: str, labels: MaybeLabels): if len(tags_dict) <= 10: self.s3.put_tags(path, tags_dict) else: - warnings.warn('S3 tags are capped at 10. New labels were ignored.') + warnings.warn(f'S3 tags are capped at 10. New labels were ignored for {self._path_to_key(path)}' + f'Existing labels: {self._get_labels(path)}. Labels to add: {labels}.') def _get_labels(self, path: str) -> MaybeLabels: tags_dict = self.s3.get_tags(path) From 05b070a49dfdc9d0fd2d036837c8744d280eb12b Mon Sep 17 00:00:00 2001 From: talgat Date: Tue, 13 Aug 2024 18:57:09 +0300 Subject: [PATCH 04/19] typo --- tarn/location/s3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tarn/location/s3.py b/tarn/location/s3.py index a692e6b..dd3ecfa 100644 --- a/tarn/location/s3.py +++ b/tarn/location/s3.py @@ -104,7 +104,7 @@ def _update_labels(self, path: str, labels: MaybeLabels): if len(tags_dict) <= 10: self.s3.put_tags(path, tags_dict) else: - warnings.warn(f'S3 tags are capped at 10. New labels were ignored for {self._path_to_key(path)}' + warnings.warn(f'S3 tags are capped at 10. New labels were ignored for {self._path_to_key(path)}. ' f'Existing labels: {self._get_labels(path)}. Labels to add: {labels}.') def _get_labels(self, path: str) -> MaybeLabels: From 9de223ff1a79ccaa02e95c8fb11c85098167568c Mon Sep 17 00:00:00 2001 From: talgat Date: Sat, 29 Mar 2025 18:53:05 +0300 Subject: [PATCH 05/19] a bit less unnecessary s3 tags record --- tarn/location/s3.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tarn/location/s3.py b/tarn/location/s3.py index dd3ecfa..03bbed5 100644 --- a/tarn/location/s3.py +++ b/tarn/location/s3.py @@ -100,7 +100,10 @@ def touch(self, key: Key): def _update_labels(self, path: str, labels: MaybeLabels): if labels is not None: tags_dict = self.s3.get_tags(path) - tags_dict.update({f'_{label}': f'_{label}' for label in labels}) + new_labels = {f'_{label}': f'_{label}' for label in labels} + if set(new_labels.keys()) <= set(tags_dict.keys()): + return + tags_dict |= new_labels if len(tags_dict) <= 10: self.s3.put_tags(path, tags_dict) else: From b0ba3a1d890c9c3df88bdd7921db16750fd8d070 Mon Sep 17 00:00:00 2001 From: talgat Date: Tue, 22 Jul 2025 13:05:25 +0300 Subject: [PATCH 06/19] bumped paramiko --- requirements.txt | 2 +- tarn/__version__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index d2a24a8..85b598a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ numpy cloudpickle>=2.0.0,<2.3.0 -paramiko==2.* +paramiko>=3.5.0,<4.0.0 scp tqdm redis>=4.0.0 diff --git a/tarn/__version__.py b/tarn/__version__.py index 092052c..c41af0b 100644 --- a/tarn/__version__.py +++ b/tarn/__version__.py @@ -1 +1 @@ -__version__ = '0.14.1' +__version__ = '0.14.2' From 53b06f74bae90c7a0ab960532e604f79bf4124f5 Mon Sep 17 00:00:00 2001 From: talgat Date: Tue, 22 Jul 2025 13:09:13 +0300 Subject: [PATCH 07/19] bumped upload-artifact workflow --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9ebb8a3..0cd1706 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -88,7 +88,7 @@ jobs: sed -i -e "s|$(echo $MODULE_PARENT/ | tr "/" .)||g" reports/coverage-${{ matrix.python-version }}-${OS}-pydantic1.xml - name: Upload artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: reports-${{ matrix.python-version }}-${OS} path: reports/*.xml @@ -130,7 +130,7 @@ jobs: pytest tests -m "not redis and not ssh and not nginx and not s3" --junitxml=reports/junit-${{ matrix.python-version }}-${OS}.xml --cov-branch - name: Upload artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: reports-${{ matrix.python-version }}-${OS} path: reports/*.xml From 11097793191790edadc56e41e70494570be6bd3b Mon Sep 17 00:00:00 2001 From: talgat Date: Tue, 22 Jul 2025 14:09:09 +0300 Subject: [PATCH 08/19] bumped workflow ubuntu --- .github/workflows/tests.yml | 2 +- .github/workflows/version.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0cd1706..a86e174 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -8,7 +8,7 @@ env: jobs: test-linux: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 env: OS: ubuntu strategy: diff --git a/.github/workflows/version.yml b/.github/workflows/version.yml index ab9f06e..f72388a 100644 --- a/.github/workflows/version.yml +++ b/.github/workflows/version.yml @@ -7,7 +7,7 @@ env: jobs: check: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 env: OS: ubuntu From 44f796c538baa691f15b4bda1e7bd3bcde9fa856 Mon Sep 17 00:00:00 2001 From: talgat Date: Tue, 22 Jul 2025 14:26:03 +0300 Subject: [PATCH 09/19] python versions --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a86e174..4c68fa2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -13,7 +13,7 @@ jobs: OS: ubuntu strategy: matrix: - python-version: ['3.7', '3.8', '3.9', '3.10', '3.11' ] + python-version: ['3.10', '3.11', '3.12', '3.13'] services: redis: @@ -107,7 +107,7 @@ jobs: OS: windows strategy: matrix: - python-version: ['3.7', '3.8', '3.9', '3.10', '3.11' ] + python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12', '3.13'] steps: - uses: actions/checkout@v3 From 67f90949ca980a1978f360c3a21a016b52541b39 Mon Sep 17 00:00:00 2001 From: talgat Date: Tue, 22 Jul 2025 14:28:42 +0300 Subject: [PATCH 10/19] python versions --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 4c68fa2..2d219d6 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -13,7 +13,7 @@ jobs: OS: ubuntu strategy: matrix: - python-version: ['3.10', '3.11', '3.12', '3.13'] + python-version: ['3.10', '3.11', '3.12'] services: redis: @@ -107,7 +107,7 @@ jobs: OS: windows strategy: matrix: - python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12', '3.13'] + python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12'] steps: - uses: actions/checkout@v3 From 3f8755901811e3854e15d291b578ae62e10bca27 Mon Sep 17 00:00:00 2001 From: talgat Date: Tue, 22 Jul 2025 14:33:16 +0300 Subject: [PATCH 11/19] explicitly installing setuptools --- .github/workflows/tests.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2d219d6..3552328 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -13,7 +13,7 @@ jobs: OS: ubuntu strategy: matrix: - python-version: ['3.10', '3.11', '3.12'] + python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12', '3.13'] services: redis: @@ -42,6 +42,10 @@ jobs: with: python-version: ${{ matrix.python-version }} + - name: Ensure setuptools, and wheel are installed + run: | + python -m pip install --upgrade pip setuptools wheel + - name: Build the package run: | python setup.py sdist @@ -107,7 +111,7 @@ jobs: OS: windows strategy: matrix: - python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12'] + python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12', '3.13'] steps: - uses: actions/checkout@v3 @@ -115,6 +119,9 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} + - name: Ensure setuptools, and wheel are installed + run: | + python -m pip install --upgrade pip setuptools wheel - name: Build the package run: | From eee5d2c9005accccf18f59ed85869be5818fc6a2 Mon Sep 17 00:00:00 2001 From: talgat Date: Tue, 22 Jul 2025 15:45:07 +0300 Subject: [PATCH 12/19] adding docker compose manually --- .github/workflows/tests.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3552328..fb417a8 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -68,8 +68,10 @@ jobs: cd .. echo $MODULE_PARENT echo "MODULE_PARENT=$(echo $MODULE_PARENT)" >> $GITHUB_ENV - + + - uses: docker/setup-compose-action@v1 - name: Setup docker-compose services + run: | mkdir -p /tmp/http/storage docker-compose -f tests/assets/docker-compose.yml up -d From 466baae21023290e10f73d035c5dab55bd311749 Mon Sep 17 00:00:00 2001 From: talgat Date: Tue, 22 Jul 2025 15:48:47 +0300 Subject: [PATCH 13/19] no hyphen.. --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index fb417a8..1fdd9ab 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -74,7 +74,7 @@ jobs: run: | mkdir -p /tmp/http/storage - docker-compose -f tests/assets/docker-compose.yml up -d + docker compose -f tests/assets/docker-compose.yml up -d - name: Test with pytest run: | From 8de04fa3d926d899ada8e7428f2df22906d7f659 Mon Sep 17 00:00:00 2001 From: talgat Date: Tue, 22 Jul 2025 16:10:49 +0300 Subject: [PATCH 14/19] fixed redis connection_kwargs --- tarn/location/redis.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tarn/location/redis.py b/tarn/location/redis.py index e5749cc..8cd86d7 100644 --- a/tarn/location/redis.py +++ b/tarn/location/redis.py @@ -135,7 +135,9 @@ def _from_args(cls, prefix, kwargs): return cls(prefix=prefix, **kwargs) def __reduce__(self): - return self._from_args, (self.prefix, self.redis.get_connection_kwargs()) + connection_kwargs = self.redis.get_connection_kwargs() + connection_kwargs.pop('retry', None) + return self._from_args, (self.prefix, connection_kwargs) def __eq__(self, other): return isinstance(other, RedisLocation) and self.__reduce__() == other.__reduce__() From 44672e16b01de7c488d790a0cf839a7df6b3e44a Mon Sep 17 00:00:00 2001 From: talgat Date: Tue, 22 Jul 2025 16:19:53 +0300 Subject: [PATCH 15/19] one more try --- tests/test_locations/test_redis.py | 6 +++++- tests/test_tools/test_locker.py | 7 +++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/test_locations/test_redis.py b/tests/test_locations/test_redis.py index bd534f7..2c77a06 100644 --- a/tests/test_locations/test_redis.py +++ b/tests/test_locations/test_redis.py @@ -39,5 +39,9 @@ def test_redis_pickle(redis_hostname): x = cloudpickle.loads(cloudpickle.dumps(redis)) xx = cloudpickle.loads(cloudpickle.dumps(x)) - assert x.redis.get_connection_kwargs() == xx.redis.get_connection_kwargs() + kwargs0 = x._redis.get_connection_kwargs() + kwargs1 = xx._redis.get_connection_kwargs() + kwargs0.pop('retry', None) + kwargs1.pop('retry', None) + assert kwargs0 == kwargs1 assert x.prefix == xx.prefix == b':' diff --git a/tests/test_tools/test_locker.py b/tests/test_tools/test_locker.py index 8cbd0b3..971fb9b 100644 --- a/tests/test_tools/test_locker.py +++ b/tests/test_tools/test_locker.py @@ -27,8 +27,11 @@ def test_redis_pickle(redis_hostname): locker = RedisLocker(redis_hostname, prefix='', expire=1) x = cloudpickle.loads(cloudpickle.dumps(locker)) xx = cloudpickle.loads(cloudpickle.dumps(x)) - - assert x._redis.get_connection_kwargs() == xx._redis.get_connection_kwargs() + kwargs0 = x._redis.get_connection_kwargs() + kwargs1 = xx._redis.get_connection_kwargs() + kwargs0.pop('retry', None) + kwargs1.pop('retry', None) + assert kwargs0 == kwargs1 assert x._prefix == xx._prefix == b':' assert x._expire == xx._expire == locker._expire == 1 From 81d1ba2dd865ecd51948675b7a588a6215cdd9d8 Mon Sep 17 00:00:00 2001 From: talgat Date: Tue, 22 Jul 2025 16:25:51 +0300 Subject: [PATCH 16/19] fix --- tests/test_locations/test_redis.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_locations/test_redis.py b/tests/test_locations/test_redis.py index 2c77a06..292e08d 100644 --- a/tests/test_locations/test_redis.py +++ b/tests/test_locations/test_redis.py @@ -39,8 +39,8 @@ def test_redis_pickle(redis_hostname): x = cloudpickle.loads(cloudpickle.dumps(redis)) xx = cloudpickle.loads(cloudpickle.dumps(x)) - kwargs0 = x._redis.get_connection_kwargs() - kwargs1 = xx._redis.get_connection_kwargs() + kwargs0 = x.redis.get_connection_kwargs() + kwargs1 = xx.redis.get_connection_kwargs() kwargs0.pop('retry', None) kwargs1.pop('retry', None) assert kwargs0 == kwargs1 From 39c4a649706fcd61bea10f790729eb2a05f55de7 Mon Sep 17 00:00:00 2001 From: talgat Date: Tue, 22 Jul 2025 16:31:18 +0300 Subject: [PATCH 17/19] python 3.7-8 compatibility --- tarn/location/s3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tarn/location/s3.py b/tarn/location/s3.py index 03bbed5..a719971 100644 --- a/tarn/location/s3.py +++ b/tarn/location/s3.py @@ -103,7 +103,7 @@ def _update_labels(self, path: str, labels: MaybeLabels): new_labels = {f'_{label}': f'_{label}' for label in labels} if set(new_labels.keys()) <= set(tags_dict.keys()): return - tags_dict |= new_labels + tags_dict.update(new_labels) if len(tags_dict) <= 10: self.s3.put_tags(path, tags_dict) else: From f3ffbb118e1dcd472d4929f5995f0a77d3c30487 Mon Sep 17 00:00:00 2001 From: talgat Date: Wed, 23 Jul 2025 09:00:14 +0300 Subject: [PATCH 18/19] fixed artifact name --- .github/workflows/tests.yml | 4 ++-- pyproject.toml | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1fdd9ab..1606c8e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -96,7 +96,7 @@ jobs: - name: Upload artifacts uses: actions/upload-artifact@v4 with: - name: reports-${{ matrix.python-version }}-${OS} + name: reports-${{ matrix.python-version }}-${{ runner.os }} path: reports/*.xml if: ${{ always() }} @@ -141,6 +141,6 @@ jobs: - name: Upload artifacts uses: actions/upload-artifact@v4 with: - name: reports-${{ matrix.python-version }}-${OS} + name: reports-${{ matrix.python-version }}-${{ runner.os }} path: reports/*.xml if: ${{ always() }} diff --git a/pyproject.toml b/pyproject.toml index a338f35..5d3b168 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,6 +20,8 @@ classifiers = [ 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', + 'Programming Language :: Python :: 3.13', 'Programming Language :: Python :: 3 :: Only', ] From 9adae7f832338ca8d2cbbf081b2e37b302a6df73 Mon Sep 17 00:00:00 2001 From: talgat Date: Thu, 24 Jul 2025 08:57:46 +0300 Subject: [PATCH 19/19] added codecov token --- .github/workflows/tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1606c8e..69524e7 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -102,6 +102,8 @@ jobs: - name: Upload coverage results uses: codecov/codecov-action@v3 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: fail_ci_if_error: true files: reports/coverage-*.xml