diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9ebb8a3..69524e7 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -8,12 +8,12 @@ env: jobs: test-linux: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 env: OS: ubuntu strategy: matrix: - python-version: ['3.7', '3.8', '3.9', '3.10', '3.11' ] + python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12', '3.13'] services: redis: @@ -42,6 +42,10 @@ jobs: with: python-version: ${{ matrix.python-version }} + - name: Ensure setuptools, and wheel are installed + run: | + python -m pip install --upgrade pip setuptools wheel + - name: Build the package run: | python setup.py sdist @@ -64,11 +68,13 @@ jobs: cd .. echo $MODULE_PARENT echo "MODULE_PARENT=$(echo $MODULE_PARENT)" >> $GITHUB_ENV - + + - uses: docker/setup-compose-action@v1 - name: Setup docker-compose services + run: | mkdir -p /tmp/http/storage - docker-compose -f tests/assets/docker-compose.yml up -d + docker compose -f tests/assets/docker-compose.yml up -d - name: Test with pytest run: | @@ -88,14 +94,16 @@ jobs: sed -i -e "s|$(echo $MODULE_PARENT/ | tr "/" .)||g" reports/coverage-${{ matrix.python-version }}-${OS}-pydantic1.xml - name: Upload artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: reports-${{ matrix.python-version }}-${OS} + name: reports-${{ matrix.python-version }}-${{ runner.os }} path: reports/*.xml if: ${{ always() }} - name: Upload coverage results uses: codecov/codecov-action@v3 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: fail_ci_if_error: true files: reports/coverage-*.xml @@ -107,7 +115,7 @@ jobs: OS: windows strategy: matrix: - python-version: ['3.7', '3.8', '3.9', '3.10', '3.11' ] + python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12', '3.13'] steps: - uses: actions/checkout@v3 @@ -115,6 +123,9 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} + - name: Ensure setuptools, and wheel are installed + run: | + python -m pip install --upgrade pip setuptools wheel - name: Build the package run: | @@ -130,8 +141,8 @@ jobs: pytest tests -m "not redis and not ssh and not nginx and not s3" --junitxml=reports/junit-${{ matrix.python-version }}-${OS}.xml --cov-branch - name: Upload artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: reports-${{ matrix.python-version }}-${OS} + name: reports-${{ matrix.python-version }}-${{ runner.os }} path: reports/*.xml if: ${{ always() }} diff --git a/.github/workflows/version.yml b/.github/workflows/version.yml index ab9f06e..f72388a 100644 --- a/.github/workflows/version.yml +++ b/.github/workflows/version.yml @@ -7,7 +7,7 @@ env: jobs: check: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 env: OS: ubuntu diff --git a/pyproject.toml b/pyproject.toml index a338f35..5d3b168 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,6 +20,8 @@ classifiers = [ 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', + 'Programming Language :: Python :: 3.13', 'Programming Language :: Python :: 3 :: Only', ] diff --git a/requirements.txt b/requirements.txt index d2a24a8..85b598a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ numpy cloudpickle>=2.0.0,<2.3.0 -paramiko==2.* +paramiko>=3.5.0,<4.0.0 scp tqdm redis>=4.0.0 diff --git a/tarn/__version__.py b/tarn/__version__.py index ef91994..c41af0b 100644 --- a/tarn/__version__.py +++ b/tarn/__version__.py @@ -1 +1 @@ -__version__ = '0.14.0' +__version__ = '0.14.2' diff --git a/tarn/location/redis.py b/tarn/location/redis.py index e5749cc..8cd86d7 100644 --- a/tarn/location/redis.py +++ b/tarn/location/redis.py @@ -135,7 +135,9 @@ def _from_args(cls, prefix, kwargs): return cls(prefix=prefix, **kwargs) def __reduce__(self): - return self._from_args, (self.prefix, self.redis.get_connection_kwargs()) + connection_kwargs = self.redis.get_connection_kwargs() + connection_kwargs.pop('retry', None) + return self._from_args, (self.prefix, connection_kwargs) def __eq__(self, other): return isinstance(other, RedisLocation) and self.__reduce__() == other.__reduce__() diff --git a/tarn/location/s3.py b/tarn/location/s3.py index 9e8e35f..a719971 100644 --- a/tarn/location/s3.py +++ b/tarn/location/s3.py @@ -100,8 +100,15 @@ def touch(self, key: Key): def _update_labels(self, path: str, labels: MaybeLabels): if labels is not None: tags_dict = self.s3.get_tags(path) - tags_dict.update({f'_{label}': f'_{label}' for label in labels}) - self.s3.put_tags(path, tags_dict) + new_labels = {f'_{label}': f'_{label}' for label in labels} + if set(new_labels.keys()) <= set(tags_dict.keys()): + return + tags_dict.update(new_labels) + if len(tags_dict) <= 10: + self.s3.put_tags(path, tags_dict) + else: + warnings.warn(f'S3 tags are capped at 10. New labels were ignored for {self._path_to_key(path)}. ' + f'Existing labels: {self._get_labels(path)}. Labels to add: {labels}.') def _get_labels(self, path: str) -> MaybeLabels: tags_dict = self.s3.get_tags(path) diff --git a/tarn/utils.py b/tarn/utils.py index ecf98b1..8eda415 100644 --- a/tarn/utils.py +++ b/tarn/utils.py @@ -8,6 +8,7 @@ from .compat import set_path_attrs from .interface import Value + # TODO: legacy PathLike = Union[Path, str] @@ -58,12 +59,12 @@ def match_buffers(first: BinaryIO, second: BinaryIO, context: str): @contextmanager -def value_to_buffer(value: Union[Value, bytes]): +def value_to_buffer(value: Union[Value, bytes], mode: str = 'rb'): if isinstance(value, bytes): yield BytesIO(value) elif isinstance(value, (str, os.PathLike)): - with open(value, 'rb') as file: + with open(value, mode) as file: yield file else: diff --git a/tests/test_locations/test_redis.py b/tests/test_locations/test_redis.py index bd534f7..292e08d 100644 --- a/tests/test_locations/test_redis.py +++ b/tests/test_locations/test_redis.py @@ -39,5 +39,9 @@ def test_redis_pickle(redis_hostname): x = cloudpickle.loads(cloudpickle.dumps(redis)) xx = cloudpickle.loads(cloudpickle.dumps(x)) - assert x.redis.get_connection_kwargs() == xx.redis.get_connection_kwargs() + kwargs0 = x.redis.get_connection_kwargs() + kwargs1 = xx.redis.get_connection_kwargs() + kwargs0.pop('retry', None) + kwargs1.pop('retry', None) + assert kwargs0 == kwargs1 assert x.prefix == xx.prefix == b':' diff --git a/tests/test_tools/test_locker.py b/tests/test_tools/test_locker.py index 8cbd0b3..971fb9b 100644 --- a/tests/test_tools/test_locker.py +++ b/tests/test_tools/test_locker.py @@ -27,8 +27,11 @@ def test_redis_pickle(redis_hostname): locker = RedisLocker(redis_hostname, prefix='', expire=1) x = cloudpickle.loads(cloudpickle.dumps(locker)) xx = cloudpickle.loads(cloudpickle.dumps(x)) - - assert x._redis.get_connection_kwargs() == xx._redis.get_connection_kwargs() + kwargs0 = x._redis.get_connection_kwargs() + kwargs1 = xx._redis.get_connection_kwargs() + kwargs0.pop('retry', None) + kwargs1.pop('retry', None) + assert kwargs0 == kwargs1 assert x._prefix == xx._prefix == b':' assert x._expire == xx._expire == locker._expire == 1