bumped paramiko; some minor fixes

.github/workflows/tests.yml

@@ -8,12 +8,12 @@ env:
 
 jobs:
   test-linux:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     env:
       OS: ubuntu
     strategy:
       matrix:
-        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11' ]
+        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12', '3.13']
 
     services:
       redis:
@@ -42,6 +42,10 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
+      - name: Ensure setuptools, and wheel are installed
+        run: |
+          python -m pip install --upgrade pip setuptools wheel
+
       - name: Build the package
         run: |
           python setup.py sdist
@@ -64,11 +68,13 @@ jobs:
           cd ..
           echo $MODULE_PARENT
           echo "MODULE_PARENT=$(echo $MODULE_PARENT)" >> $GITHUB_ENV
-
+
+      - uses: docker/setup-compose-action@v1
       - name: Setup docker-compose services
+
         run: |
           mkdir -p /tmp/http/storage
-          docker-compose -f tests/assets/docker-compose.yml up -d
+          docker compose -f tests/assets/docker-compose.yml up -d
 
       - name: Test with pytest
         run: |
@@ -88,14 +94,16 @@ jobs:
           sed -i -e "s|$(echo $MODULE_PARENT/ | tr "/" .)||g" reports/coverage-${{ matrix.python-version }}-${OS}-pydantic1.xml
 
       - name: Upload artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: reports-${{ matrix.python-version }}-${OS}
+          name: reports-${{ matrix.python-version }}-${{ runner.os }}
           path: reports/*.xml
         if: ${{ always() }}
 
       - name: Upload coverage results
         uses: codecov/codecov-action@v3
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
         with:
           fail_ci_if_error: true
           files: reports/coverage-*.xml
@@ -107,14 +115,17 @@ jobs:
       OS: windows
     strategy:
       matrix:
-        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11' ]
+        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12', '3.13']
 
     steps:
       - uses: actions/checkout@v3
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
+      - name: Ensure setuptools, and wheel are installed
+        run: |
+          python -m pip install --upgrade pip setuptools wheel
 
       - name: Build the package
         run: |
@@ -130,8 +141,8 @@ jobs:
           pytest tests -m "not redis and not ssh and not nginx and not s3" --junitxml=reports/junit-${{ matrix.python-version }}-${OS}.xml --cov-branch
 
       - name: Upload artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: reports-${{ matrix.python-version }}-${OS}
+          name: reports-${{ matrix.python-version }}-${{ runner.os }}
           path: reports/*.xml
         if: ${{ always() }}

.github/workflows/version.yml

@@ -7,7 +7,7 @@ env:
 
 jobs:
   check:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     env:
       OS: ubuntu
 

pyproject.toml

@@ -20,6 +20,8 @@ classifiers = [
     'Programming Language :: Python :: 3.9',
     'Programming Language :: Python :: 3.10',
     'Programming Language :: Python :: 3.11',
+    'Programming Language :: Python :: 3.12',
+    'Programming Language :: Python :: 3.13',
     'Programming Language :: Python :: 3 :: Only',
 ]
 

requirements.txt

@@ -1,6 +1,6 @@
 numpy
 cloudpickle>=2.0.0,<2.3.0
-paramiko==2.*
+paramiko>=3.5.0,<4.0.0
 scp
 tqdm
 redis>=4.0.0

tarn/__version__.py

@@ -1 +1 @@
-__version__ = '0.14.0'
+__version__ = '0.14.2'

tarn/location/redis.py

@@ -135,7 +135,9 @@ def _from_args(cls, prefix, kwargs):
         return cls(prefix=prefix, **kwargs)
 
     def __reduce__(self):
-        return self._from_args, (self.prefix, self.redis.get_connection_kwargs())
+        connection_kwargs = self.redis.get_connection_kwargs()
+        connection_kwargs.pop('retry', None)
+        return self._from_args, (self.prefix, connection_kwargs)
 
     def __eq__(self, other):
         return isinstance(other, RedisLocation) and self.__reduce__() == other.__reduce__()

tarn/location/s3.py

@@ -100,8 +100,15 @@ def touch(self, key: Key):
     def _update_labels(self, path: str, labels: MaybeLabels):
         if labels is not None:
             tags_dict = self.s3.get_tags(path)
-            tags_dict.update({f'_{label}': f'_{label}' for label in labels})
-            self.s3.put_tags(path, tags_dict)
+            new_labels = {f'_{label}': f'_{label}' for label in labels}
+            if set(new_labels.keys()) <= set(tags_dict.keys()):
+                return
+            tags_dict.update(new_labels)
+            if len(tags_dict) <= 10:
+                self.s3.put_tags(path, tags_dict)
+            else:
+                warnings.warn(f'S3 tags are capped at 10. New labels were ignored for {self._path_to_key(path)}. '
+                              f'Existing labels: {self._get_labels(path)}. Labels to add: {labels}.')
 
     def _get_labels(self, path: str) -> MaybeLabels:
         tags_dict = self.s3.get_tags(path)

tarn/utils.py

@@ -8,6 +8,7 @@
 from .compat import set_path_attrs
 from .interface import Value
 
+
 # TODO: legacy
 PathLike = Union[Path, str]
 
@@ -58,12 +59,12 @@ def match_buffers(first: BinaryIO, second: BinaryIO, context: str):
 
 
 @contextmanager
-def value_to_buffer(value: Union[Value, bytes]):
+def value_to_buffer(value: Union[Value, bytes], mode: str = 'rb'):
     if isinstance(value, bytes):
         yield BytesIO(value)
 
     elif isinstance(value, (str, os.PathLike)):
-        with open(value, 'rb') as file:
+        with open(value, mode) as file:
             yield file
 
     else:

tests/test_locations/test_redis.py

@@ -39,5 +39,9 @@ def test_redis_pickle(redis_hostname):
     x = cloudpickle.loads(cloudpickle.dumps(redis))
     xx = cloudpickle.loads(cloudpickle.dumps(x))
 
-    assert x.redis.get_connection_kwargs() == xx.redis.get_connection_kwargs()
+    kwargs0 = x.redis.get_connection_kwargs()
+    kwargs1 = xx.redis.get_connection_kwargs()
+    kwargs0.pop('retry', None)
+    kwargs1.pop('retry', None)
+    assert kwargs0 == kwargs1
     assert x.prefix == xx.prefix == b':'

tests/test_tools/test_locker.py

@@ -27,8 +27,11 @@ def test_redis_pickle(redis_hostname):
     locker = RedisLocker(redis_hostname, prefix='', expire=1)
     x = cloudpickle.loads(cloudpickle.dumps(locker))
     xx = cloudpickle.loads(cloudpickle.dumps(x))
-
-    assert x._redis.get_connection_kwargs() == xx._redis.get_connection_kwargs()
+    kwargs0 = x._redis.get_connection_kwargs()
+    kwargs1 = xx._redis.get_connection_kwargs()
+    kwargs0.pop('retry', None)
+    kwargs1.pop('retry', None)
+    assert kwargs0 == kwargs1
     assert x._prefix == xx._prefix == b':'
     assert x._expire == xx._expire == locker._expire == 1