Update versions, remove tf_addons, maybe fix spatial ref in tiffs.

.github/workflows/ci.yaml

@@ -16,10 +16,10 @@ jobs:
 
     steps:
     - uses: actions/checkout@v3
-    - name: Set up Python 3.10
+    - name: Set up Python 3.11
       uses: actions/setup-python@v4
       with:
-        python-version: '3.10'
+        python-version: '3.11'
     - name: Install DELTA
       run: |
         ./scripts/setup.sh

delta/config/config.py

@@ -319,7 +319,7 @@ def load(self, yaml_file: Optional[str] = None, yaml_str: Optional[str] = None):
         if yaml_file:
             if not os.path.exists(yaml_file):
                 raise FileNotFoundError(f'Config file does not exist: {yaml_file}')
-            with open(yaml_file, 'r') as f:
+            with open(yaml_file, 'r', encoding='utf-8') as f:
                 config_data = yaml.safe_load(f)
             base_path = os.path.normpath(os.path.dirname(yaml_file))
         else:

delta/config/extensions.py

@@ -48,7 +48,6 @@ def __initialize():
     This function is called before each use of extensions to import
     the needed modules. This is only done at first use to not delay loading.
     """
-    global __extensions_to_load
     while __extensions_to_load:
         ext = __extensions_to_load.pop()
         importlib.import_module(ext)
@@ -63,7 +62,6 @@ def register_extension(name : str):
     name: str
         Name of the extension to load.
     """
-    global __extensions_to_load
     __extensions_to_load.add(name)
 
 def register_layer(layer_type : str, layer_constructor):
@@ -83,7 +81,6 @@ def register_layer(layer_type : str, layer_constructor):
     --------
     delta.ml.train.DeltaLayer : Layer wrapper with Delta extensions
     """
-    global __layers
     __layers[layer_type] = layer_constructor
 
 def register_image_reader(image_type : str, image_class):
@@ -97,7 +94,6 @@ def register_image_reader(image_type : str, image_class):
     image_class: Type[`delta.imagery.delta_image.DeltaImage`]
         A class that extends `delta.imagery.delta_image.DeltaImage`.
     """
-    global __readers
     __readers[image_type] = image_class
 
 def register_image_writer(image_type : str, writer_class):
@@ -111,7 +107,6 @@ def register_image_writer(image_type : str, writer_class):
     writer_class: Type[`delta.imagery.delta_image.DeltaImageWriter`]
         A class that extends `delta.imagery.delta_image.DeltaImageWriter`.
     """
-    global __writers
     __writers[image_type] = writer_class
 
 def register_loss(loss_type : str, custom_loss):
@@ -128,7 +123,6 @@ def register_loss(loss_type : str, custom_loss):
         Either a loss extending [Loss](https://www.tensorflow.org/api_docs/python/tf/keras/losses/Loss) or a
         function of the form loss(y_true, y_pred) which returns a tensor of the loss.
     """
-    global __losses
     __losses[loss_type] = custom_loss
 
 def register_metric(metric_type : str, custom_metric):
@@ -142,7 +136,6 @@ def register_metric(metric_type : str, custom_metric):
     custom_metric: Type[`tensorflow.keras.metrics.Metric`]
         A class extending [Metric](https://www.tensorflow.org/api_docs/python/tf/keras/metrics/Metric).
     """
-    global __metrics
     __metrics[metric_type] = custom_metric
 
 def register_callback(cb_type : str, cb):
@@ -157,7 +150,6 @@ def register_callback(cb_type : str, cb):
         A class extending [Callback](https://www.tensorflow.org/api_docs/python/tf/keras/callbacks/Callback)
         or a function that returns one.
     """
-    global __callbacks
     __callbacks[cb_type] = cb
 
 def register_preprocess(function_name : str, prep_function):
@@ -175,7 +167,6 @@ def register_preprocess(function_name : str, prep_function):
         numpy array, rectangle a `delta.imagery.rectangle.Rectangle` specifying the region covered by data,
         and bands_list is an integer list of bands loaded. The function must return a numpy array.
     """
-    global __prep_funcs
     __prep_funcs[function_name] = prep_function
 
 def register_augmentation(function_name : str, aug_function):
@@ -192,7 +183,6 @@ def register_augmentation(function_name : str, aug_function):
     aug_function:
         A function of the form aug(image, label), where image and labels are both tensors.
     """
-    global __augmentations
     __augmentations[function_name] = aug_function
 
 def layer(layer_type : str) -> str:

delta/extensions/augmentations.py

@@ -23,9 +23,7 @@
 """
 import math
 
-from packaging import version
 import tensorflow as tf
-import tensorflow_addons as tfa
 
 from delta.config.extensions import register_augmentation
 
@@ -90,46 +88,35 @@ def random_rotate(probability=0.5, max_angle=5.0):
     max_angle = max_angle * math.pi / 180.0
     def rand_rotation(image, label):
         r = tf.random.uniform(shape=[], dtype=tf.dtypes.float32)
-        theta = tf.random.uniform([], -max_angle, max_angle, tf.dtypes.float32)
-        if version.parse(tfa.__version__) < version.parse('0.12'): # fill_mode not supported
-            result = tf.cond(r > probability, lambda: (image, label),
-                             lambda: (tfa.image.rotate(image, theta),
-                                      tfa.image.rotate(label, theta)))
-        else:
-            result = tf.cond(r > probability, lambda: (image, label),
-                             lambda: (tfa.image.rotate(image, theta, fill_mode='reflect'),
-                                      tfa.image.rotate(label, theta, fill_mode='reflect')))
-        return result
+        rand = tf.random.uniform([], -max_angle, max_angle, tf.dtypes.float32)
+        theta = tf.cond(r > probability, lambda: rand, lambda: tf.constant(0.0))
+        layer = tf.keras.layers.RandomRotation((theta, theta))
+        return layer(image), layer(label)
     return rand_rotation
 
-def random_translate(probability=0.5, max_pixels=7):
+def random_translate(probability=0.5, max_factor=0.1):
     """
     Apply a random translation.
 
     Parameters
     ----------
     probability: float
         Probability to apply the transform.
-    max_pixels: int
-        If applied, the image will be rotated by a random number of pixels
-        in the range [-max_pixels, max_pixels] in both the x and y directions.
+    max_factor: int
+        If applied, the image will be translated by a random fraction
+        in the range [-max_factor, max_factor] in both the x and y directions.
 
     Returns
     -------
     Augmentation function for the specified transform.
     """
     def rand_translate(image, label):
         r = tf.random.uniform(shape=[], dtype=tf.dtypes.float32)
-        t = tf.random.uniform([2], -max_pixels, max_pixels, tf.dtypes.float32)
-        if version.parse(tfa.__version__) < version.parse('0.12'): # fill_mode not supported
-            result = tf.cond(r > probability, lambda: (image, label),
-                             lambda: (tfa.image.translate(image, t),
-                                      tfa.image.translate(label, t)))
-        else:
-            result = tf.cond(r > probability, lambda: (image, label),
-                             lambda: (tfa.image.translate(image, t, fill_mode='reflect'),
-                                      tfa.image.translate(label, t, fill_mode='reflect')))
-        return result
+        rx = tf.random.uniform([], -max_factor, max_factor, tf.dtypes.float32)
+        ry = tf.random.uniform([], -max_factor, max_factor, tf.dtypes.float32)
+        rx, ry = tf.cond(r > probability, lambda: (tf.constant(0.0), tf.constant(0.0)), lambda: (rx, ry))
+        layer = tf.keras.layers.RandomTranslation((ry, ry), (rx, rx))
+        return layer(image), layer(label)
     return rand_translate
 
 def random_brightness(probability=0.5, min_factor=0.5, max_factor=1.5):

delta/extensions/layers/gaussian_sample.py

@@ -51,7 +51,7 @@ def get_config(self):
     def callback(self):
         kl_enabled = self._kl_enabled
         class GaussianSampleCallback(Callback):
-            def on_epoch_begin(self, epoch, _=None): # pylint:disable=no-self-use
+            def on_epoch_begin(self, epoch, _=None):
                 if epoch > 0:
                     K.set_value(kl_enabled, 1.0)
         return GaussianSampleCallback()

delta/extensions/layers/pretrained.py

@@ -82,9 +82,9 @@ def pretrained(filename, encoding_layer, outputs: Optional[List[str]]=None, trai
     model = load_model(filename)
 
     if isinstance(encoding_layer, int):
-        break_point = lambda x, y: x == encoding_layer
+        break_point = lambda x, y: x == encoding_layer #pylint:disable=C3001
     elif isinstance(encoding_layer, str):
-        break_point = lambda x, y: y.name == encoding_layer
+        break_point = lambda x, y: y.name == encoding_layer #pylint:disable=C3001
 
     output_layers = _model_to_output_layers(model, break_point, trainable)
 

delta/extensions/layers/simple.py

@@ -32,10 +32,10 @@ class RepeatedGlobalAveragePooling2D(tensorflow.keras.layers.Layer):
     Takes the global average over the entire input, and repeats
     it to return a tensor the same size as the input.
     """
-    def compute_output_shape(self, input_shape): # pylint: disable=no-self-use
+    def compute_output_shape(self, input_shape):
         return input_shape
 
-    def call(self, inputs, **_): # pylint: disable=no-self-use,arguments-differ
+    def call(self, inputs, **_): # pylint: disable=arguments-differ
         ones = tf.fill(tf.shape(inputs)[:-1], 1.0)
         ones = tf.expand_dims(ones, -1)
         mean = K.mean(inputs, axis=[1, 2])

delta/extensions/losses.py

@@ -25,7 +25,6 @@
 import tensorflow.keras.losses #pylint: disable=no-name-in-module
 import tensorflow.keras.backend as K #pylint: disable=no-name-in-module
 from tensorflow.python.keras.utils import losses_utils
-import tensorflow_addons as tfa
 from scipy.ndimage import distance_transform_edt as distance
 
 from delta.config import config
@@ -38,11 +37,13 @@ def suggest_filter_size(image1, image2, power_factors, filter_size):
        input image'''
 
     cap = 2**(len(power_factors)-1)
-    if not(image1.shape[0]/cap >= filter_size and
-           image1.shape[1]/cap >= filter_size and
-           image1.shape[0]/cap >= filter_size and
-           image2.shape[1]/cap >= filter_size):
-        H = tf.math.reduce_min((image1.shape, image2.shape))
+    img1_shape = image1.shape[1:-1]
+    img2_shape = image2.shape[1:-1]
+    if not(img1_shape[0]/cap >= filter_size and
+           img1_shape[1]/cap >= filter_size and
+           img2_shape[0]/cap >= filter_size and
+           img2_shape[1]/cap >= filter_size):
+        H = tf.math.reduce_min((img1_shape, img2_shape))
         suggested_filter_size = int(H/(2**(len(power_factors)-1)))
     else:
         suggested_filter_size = filter_size
@@ -189,7 +190,7 @@ def preprocess(self, y_true, y_pred):
 
         true_convert = tf.gather(self._lookup, y_true, axis=None)
         nodata_value = config.dataset.classes.class_id('nodata')
-        nodata = (y_true == nodata_value)
+        nodata = y_true == nodata_value
 
         # ignore additional nodata classes
         for c in self._nodata_classes:
@@ -293,7 +294,6 @@ def call(self, y_true, y_pred):
 register_loss('ms_ssim_mse', ms_ssim_mse)
 register_loss('dice', dice_loss)
 register_loss('surface', surface_loss)
-register_loss('focal', tfa.losses.SigmoidFocalCrossEntropy)
 register_loss('MappedCategoricalCrossentropy', MappedCategoricalCrossentropy)
 register_loss('MappedBinaryCrossentropy', MappedBinaryCrossentropy)
 register_loss('MappedDice', MappedDiceLoss)

delta/extensions/sources/landsat.py

@@ -43,7 +43,7 @@ def _parse_mtl_file(mtl_path):
                     'REFLECTANCE_MULT', 'REFLECTANCE_ADD',
                     'K1_CONSTANT', 'K2_CONSTANT']
 
-    data = dict()
+    data = {}
     with open(mtl_path, 'r') as f:
         for line in f:
             line = line.replace('"','') # Clean up
@@ -72,7 +72,7 @@ def _parse_mtl_file(mtl_path):
                         break
 
                     if tag not in data:
-                        data[tag] = dict()
+                        data[tag] = {}
                     if tag == 'FILE_NAME':
                         data[tag][band] = value # String
                     else:

delta/extensions/sources/tiff.py

@@ -63,15 +63,15 @@ def __init__(self, path, nodata_value=None):
         self._handles = []
         for p in paths:
             if not os.path.exists(p):
-                raise Exception('Image file does not exist: ' + p)
+                raise IOError('Image file does not exist: ' + p)
             result = gdal.Open(p)
             if result is None:
-                raise Exception('Failed to open tiff file %s.' % (p))
+                raise IOError(f'Failed to open tiff file {p}.')
             self._handles.append(result)
         self._band_map = []
         for i, h in enumerate(self._handles):
             if h.RasterXSize != self._handles[0].RasterXSize or h.RasterYSize != self._handles[0].RasterYSize:
-                raise Exception('Images %s and %s have different sizes!' % (self._paths[0], self._paths[i]))
+                raise IOError(f'Images {self._paths[0]} and {self._paths[i]} have different sizes!')
             for j in range(h.RasterCount):
                 self._band_map.append((i, j + 1)) # gdal uses 1-based band indexing
 
@@ -84,7 +84,7 @@ def __init__(self, path, nodata_value=None):
     def __del__(self):
         self.close()
 
-    def _prep(self, paths): #pylint:disable=no-self-use
+    def _prep(self, paths):
         """
         Prepare the file to be opened by other tools (unpack, etc).
 
@@ -139,8 +139,7 @@ def _read(self, roi, bands, buf=None):
         else:
             s = buf[0, :, :].shape
             if s != (roi.height(), roi.width()):
-                raise IOError('Buffer shape should be (%d, %d) but is (%d, %d)!' %
-                              (roi.height(), roi.width(), s[0], s[1]))
+                raise IOError(f'Buffer shape should be ({roi.height()}, {roi.width()}) but is ({s[0]}, {s[1]})!')
         if bands:
             for i, b in enumerate(bands):
                 band_handle = self._gdal_band(b)
@@ -173,7 +172,7 @@ def dtype(self):
         dtype = self._gdal_type(0)
         if dtype in _GDAL_TO_NUMPY_TYPES:
             return _GDAL_TO_NUMPY_TYPES[dtype]
-        raise Exception('Unrecognized gdal data type: ' + str(dtype))
+        raise ValueError('Unrecognized gdal data type: ' + str(dtype))
 
     def bytes_per_pixel(self, band=0):
         """
@@ -199,9 +198,9 @@ def block_size(self):
 
     def metadata(self):
         self.__asert_open()
-        data = dict()
+        data = {}
         h = self._handles[0]
-        data['projection'] = h.GetProjection()
+        data['spatial_ref'] = h.GetSpatialRef()
         data['geotransform'] = h.GetGeoTransform()
         data['gcps'] = h.GetGCPs()
         data['gcpproj'] = h.GetGCPProjection()
@@ -213,7 +212,7 @@ def block_aligned_roi(self, desired_roi):
         self.__asert_open()
         bounds = rectangle.Rectangle(0, 0, width=self.width(), height=self.height())
         if not bounds.contains_rect(desired_roi):
-            raise Exception('desired_roi ' + str(desired_roi)
+            raise ValueError('desired_roi ' + str(desired_roi)
                             + ' is outside the bounds of image with size' + str(self.size()))
 
         block_height, block_width = self.block_size()
@@ -257,7 +256,7 @@ def save(self, path, tile_size=None, nodata_value=None, show_progress=False):
 def _numpy_dtype_to_gdal_type(dtype): #pylint: disable=R0911
     if dtype in _NUMPY_TO_GDAL_TYPES:
         return _NUMPY_TO_GDAL_TYPES[dtype]
-    raise Exception('Unrecognized numpy data type: ' + str(dtype))
+    raise ValueError('Unrecognized numpy data type: ' + str(dtype))
 
 def write_tiff(output_path: str, data: np.ndarray=None, image: delta_image.DeltaImage=None,
                nodata=None, metadata: dict=None, block_size=None, show_progress: bool=False):
@@ -353,15 +352,15 @@ def __initialize(self, path, num_bands, data_type, nodata_value, metadata):
         self._handle = driver.Create(path, ysize=self._height, xsize=self._width,
                                      bands=num_bands, eType=data_type, options=options)
         if not self._handle:
-            raise Exception('Failed to create output file: ' + path)
+            raise IOError('Failed to create output file: ' + path)
 
         if nodata_value is not None:
             for i in range(1,num_bands+1):
                 self._handle.GetRasterBand(i).SetNoDataValue(nodata_value)
 
         if metadata:
             self._handle.SetGCPs        (metadata['gcps'], metadata['gcpproj'])
-            self._handle.SetProjection  (metadata['projection'  ])
+            self._handle.SetSpatialRef  (metadata['spatial_ref' ])
             self._handle.SetGeoTransform(metadata['geotransform'])
             self._handle.SetMetadata    (metadata['metadata'    ])
             self._handle.SetSpatialRef  (metadata['spatial_ref' ])
@@ -398,7 +397,7 @@ def write_block(self, data, block_y, block_x, band=0):
         # Check that the tile position is valid
         num_tiles = self.get_num_tiles()
         if (block_x >= num_tiles[1]) or (block_y >= num_tiles[0]):
-            raise Exception('Block position ' + str((block_x, block_y))
+            raise ValueError('Block position ' + str((block_x, block_y))
                             + ' is outside the tile count: ' + str(num_tiles))
         is_edge_block = ((block_x == num_tiles[1]-1) or
                          (block_y == num_tiles[0]-1))
@@ -407,14 +406,14 @@ def write_block(self, data, block_y, block_x, band=0):
             max_x = block_x * self._tile_width  + data.shape[1]
             max_y = block_y * self._tile_height + data.shape[0]
             if max_x > self._width or max_y > self._height:
-                raise Exception('Error: Data block max position '
+                raise ValueError('Error: Data block max position '
                                 + str((max_y, max_x))
                                 + ' falls outside the image bounds: '
                                 + str((self._height, self._width)))
         else: # Shape must be exactly one tile
             if ( (data.shape[1] != self._tile_width) or
                  (data.shape[0] != self._tile_height)  ):
-                raise Exception('Error: Data block size is ' + str(data.shape)
+                raise ValueError('Error: Data block size is ' + str(data.shape)
                                 + ', output file block size is '
                                 + str((self._tile_height, self._tile_width)))
 

delta/extensions/sources/worldview.py

@@ -117,7 +117,7 @@ def _prep(self, paths):
 
         if ext == '.zip': # Need to unpack
 
-            zip_file = zipfile.ZipFile(paths, 'r')
+            zip_file = zipfile.ZipFile(paths, 'r') #pylint:disable=R1732
             tif_names = list(filter(lambda x: x.lower().endswith('.tif'), zip_file.namelist()))
             assert len(tif_names) > 0, f'Error: no tif files in the file {paths}'
             assert len(tif_names) == 1, f'Error: too many tif files in {paths}: {tif_names}'