From de85f8ac963e9054b83e53d62f40fbd7eb67becd Mon Sep 17 00:00:00 2001 From: Brian Coltin Date: Tue, 11 Jul 2023 14:51:38 -0700 Subject: [PATCH 1/5] Update versions, remove tf_addons, maybe fix spatial ref in tiffs. --- delta/extensions/augmentations.py | 39 +++++++++++-------------------- delta/extensions/losses.py | 16 ++++++------- delta/extensions/sources/tiff.py | 31 ++++++++++++------------ scripts/linter/pylintrc | 31 ++---------------------- setup.py | 7 +++--- tests/test_config.py | 30 ++++++++++++------------ tests/test_tiff.py | 4 ++-- 7 files changed, 58 insertions(+), 100 deletions(-) diff --git a/delta/extensions/augmentations.py b/delta/extensions/augmentations.py index d908e8fc..03b91257 100644 --- a/delta/extensions/augmentations.py +++ b/delta/extensions/augmentations.py @@ -23,9 +23,7 @@ """ import math -from packaging import version import tensorflow as tf -import tensorflow_addons as tfa from delta.config.extensions import register_augmentation @@ -90,19 +88,13 @@ def random_rotate(probability=0.5, max_angle=5.0): max_angle = max_angle * math.pi / 180.0 def rand_rotation(image, label): r = tf.random.uniform(shape=[], dtype=tf.dtypes.float32) - theta = tf.random.uniform([], -max_angle, max_angle, tf.dtypes.float32) - if version.parse(tfa.__version__) < version.parse('0.12'): # fill_mode not supported - result = tf.cond(r > probability, lambda: (image, label), - lambda: (tfa.image.rotate(image, theta), - tfa.image.rotate(label, theta))) - else: - result = tf.cond(r > probability, lambda: (image, label), - lambda: (tfa.image.rotate(image, theta, fill_mode='reflect'), - tfa.image.rotate(label, theta, fill_mode='reflect'))) - return result + rand = tf.random.uniform([], -max_angle, max_angle, tf.dtypes.float32) + theta = tf.cond(r > probability, lambda: rand, lambda: tf.constant(0.0)) + layer = tf.keras.layers.RandomRotation((theta, theta)) + return layer(image), layer(label) return rand_rotation -def random_translate(probability=0.5, max_pixels=7): +def random_translate(probability=0.5, max_factor=0.1): """ Apply a random translation. @@ -110,9 +102,9 @@ def random_translate(probability=0.5, max_pixels=7): ---------- probability: float Probability to apply the transform. - max_pixels: int - If applied, the image will be rotated by a random number of pixels - in the range [-max_pixels, max_pixels] in both the x and y directions. + max_factor: int + If applied, the image will be translated by a random fraction + in the range [-max_factor, max_factor] in both the x and y directions. Returns ------- @@ -120,16 +112,11 @@ def random_translate(probability=0.5, max_pixels=7): """ def rand_translate(image, label): r = tf.random.uniform(shape=[], dtype=tf.dtypes.float32) - t = tf.random.uniform([2], -max_pixels, max_pixels, tf.dtypes.float32) - if version.parse(tfa.__version__) < version.parse('0.12'): # fill_mode not supported - result = tf.cond(r > probability, lambda: (image, label), - lambda: (tfa.image.translate(image, t), - tfa.image.translate(label, t))) - else: - result = tf.cond(r > probability, lambda: (image, label), - lambda: (tfa.image.translate(image, t, fill_mode='reflect'), - tfa.image.translate(label, t, fill_mode='reflect'))) - return result + rx = tf.random.uniform([], -max_factor, max_factor, tf.dtypes.float32) + ry = tf.random.uniform([], -max_factor, max_factor, tf.dtypes.float32) + rx, ry = tf.cond(r > probability, lambda: (tf.constant(0.0), tf.constant(0.0)), lambda: (rx, ry)) + layer = tf.keras.layers.RandomTranslation((ry, ry), (rx, rx)) + return layer(image), layer(label) return rand_translate def random_brightness(probability=0.5, min_factor=0.5, max_factor=1.5): diff --git a/delta/extensions/losses.py b/delta/extensions/losses.py index 27bb91a6..f00ed2a1 100644 --- a/delta/extensions/losses.py +++ b/delta/extensions/losses.py @@ -25,7 +25,6 @@ import tensorflow.keras.losses #pylint: disable=no-name-in-module import tensorflow.keras.backend as K #pylint: disable=no-name-in-module from tensorflow.python.keras.utils import losses_utils -import tensorflow_addons as tfa from scipy.ndimage import distance_transform_edt as distance from delta.config import config @@ -38,11 +37,13 @@ def suggest_filter_size(image1, image2, power_factors, filter_size): input image''' cap = 2**(len(power_factors)-1) - if not(image1.shape[0]/cap >= filter_size and - image1.shape[1]/cap >= filter_size and - image1.shape[0]/cap >= filter_size and - image2.shape[1]/cap >= filter_size): - H = tf.math.reduce_min((image1.shape, image2.shape)) + img1_shape = image1.shape[1:-1] + img2_shape = image2.shape[1:-1] + if not(img1_shape[0]/cap >= filter_size and + img1_shape[1]/cap >= filter_size and + img2_shape[0]/cap >= filter_size and + img2_shape[1]/cap >= filter_size): + H = tf.math.reduce_min((img1_shape, img2_shape)) suggested_filter_size = int(H/(2**(len(power_factors)-1))) else: suggested_filter_size = filter_size @@ -189,7 +190,7 @@ def preprocess(self, y_true, y_pred): true_convert = tf.gather(self._lookup, y_true, axis=None) nodata_value = config.dataset.classes.class_id('nodata') - nodata = (y_true == nodata_value) + nodata = y_true == nodata_value # ignore additional nodata classes for c in self._nodata_classes: @@ -293,7 +294,6 @@ def call(self, y_true, y_pred): register_loss('ms_ssim_mse', ms_ssim_mse) register_loss('dice', dice_loss) register_loss('surface', surface_loss) -register_loss('focal', tfa.losses.SigmoidFocalCrossEntropy) register_loss('MappedCategoricalCrossentropy', MappedCategoricalCrossentropy) register_loss('MappedBinaryCrossentropy', MappedBinaryCrossentropy) register_loss('MappedDice', MappedDiceLoss) diff --git a/delta/extensions/sources/tiff.py b/delta/extensions/sources/tiff.py index 19b6b850..d9047117 100755 --- a/delta/extensions/sources/tiff.py +++ b/delta/extensions/sources/tiff.py @@ -63,15 +63,15 @@ def __init__(self, path, nodata_value=None): self._handles = [] for p in paths: if not os.path.exists(p): - raise Exception('Image file does not exist: ' + p) + raise IOError('Image file does not exist: ' + p) result = gdal.Open(p) if result is None: - raise Exception('Failed to open tiff file %s.' % (p)) + raise IOError(f'Failed to open tiff file {p}.') self._handles.append(result) self._band_map = [] for i, h in enumerate(self._handles): if h.RasterXSize != self._handles[0].RasterXSize or h.RasterYSize != self._handles[0].RasterYSize: - raise Exception('Images %s and %s have different sizes!' % (self._paths[0], self._paths[i])) + raise IOError(f'Images {self._paths[0]} and {self._paths[i]} have different sizes!') for j in range(h.RasterCount): self._band_map.append((i, j + 1)) # gdal uses 1-based band indexing @@ -84,7 +84,7 @@ def __init__(self, path, nodata_value=None): def __del__(self): self.close() - def _prep(self, paths): #pylint:disable=no-self-use + def _prep(self, paths): """ Prepare the file to be opened by other tools (unpack, etc). @@ -139,8 +139,7 @@ def _read(self, roi, bands, buf=None): else: s = buf[0, :, :].shape if s != (roi.height(), roi.width()): - raise IOError('Buffer shape should be (%d, %d) but is (%d, %d)!' % - (roi.height(), roi.width(), s[0], s[1])) + raise IOError(f'Buffer shape should be ({roi.height()}, {roi.width()}) but is ({s[0]}, {s[1]})!') if bands: for i, b in enumerate(bands): band_handle = self._gdal_band(b) @@ -173,7 +172,7 @@ def dtype(self): dtype = self._gdal_type(0) if dtype in _GDAL_TO_NUMPY_TYPES: return _GDAL_TO_NUMPY_TYPES[dtype] - raise Exception('Unrecognized gdal data type: ' + str(dtype)) + raise ValueError('Unrecognized gdal data type: ' + str(dtype)) def bytes_per_pixel(self, band=0): """ @@ -199,9 +198,9 @@ def block_size(self): def metadata(self): self.__asert_open() - data = dict() + data = {} h = self._handles[0] - data['projection'] = h.GetProjection() + data['spatial_ref'] = h.GetSpatialRef() data['geotransform'] = h.GetGeoTransform() data['gcps'] = h.GetGCPs() data['gcpproj'] = h.GetGCPProjection() @@ -213,7 +212,7 @@ def block_aligned_roi(self, desired_roi): self.__asert_open() bounds = rectangle.Rectangle(0, 0, width=self.width(), height=self.height()) if not bounds.contains_rect(desired_roi): - raise Exception('desired_roi ' + str(desired_roi) + raise ValueError('desired_roi ' + str(desired_roi) + ' is outside the bounds of image with size' + str(self.size())) block_height, block_width = self.block_size() @@ -257,7 +256,7 @@ def save(self, path, tile_size=None, nodata_value=None, show_progress=False): def _numpy_dtype_to_gdal_type(dtype): #pylint: disable=R0911 if dtype in _NUMPY_TO_GDAL_TYPES: return _NUMPY_TO_GDAL_TYPES[dtype] - raise Exception('Unrecognized numpy data type: ' + str(dtype)) + raise ValueError('Unrecognized numpy data type: ' + str(dtype)) def write_tiff(output_path: str, data: np.ndarray=None, image: delta_image.DeltaImage=None, nodata=None, metadata: dict=None, block_size=None, show_progress: bool=False): @@ -353,7 +352,7 @@ def __initialize(self, path, num_bands, data_type, nodata_value, metadata): self._handle = driver.Create(path, ysize=self._height, xsize=self._width, bands=num_bands, eType=data_type, options=options) if not self._handle: - raise Exception('Failed to create output file: ' + path) + raise IOError('Failed to create output file: ' + path) if nodata_value is not None: for i in range(1,num_bands+1): @@ -361,7 +360,7 @@ def __initialize(self, path, num_bands, data_type, nodata_value, metadata): if metadata: self._handle.SetGCPs (metadata['gcps'], metadata['gcpproj']) - self._handle.SetProjection (metadata['projection' ]) + self._handle.SetSpatialRef (metadata['spatial_ref' ]) self._handle.SetGeoTransform(metadata['geotransform']) self._handle.SetMetadata (metadata['metadata' ]) self._handle.SetSpatialRef (metadata['spatial_ref' ]) @@ -398,7 +397,7 @@ def write_block(self, data, block_y, block_x, band=0): # Check that the tile position is valid num_tiles = self.get_num_tiles() if (block_x >= num_tiles[1]) or (block_y >= num_tiles[0]): - raise Exception('Block position ' + str((block_x, block_y)) + raise ValueError('Block position ' + str((block_x, block_y)) + ' is outside the tile count: ' + str(num_tiles)) is_edge_block = ((block_x == num_tiles[1]-1) or (block_y == num_tiles[0]-1)) @@ -407,14 +406,14 @@ def write_block(self, data, block_y, block_x, band=0): max_x = block_x * self._tile_width + data.shape[1] max_y = block_y * self._tile_height + data.shape[0] if max_x > self._width or max_y > self._height: - raise Exception('Error: Data block max position ' + raise ValueError('Error: Data block max position ' + str((max_y, max_x)) + ' falls outside the image bounds: ' + str((self._height, self._width))) else: # Shape must be exactly one tile if ( (data.shape[1] != self._tile_width) or (data.shape[0] != self._tile_height) ): - raise Exception('Error: Data block size is ' + str(data.shape) + raise ValueError('Error: Data block size is ' + str(data.shape) + ', output file block size is ' + str((self._tile_height, self._tile_width))) diff --git a/scripts/linter/pylintrc b/scripts/linter/pylintrc index 6501cc16..2096a13f 100644 --- a/scripts/linter/pylintrc +++ b/scripts/linter/pylintrc @@ -8,9 +8,6 @@ # Manually added PYTHONPATH to sys.path because there's a bug with pylint and virtualenvs correctly inheriting PYTHONPATH init-hook='import sys; import os; sys.path.extend([os.path.abspath("."),*os.environ["PYTHONPATH"].split(":")])' -# Profiled execution. -profile=no - # Add files or directories to the blacklist. They should be base names, not # paths. ignore=CVS,.git @@ -39,8 +36,7 @@ load-plugins=delta_lint # --enable=similarities". If you want to run only the classes checker, but have # no Warning level messages displayed, use"--disable=all --enable=classes # --disable=W" -# would rather only disable part of C0326... -disable=C0103,C0111,W0511,C0321,C0326,similarities +disable=C0103,C0111,W0511,C0321,similarities [REPORTS] @@ -50,11 +46,6 @@ disable=C0103,C0111,W0511,C0321,C0326,similarities # mypackage.mymodule.MyReporterClass. output-format=colorized -# Put messages in a separate file for each module / package specified on the -# command line instead of printing them on stdout. Reports (if any) will be -# written in a file name "pylint_global.[txt|html]". -files-output=no - # Tells whether to display a full report or only the messages reports=no @@ -65,10 +56,6 @@ reports=no # (RP0004). evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) -# Add a comment according to your evaluation note. This is used by the global -# evaluation report (RP0004). -comment=no - # Template used to display messages. This is a python new-style format string # used to format the massage information. See doc for all details #msg-template= @@ -118,10 +105,6 @@ ignore-mixin-members=yes ignored-classes=SQLObject,pytest ignored-modules=tensorflow -# When zope mode is activated, add a predefined set of Zope acquired attributes -# to generated-members. -zope=no - # List of members which are set dynamically and missed by pylint inference # system, and so shouldn't trigger E0201 when accessed. Python regular # expressions are accepted. @@ -144,12 +127,6 @@ additional-builtins= [BASIC] -# Required attributes for module, separated by a comma -required-attributes= - -# List of builtins function names that should not be used, separated by a comma -bad-functions=filter,apply,input - # Regular expression which should only match correct module names module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ @@ -205,10 +182,6 @@ notes=FIXME,XXX,TODO [CLASSES] -# List of interface methods to ignore, separated by a comma. This is used for -# instance to not check methods defines in Zope's Interface base class. -ignore-iface-methods=isImplementedBy,deferred,extends,names,namesAndDescriptions,queryDescriptionFor,getBases,getDescriptionFor,getDoc,getName,getTaggedValue,getTaggedValueTags,isEqualOrExtendedBy,setTaggedValue,isImplementedByInstancesOf,adaptWith,is_implemented_by - # List of method names used to declare (i.e. assign) instance attributes. defining-attr-methods=__init__,__new__,setUp @@ -275,4 +248,4 @@ max-public-methods=20 # Exceptions that will emit a warning when being caught. Defaults to # "Exception" -overgeneral-exceptions=Exception +overgeneral-exceptions=exceptions.Exception diff --git a/setup.py b/setup.py index 8e595456..7dc34cac 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ # so we can list dependencies in docker if os.path.exists('README.md'): - with open("README.md", "r") as f: + with open("README.md", "r", encoding="utf-8") as f: long_description = f.read() else: long_description = 'DELTA' @@ -35,7 +35,7 @@ setuptools.setup( name="delta", - version="1.0.1", + version="1.0.2", author="NASA Ames", author_email="todo@todo", description="Deep learning for satellite imagery", @@ -52,14 +52,13 @@ ], install_requires=[ 'tensorflow>=2.1', - 'tensorflow_addons', 'usgs<0.3', 'scipy', 'matplotlib', 'mlflow', 'portalocker', 'appdirs', - 'gdal', + 'gdal>=3.0', 'shapely', 'pillow' #'numpy', # these are included by tensorflow with restrictions diff --git a/tests/test_config.py b/tests/test_config.py index df0bb8ff..67070702 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -58,14 +58,14 @@ def test_general(): def test_images_dir(): config_reset() dir_path = os.path.join(os.path.dirname(__file__), 'data') - test_str = ''' + test_str = f''' dataset: images: type: tiff preprocess: ~ - directory: %s/ + directory: {dir_path}/ extension: .tiff - ''' % (dir_path) + ''' config.load(yaml_str=test_str) im = config.dataset.images() assert im.type() == 'tiff' @@ -95,13 +95,13 @@ def test_preprocess(): def test_images_files(): config_reset() file_path = os.path.join(os.path.dirname(__file__), 'data', 'landsat.tiff') - test_str = ''' + test_str = f''' dataset: images: type: tiff preprocess: ~ - files: [%s] - ''' % (file_path) + files: [{file_path}] + ''' config.load(yaml_str=test_str) im = config.dataset.images() assert im.type() == 'tiff' @@ -227,14 +227,14 @@ def test_pretrained_layer(): tf.keras.models.save_model(m1, tmp_filename) - pretrained_model = ''' + pretrained_model = f''' params: v1 : 10 layers: - Input: shape: in_shape - Pretrained: - filename: %s + filename: {tmp_filename} encoding_layer: encoding - Dense: units: 100 @@ -242,7 +242,7 @@ def test_pretrained_layer(): - Dense: units: out_shape activation: softmax - ''' % tmp_filename + ''' m2 = config_parser.model_from_dict(yaml.safe_load(pretrained_model), params_exposed)() m2.compile(optimizer='adam', loss='mse') assert len(m2.layers[1].layers) == 3 @@ -252,14 +252,14 @@ def test_pretrained_layer(): break # test using internal layer of pretrained as input - pretrained_model = ''' + pretrained_model = f''' params: v1 : 10 layers: - Input: shape: in_shape - Pretrained: - filename: %s + filename: {tmp_filename} encoding_layer: encoding name: pretrained outputs: [encoding] @@ -270,7 +270,7 @@ def test_pretrained_layer(): - Dense: units: out_shape activation: softmax - ''' % tmp_filename + ''' m2 = config_parser.model_from_dict(yaml.safe_load(pretrained_model), params_exposed)() m2.compile(optimizer='adam', loss='mse') assert len(m2.layers[1].layers) == (len(m1.layers) - 1) # also don't take the input layer @@ -445,7 +445,7 @@ def test_augmentations(): max_angle: 0.5 - random_translate: probability: 1.0 - max_pixels: 10 + max_factor: 0.1 ''' config.load(yaml_str=test_str) aug = config_parser.config_augmentation() @@ -498,8 +498,8 @@ def test_argparser(): config.setup_arg_parser(parser) file_path = os.path.join(os.path.dirname(__file__), 'data', 'landsat.tiff') - options = parser.parse_args(('--image-type tiff --image %s' % (file_path) + - ' --label-type tiff --label %s' % (file_path)).split()) + options = parser.parse_args((f'--image-type tiff --image {file_path}' + + f' --label-type tiff --label {file_path}').split()) config.parse_args(options) im = config.dataset.images() diff --git a/tests/test_tiff.py b/tests/test_tiff.py index 737a6f63..1a6f945a 100644 --- a/tests/test_tiff.py +++ b/tests/test_tiff.py @@ -45,7 +45,7 @@ def check_landsat_tiff(filename): assert geo[5] == pytest.approx(0.0, abs=0.01) assert 'gcps' in meta assert 'gcpproj' in meta - assert 'projection' in meta + assert 'spatial_ref' in meta assert 'metadata' in meta r = rectangle.Rectangle(0, 0, width=input_reader.size()[1], @@ -72,7 +72,7 @@ def check_same(filename1, filename2, data_only=False): assert m_1['geotransform'] == m_2['geotransform'] assert m_1['gcps'] == m_2['gcps'] assert m_1['gcpproj'] == m_2['gcpproj'] - assert m_1['projection'] == m_2['projection'] + assert m_1['spatial_ref'].IsSameGeogCS(m_2['spatial_ref']) assert m_1['metadata'] == m_2['metadata'] d1 = in1.read() From 9fca82517b9fb1f60fd17b66d588102b28f7147e Mon Sep 17 00:00:00 2001 From: Brian Coltin Date: Tue, 11 Jul 2023 15:16:01 -0700 Subject: [PATCH 2/5] Fix lint. --- delta/config/config.py | 2 +- delta/config/extensions.py | 10 ---------- delta/extensions/layers/gaussian_sample.py | 2 +- delta/extensions/layers/pretrained.py | 4 ++-- delta/extensions/layers/simple.py | 4 ++-- delta/extensions/sources/landsat.py | 4 ++-- delta/extensions/sources/worldview.py | 2 +- delta/imagery/delta_image.py | 6 +++--- delta/imagery/imagery_config.py | 4 ++-- delta/ml/config_parser.py | 8 ++++---- delta/ml/ml_config.py | 2 +- delta/ml/train.py | 2 +- delta/subcommands/classify.py | 2 +- delta/subcommands/visualize.py | 2 +- scripts/convert/pbs_functions.py | 2 +- scripts/fetch/get_landsat_dswe_labels.py | 2 +- scripts/fetch/random_folder_split.py | 2 +- scripts/linter/delta_lint.py | 2 +- scripts/linter/pylintrc | 2 +- 19 files changed, 27 insertions(+), 37 deletions(-) diff --git a/delta/config/config.py b/delta/config/config.py index 50513357..18029b18 100644 --- a/delta/config/config.py +++ b/delta/config/config.py @@ -319,7 +319,7 @@ def load(self, yaml_file: Optional[str] = None, yaml_str: Optional[str] = None): if yaml_file: if not os.path.exists(yaml_file): raise FileNotFoundError(f'Config file does not exist: {yaml_file}') - with open(yaml_file, 'r') as f: + with open(yaml_file, 'r', encoding='utf-8') as f: config_data = yaml.safe_load(f) base_path = os.path.normpath(os.path.dirname(yaml_file)) else: diff --git a/delta/config/extensions.py b/delta/config/extensions.py index 8d6fae15..4d53c4a6 100644 --- a/delta/config/extensions.py +++ b/delta/config/extensions.py @@ -48,7 +48,6 @@ def __initialize(): This function is called before each use of extensions to import the needed modules. This is only done at first use to not delay loading. """ - global __extensions_to_load while __extensions_to_load: ext = __extensions_to_load.pop() importlib.import_module(ext) @@ -63,7 +62,6 @@ def register_extension(name : str): name: str Name of the extension to load. """ - global __extensions_to_load __extensions_to_load.add(name) def register_layer(layer_type : str, layer_constructor): @@ -83,7 +81,6 @@ def register_layer(layer_type : str, layer_constructor): -------- delta.ml.train.DeltaLayer : Layer wrapper with Delta extensions """ - global __layers __layers[layer_type] = layer_constructor def register_image_reader(image_type : str, image_class): @@ -97,7 +94,6 @@ def register_image_reader(image_type : str, image_class): image_class: Type[`delta.imagery.delta_image.DeltaImage`] A class that extends `delta.imagery.delta_image.DeltaImage`. """ - global __readers __readers[image_type] = image_class def register_image_writer(image_type : str, writer_class): @@ -111,7 +107,6 @@ def register_image_writer(image_type : str, writer_class): writer_class: Type[`delta.imagery.delta_image.DeltaImageWriter`] A class that extends `delta.imagery.delta_image.DeltaImageWriter`. """ - global __writers __writers[image_type] = writer_class def register_loss(loss_type : str, custom_loss): @@ -128,7 +123,6 @@ def register_loss(loss_type : str, custom_loss): Either a loss extending [Loss](https://www.tensorflow.org/api_docs/python/tf/keras/losses/Loss) or a function of the form loss(y_true, y_pred) which returns a tensor of the loss. """ - global __losses __losses[loss_type] = custom_loss def register_metric(metric_type : str, custom_metric): @@ -142,7 +136,6 @@ def register_metric(metric_type : str, custom_metric): custom_metric: Type[`tensorflow.keras.metrics.Metric`] A class extending [Metric](https://www.tensorflow.org/api_docs/python/tf/keras/metrics/Metric). """ - global __metrics __metrics[metric_type] = custom_metric def register_callback(cb_type : str, cb): @@ -157,7 +150,6 @@ def register_callback(cb_type : str, cb): A class extending [Callback](https://www.tensorflow.org/api_docs/python/tf/keras/callbacks/Callback) or a function that returns one. """ - global __callbacks __callbacks[cb_type] = cb def register_preprocess(function_name : str, prep_function): @@ -175,7 +167,6 @@ def register_preprocess(function_name : str, prep_function): numpy array, rectangle a `delta.imagery.rectangle.Rectangle` specifying the region covered by data, and bands_list is an integer list of bands loaded. The function must return a numpy array. """ - global __prep_funcs __prep_funcs[function_name] = prep_function def register_augmentation(function_name : str, aug_function): @@ -192,7 +183,6 @@ def register_augmentation(function_name : str, aug_function): aug_function: A function of the form aug(image, label), where image and labels are both tensors. """ - global __augmentations __augmentations[function_name] = aug_function def layer(layer_type : str) -> str: diff --git a/delta/extensions/layers/gaussian_sample.py b/delta/extensions/layers/gaussian_sample.py index 8311ea29..6312b882 100644 --- a/delta/extensions/layers/gaussian_sample.py +++ b/delta/extensions/layers/gaussian_sample.py @@ -51,7 +51,7 @@ def get_config(self): def callback(self): kl_enabled = self._kl_enabled class GaussianSampleCallback(Callback): - def on_epoch_begin(self, epoch, _=None): # pylint:disable=no-self-use + def on_epoch_begin(self, epoch, _=None): if epoch > 0: K.set_value(kl_enabled, 1.0) return GaussianSampleCallback() diff --git a/delta/extensions/layers/pretrained.py b/delta/extensions/layers/pretrained.py index 2d78596f..16d5bf32 100644 --- a/delta/extensions/layers/pretrained.py +++ b/delta/extensions/layers/pretrained.py @@ -82,9 +82,9 @@ def pretrained(filename, encoding_layer, outputs: Optional[List[str]]=None, trai model = load_model(filename) if isinstance(encoding_layer, int): - break_point = lambda x, y: x == encoding_layer + break_point = lambda x, y: x == encoding_layer #pylint:disable=C3001 elif isinstance(encoding_layer, str): - break_point = lambda x, y: y.name == encoding_layer + break_point = lambda x, y: y.name == encoding_layer #pylint:disable=C3001 output_layers = _model_to_output_layers(model, break_point, trainable) diff --git a/delta/extensions/layers/simple.py b/delta/extensions/layers/simple.py index b12765a8..88ead5ed 100644 --- a/delta/extensions/layers/simple.py +++ b/delta/extensions/layers/simple.py @@ -32,10 +32,10 @@ class RepeatedGlobalAveragePooling2D(tensorflow.keras.layers.Layer): Takes the global average over the entire input, and repeats it to return a tensor the same size as the input. """ - def compute_output_shape(self, input_shape): # pylint: disable=no-self-use + def compute_output_shape(self, input_shape): return input_shape - def call(self, inputs, **_): # pylint: disable=no-self-use,arguments-differ + def call(self, inputs, **_): # pylint: disable=arguments-differ ones = tf.fill(tf.shape(inputs)[:-1], 1.0) ones = tf.expand_dims(ones, -1) mean = K.mean(inputs, axis=[1, 2]) diff --git a/delta/extensions/sources/landsat.py b/delta/extensions/sources/landsat.py index 78a4707a..9488058e 100644 --- a/delta/extensions/sources/landsat.py +++ b/delta/extensions/sources/landsat.py @@ -43,7 +43,7 @@ def _parse_mtl_file(mtl_path): 'REFLECTANCE_MULT', 'REFLECTANCE_ADD', 'K1_CONSTANT', 'K2_CONSTANT'] - data = dict() + data = {} with open(mtl_path, 'r') as f: for line in f: line = line.replace('"','') # Clean up @@ -72,7 +72,7 @@ def _parse_mtl_file(mtl_path): break if tag not in data: - data[tag] = dict() + data[tag] = {} if tag == 'FILE_NAME': data[tag][band] = value # String else: diff --git a/delta/extensions/sources/worldview.py b/delta/extensions/sources/worldview.py index 8b5771ce..3a3a10f6 100644 --- a/delta/extensions/sources/worldview.py +++ b/delta/extensions/sources/worldview.py @@ -117,7 +117,7 @@ def _prep(self, paths): if ext == '.zip': # Need to unpack - zip_file = zipfile.ZipFile(paths, 'r') + zip_file = zipfile.ZipFile(paths, 'r') #pylint:disable=R1732 tif_names = list(filter(lambda x: x.lower().endswith('.tif'), zip_file.namelist())) assert len(tif_names) > 0, f'Error: no tif files in the file {paths}' assert len(tif_names) == 1, f'Error: too many tif files in {paths}: {tif_names}' diff --git a/delta/imagery/delta_image.py b/delta/imagery/delta_image.py index e7fc63dd..37bdc3aa 100644 --- a/delta/imagery/delta_image.py +++ b/delta/imagery/delta_image.py @@ -134,7 +134,7 @@ def _read(self, roi: rectangle.Rectangle, bands: List[int], buf: np.ndarray=None The relevant part of the image as a numpy array. """ - def metadata(self): #pylint:disable=no-self-use + def metadata(self): """ Returns ------- @@ -169,7 +169,7 @@ def dtype(self) -> np.dtype: The underlying data type of the image. """ - def block_aligned_roi(self, desired_roi: rectangle.Rectangle) -> rectangle.Rectangle:#pylint:disable=no-self-use + def block_aligned_roi(self, desired_roi: rectangle.Rectangle) -> rectangle.Rectangle: """ Parameters ---------- @@ -183,7 +183,7 @@ def block_aligned_roi(self, desired_roi: rectangle.Rectangle) -> rectangle.Recta """ return desired_roi - def block_size(self) -> Tuple[int, int]: #pylint: disable=no-self-use + def block_size(self) -> Tuple[int, int]: """ Returns ------- diff --git a/delta/imagery/imagery_config.py b/delta/imagery/imagery_config.py index dfa35c59..6e81c5a6 100644 --- a/delta/imagery/imagery_config.py +++ b/delta/imagery/imagery_config.py @@ -213,7 +213,7 @@ def class_shift(data, _, dummy): data = pre_orig(data, _, dummy) # set any nodata values to be past the expected range if labels_nodata is not None: - nodata_indices = (data == labels_nodata) + nodata_indices = data == labels_nodata conv = classes_comp.classes_to_indices_func() if conv is not None: data = conv(data) @@ -262,7 +262,7 @@ def function(self, image_type): Callable: The specified preprocessing function to apply to the image. """ - prep = lambda data, _, dummy: data + prep = lambda data, _, dummy: data # pylint:disable=C3001 for (name, args) in self._functions: t = preprocess_function(name) assert t is not None, 'Preprocess function %s not found.' % (name) diff --git a/delta/ml/config_parser.py b/delta/ml/config_parser.py index 67cacc1e..20393660 100644 --- a/delta/ml/config_parser.py +++ b/delta/ml/config_parser.py @@ -34,7 +34,7 @@ import tensorflow.keras.models #pylint: disable=no-name-in-module from delta.config import config -import delta.config.extensions as extensions +from delta.config import extensions class _LayerWrapper: def __init__(self, layer_type, layer_name, inputs, params, all_layers): @@ -119,7 +119,7 @@ def _make_layer(layer_dict, layer_id, prev_layer, all_layers): """ if len(layer_dict.keys()) > 1: raise ValueError('Layer with multiple types.') - layer_type = next(layer_dict.keys().__iter__()) + layer_type = next(iter(layer_dict.keys())) l = layer_dict[layer_type] if l is None: l = {} @@ -190,7 +190,7 @@ def apply_params(s): # checks if the first layer is an Input, if not insert one layer_list = model_dict_copy['layers'] assert layer_list is not None, 'No model specified!' - first_layer_type = next(layer_list[0].keys().__iter__()) + first_layer_type = next(iter(layer_list[0].keys())) if first_layer_type != 'Input' and 'input' not in layer_list[0][first_layer_type]: model_dict_copy['layers'] = [{'Input' : {'shape' : params['in_shape']}}] + layer_list @@ -418,5 +418,5 @@ def config_augmentation(): func = None for a in augs: f = augmentation_from_dict(a) - func = f if func is None else (lambda f1, f2: lambda x, y: f1(*f2(x, y)))(f, func) + func = f if func is None else (lambda f1, f2: lambda x, y: f1(*f2(x, y)))(f, func) #pylint:disable=C3002 return func diff --git a/delta/ml/ml_config.py b/delta/ml/ml_config.py index a333f971..e8689685 100644 --- a/delta/ml/ml_config.py +++ b/delta/ml/ml_config.py @@ -31,7 +31,7 @@ import yaml from delta.imagery.imagery_config import ImageSet, ImageSetConfig, load_images_labels -import delta.config as config +from delta import config class ValidationSet:#pylint:disable=too-few-public-methods """ diff --git a/delta/ml/train.py b/delta/ml/train.py index b98145ae..9147d42b 100644 --- a/delta/ml/train.py +++ b/delta/ml/train.py @@ -43,7 +43,7 @@ class DeltaLayer(Layer): Extentds `tensorflow.keras.layers.Layer`. """ - def callback(self): # pylint:disable=no-self-use + def callback(self): """ Override this method to make a layer automatically register a training callback. diff --git a/delta/subcommands/classify.py b/delta/subcommands/classify.py index 12eb9e15..3d404552 100644 --- a/delta/subcommands/classify.py +++ b/delta/subcommands/classify.py @@ -76,7 +76,7 @@ def ae_convert(data): def print_classes(output_file, cm, metrics, comment): if output_file is not None: - file_handle = open(output_file, 'a') + file_handle = open(output_file, 'a') #pylint:disable=R0402 print(comment) if output_file is not None: diff --git a/delta/subcommands/visualize.py b/delta/subcommands/visualize.py index 48f2dcd5..63939821 100644 --- a/delta/subcommands/visualize.py +++ b/delta/subcommands/visualize.py @@ -36,7 +36,7 @@ figure = None def plot_images(images, labels): - global done_plot, img_plots, lab_plots, figure #pylint: disable=global-statement + global done_plot, figure #pylint: disable=global-statement if figure is None: figure, axarr = plt.subplots(len(images), images[0].shape[2] + 1) label_norm = matplotlib.colors.Normalize(0, len(config.dataset.classes) + 1) diff --git a/scripts/convert/pbs_functions.py b/scripts/convert/pbs_functions.py index 29d5aaf2..33b79096 100644 --- a/scripts/convert/pbs_functions.py +++ b/scripts/convert/pbs_functions.py @@ -36,7 +36,7 @@ def isNotString(a): # Python 2/3 compatibilty try: - basestring + basestring #pylint:disable=E0601 except NameError: basestring = str diff --git a/scripts/fetch/get_landsat_dswe_labels.py b/scripts/fetch/get_landsat_dswe_labels.py index ee81dab6..238712f1 100755 --- a/scripts/fetch/get_landsat_dswe_labels.py +++ b/scripts/fetch/get_landsat_dswe_labels.py @@ -158,7 +158,7 @@ def fetch_dswe_images(date, ll_coord, ur_coord, output_folder, user, password, f if not r['data']: raise Exception('Failed to get download URL!') url = r['data'][0]['url'] - cmd = ('wget "%s" --user %s --password %s -O %s' % (url, user, password, output_path)) + cmd = 'wget "%s" --user %s --password %s -O %s' % (url, user, password, output_path) print(cmd) os.system(cmd) diff --git a/scripts/fetch/random_folder_split.py b/scripts/fetch/random_folder_split.py index 0b6ee377..2f94bce2 100644 --- a/scripts/fetch/random_folder_split.py +++ b/scripts/fetch/random_folder_split.py @@ -127,7 +127,7 @@ def main(argsIn): #pylint: disable=R0912 continue # Use for validation or for training? - use_for_valid = (random.random() < options.validate_fraction) + use_for_valid = random.random() < options.validate_fraction # Handle the image file target_name = image_name diff --git a/scripts/linter/delta_lint.py b/scripts/linter/delta_lint.py index d4dfb71c..16b521c3 100644 --- a/scripts/linter/delta_lint.py +++ b/scripts/linter/delta_lint.py @@ -19,7 +19,7 @@ from pylint.checkers import BaseChecker COPYRIGHT_HEADER = \ -u""" +""" # Copyright © 2020, United States Government, as represented by the # Administrator of the National Aeronautics and Space Administration. # All rights reserved. diff --git a/scripts/linter/pylintrc b/scripts/linter/pylintrc index 2096a13f..35a71f4b 100644 --- a/scripts/linter/pylintrc +++ b/scripts/linter/pylintrc @@ -36,7 +36,7 @@ load-plugins=delta_lint # --enable=similarities". If you want to run only the classes checker, but have # no Warning level messages displayed, use"--disable=all --enable=classes # --disable=W" -disable=C0103,C0111,W0511,C0321,similarities +disable=C0103,C0111,W0511,C0321,C0209,W1514,R1732,similarities [REPORTS] From 7ec0650688d73b85f941c66431e29074143df483 Mon Sep 17 00:00:00 2001 From: Brian Coltin Date: Tue, 11 Jul 2023 15:42:29 -0700 Subject: [PATCH 3/5] Bump python version. --- .github/workflows/ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 190f45d1..48087c8b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -16,10 +16,10 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Set up Python 3.10 + - name: Set up Python 3.11 uses: actions/setup-python@v4 with: - python-version: '3.10' + python-version: '3.11' - name: Install DELTA run: | ./scripts/setup.sh From d5c5f9808c1f36e5f82b2c7603873ed8f90b1fa1 Mon Sep 17 00:00:00 2001 From: Brian Coltin Date: Tue, 11 Jul 2023 16:39:24 -0700 Subject: [PATCH 4/5] Try to fix linter again. --- scripts/linter/install_linter.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/linter/install_linter.sh b/scripts/linter/install_linter.sh index ab929b05..50aea767 100755 --- a/scripts/linter/install_linter.sh +++ b/scripts/linter/install_linter.sh @@ -2,7 +2,7 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -python3 -m pip install "pylint<2.6" +python3 -m pip install pylint cp $DIR/pre-commit $DIR/../../.git/hooks/pre-commit echo "Linter installed as pre-commit hook." From 88622251854229709bd5e6666e57205cd12ca639 Mon Sep 17 00:00:00 2001 From: Brian Coltin Date: Tue, 11 Jul 2023 17:32:19 -0700 Subject: [PATCH 5/5] Try build again... --- scripts/setup.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/setup.sh b/scripts/setup.sh index 3ec22588..19eef1f9 100755 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -13,8 +13,6 @@ sudo apt install -y libgdal-dev || { echo >&2 "ERROR. Failed to install gdal-dev python3 -m pip install --upgrade pip python3 -m pip install --upgrade requests numpy six -python3 -m pip install setuptools==57.5.0 -python3 -m pip install --use-deprecated=legacy-resolver gdal== python3 -m pip install GDAL==`gdal-config --version` --global-option=build_ext --global-option="-I/usr/include/gdal/" $DIR/linter/install_linter.sh || { echo >&2 "ERROR. Failed to install linter."; exit 1; }