Skip to content

Fix Docker and black CI builds #366

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
jeremyh wants to merge 9 commits into
base: develop
Choose a base branch
from
Draft

Fix Docker and black CI builds #366

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
180ef05
Fix/Stabilise black CI build
jeremyh Jul 12, 2023
70ac000
Use per-file ignore rather than a global disable
jeremyh Jul 14, 2023
f3c43f6
Need a py38-supporting scipy version
jeremyh Jul 14, 2023
8af91f4
Install scipy via condaforge for stability
jeremyh Jul 14, 2023
1c9ae23
Update conda version
jeremyh Jul 14, 2023
ceea3f3
Allow current library versions.
jeremyh Jul 14, 2023
9a0b957
Use actual miniconda version
jeremyh Jul 14, 2023
d51f663
Original conda version
jeremyh Jul 14, 2023
45cc897
Certain libraries need matching versions
jeremyh Jul 14, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .circleci/config.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ jobs:
paths:
- /root/miniconda
key: v1-dependencies-{{ checksum ".circleci/environment.yml" }}

# run tests!
- run:
name: run tests
Expand Down
11 changes: 0 additions & 11 deletions .github/workflows/black.yml
View file
Open in desktop

This file was deleted.

20 changes: 0 additions & 20 deletions .github/workflows/flake8.yml
View file
Open in desktop

This file was deleted.

11 changes: 11 additions & 0 deletions .github/workflows/pre-commit.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
name: pre-commit

on: [push, pull_request]

jobs:
pre-commit:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v3
- uses: pre-commit/action@v3.0.0
26 changes: 14 additions & 12 deletions .pre-commit-config.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,18 +2,20 @@
# See https://pre-commit.com/hooks.html for more hooks
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v2.4.0
rev: v4.4.0
hooks:
- id: trailing-whitespace
- id: end-of-file-fixer
# These create too noisy a diff, as they weren't historically run.
# - id: trailing-whitespace
# - id: end-of-file-fixer
- id: check-yaml
- id: check-added-large-files
- repo: https://github.com/psf/black
rev: 19.3b0
hooks:
- id: black
language_version: python3
- repo: https://gitlab.com/pycqa/flake8
rev: '' # pick a git hash / tag to point to
hooks:
- id: flake8
- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.0.278
hooks:
- id: ruff
args: [ --fix, --exit-non-zero-on-fix ]
- repo: https://github.com/psf/black
rev: 23.7.0
hooks:
- id: black
language_version: python3
6 changes: 4 additions & 2 deletions Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,14 +28,16 @@ RUN chmod +x /root/miniconda.sh && /root/miniconda.sh -b -f -p conda
# GDAL 3.1 is being used because https://gdal.org/api/python.html#usage
RUN conda install -c conda-forge \
gdal==3.1.4 \
python-fmask==0.5.5
python-fmask==0.5.5 \
scipy pandas scikit-image

WORKDIR ${WAGL_DIR}
ADD . ./

# Install dependencies required for unittests
RUN pip install -r requirements.txt
RUN pip install git+https://github.com/sixy6e/idl-functions.git#egg=master
RUN pip install -e .
RUN pip freeze

# include basic details for diagnostics
RUN which python; python --version;
Expand Down
2 changes: 1 addition & 1 deletion docker-compose.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,4 @@ services:
wagl:
build: .
volumes:
- ./:/build/wagl
- ./:/build/wagl
4 changes: 0 additions & 4 deletions env_tests/test_modtran_json_format.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@ class JsonTest(unittest.TestCase):
"""

def test_midlat_summer_albedo(self):

"""
Test the mid latitude summer albedo configuration.
"""
Expand Down Expand Up @@ -56,7 +55,6 @@ def test_midlat_summer_albedo(self):
self.assertTrue(d == data)

def test_tropical_albedo(self):

"""
Test the tropical albedo configuration.
"""
Expand Down Expand Up @@ -87,13 +85,11 @@ def test_tropical_albedo(self):
self.assertTrue(d == data)

def test_thermal_channel(self):

"""
Test surface brightness configuration.
"""
pass


if __name__ == "__main__":

unittest.main()
41 changes: 41 additions & 0 deletions pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,46 @@
[build-system]
requires = ["setuptools>=42", "wheel", "numpy", "setuptools_scm[toml]>=3.4"]

[tool.ruff]
target-version = "py38"

line-length = 90

# Which checkers to enable?
select = [
"E", # pycodestyle
"F", # pyflakes
# "UP", # pyupgrade
"N", # pep8-naming
]

[tool.ruff.per-file-ignores]

# These historically don't follow pep8 naming.
"tests/test_acquisition.py" = ["N802", "N803", "N806", "N816", "N817"]
"tests/test_blrb.py" = ["N802", "N803", "N806", "N816", "N817"]
"tests/test_geobox.py" = ["N802", "N803", "N806", "N816", "N817"]
"tests/test_lon_lat.py" = ["N802", "N803", "N806", "N816", "N817"]
"tests/test_random_locations.py" = ["N802", "N803", "N806", "N816", "N817"]
"tests/test_read_subset.py" = ["N802", "N803", "N806", "N816", "N817"]
"tests/test_vincenty.py" = ["N802", "N803", "N806", "N816", "N817"]
"wagl/acquisition/__init__.py" = ["N802", "N803", "N806", "N816", "N817"]
"wagl/brdf.py" = ["N802", "N803", "N806", "N816", "N817"]
"wagl/fmask_cloud_masking_wrapper.py" = ["N802", "N803", "N806", "N816", "N817"]
"wagl/geobox.py" = ["N802", "N803", "N806", "N816", "N817"]
"wagl/interpolation.py" = ["N802", "N803", "N806", "N816", "N817"]
"wagl/land_sea_masking.py" = ["N802", "N803", "N806", "N816", "N817"]
"wagl/land_sea.py" = ["N802", "N803", "N806", "N816", "N817"]
"wagl/logs.py" = ["N802", "N803", "N806", "N816", "N817"]
"wagl/modtran.py" = ["N802", "N803", "N806", "N816", "N817"]
"wagl/pq.py" = ["N802", "N803", "N806", "N816", "N817"]
"wagl/reflectance.py" = ["N802", "N803", "N806", "N816", "N817"]
"wagl/standardise.py" = ["N802", "N803", "N806", "N816", "N817"]
"wagl/temperature.py" = ["N802", "N803", "N806", "N816", "N817"]
"wagl/unittesting_tools.py" = ["N802", "N803", "N806", "N816", "N817"]
"wagl/vincenty.py" = ["N802", "N803", "N806", "N816", "N817"]


[tool.black]
line-length = 90
target-version = ['py36', 'py37', 'py38']
Expand All @@ -27,3 +67,4 @@ exclude = '''
[tool.setuptools_scm]
write_to = "wagl/version.py"
version_scheme = "post-release"

1 change: 0 additions & 1 deletion tests/test_read_subset.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@


class TestReadSubset(unittest.TestCase):

img, geobox = ut.create_test_image((1200, 1500))
img[:] = 1

Expand Down
2 changes: 0 additions & 2 deletions tests/test_reformat_tp5.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@ def test_midlat_summer_albdo0(self):
ref_albedo = "".join(src.readlines())

with tempfile.TemporaryDirectory() as tmpdir:

out_fname = pjoin(tmpdir, "test-midlat-summer.tp5")
with open(out_fname, "w") as src:
kwargs = {
Expand Down Expand Up @@ -66,7 +65,6 @@ def test_midlat_summer_transmittance(self):
ref_trans = "".join(src.readlines())

with tempfile.TemporaryDirectory() as tmpdir:

out_fname = pjoin(tmpdir, "test-midlat-summer-trans.tp5")
with open(out_fname, "w") as src:
kwargs = {
Expand Down
8 changes: 4 additions & 4 deletions tests/test_tiling.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,20 +72,20 @@ def test_boundary_2(self):

def test_exception_1(self):
"""Test empty image:"""
for (samples, lines, xtile, ytile) in self.exception_input1:
for samples, lines, xtile, ytile in self.exception_input1:
tiles_list = list(generate_tiles(samples, lines, xtile, ytile))
self.assertEqual(tiles_list, [], "Expected an empty tile list.")

def test_exception_2(self):
"""Test empty tiles:"""
for (samples, lines, xtile, ytile) in self.exception_input2:
for samples, lines, xtile, ytile in self.exception_input2:
self.assertRaises(
ZeroDivisionError, generate_tiles, samples, lines, xtile, ytile
)

def do_test(self, test_input):
"""Check sizes and coverage for a list of test input."""
for (samples, lines, xtile, ytile) in test_input:
for samples, lines, xtile, ytile in test_input:
tiles_list = list(generate_tiles(samples, lines, xtile, ytile))

self.check_sizes(xtile, ytile, tiles_list)
Expand Down Expand Up @@ -126,7 +126,7 @@ def check_tiling(self, samples, lines, tiles_list):
# keyword flags. This confuses pylint.
# pylint: enable=unbalanced-tuple-unpacking

for (tag, flat_index) in zip(unique, unique_indices):
for tag, flat_index in zip(unique, unique_indices):
index = numpy.unravel_index(flat_index, (lines, samples))
self.assertGreater(tag, 0, "Hole in coverage detected at " + repr(index))
self.assertIn(tag, tag_dict, "Tile overlap detected at " + repr(index))
Expand Down
1 change: 0 additions & 1 deletion wagl/acquisition/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -510,7 +510,6 @@ def acquisitions_s2_sinergise(pathname):

acqs = []
for band_id in band_configurations:

# If it is a configured B-format transform it to the correct format
if re.match("[0-9].?", band_id):
band_name = "B{}".format(band_id.zfill(2))
Expand Down
1 change: 0 additions & 1 deletion wagl/acquisition/sentinel.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,6 @@ def __init__(
band_id="1",
metadata=None,
):

super(Sentinel2Acquisition, self).__init__(
pathname,
uri,
Expand Down
3 changes: 0 additions & 3 deletions wagl/ancillary.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -167,7 +167,6 @@ def _collect_ancillary(
with h5py.File(satellite_solar_fname, "r") as fid, h5py.File(
out_fname, "w"
) as out_fid:

sat_sol_grp = fid[GroupName.SAT_SOL_GROUP.value]
collect_ancillary(
container,
Expand Down Expand Up @@ -288,7 +287,6 @@ def collect_ancillary(
check_interpolation_sample_geometry(container, group, grp_name)
for grp_name in container.supported_groups
):

coord_dset[:] = default_interpolation_grid(acquisition, vertices, boxline_dataset)
attach_table_attributes(coord_dset, title="Coordinator", attrs=attrs)

Expand Down Expand Up @@ -796,7 +794,6 @@ def get_water_vapour(acquisition, water_vapour_dict, scale_factor=0.1, tolerance
datafile = pjoin(water_vapour_path, filename)

if os.path.isfile(datafile):

with h5py.File(datafile, "r") as fid:
index = read_h5_table(fid, "INDEX")

Expand Down
1 change: 0 additions & 1 deletion wagl/brdf.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -469,7 +469,6 @@ def get_brdf_data(

# Determine if we're being forced into fallback
if os.path.isdir(brdf_primary_path):

# Compare the scene date and MODIS BRDF start date to select the
# BRDF data root directory.
# Scene dates outside this range are to use the fallback data
Expand Down
2 changes: 0 additions & 2 deletions wagl/cloud_shadow_masking.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -522,7 +522,6 @@ def stdev(b1, b2, b3, b4, b5, b7):

i = 1
for lr in lapse_rates:

cheight = cloud_height(ctherm, surface_temp=surfaceTemp, lapse_rate=lr)

d = shadow_length(cheight, rad_elev)
Expand Down Expand Up @@ -556,7 +555,6 @@ def stdev(b1, b2, b3, b4, b5, b7):

i = 1
for lr in lapse_rates:

cheight = cloud_height(ctherm, surface_temp=surfaceTemp, lapse_rate=lr)

shad_length = shadow_length(cheight, rad_elev)
Expand Down
4 changes: 0 additions & 4 deletions wagl/fmask_cloud_masking.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -413,7 +413,6 @@


def match_file(dir_path, pattern):

# Ignore filenames starting with '.' character, e.g. rsync work files.

for f in sorted([x for x in os.listdir(dir_path) if not x.startswith(".")]):
Expand Down Expand Up @@ -1020,7 +1019,6 @@ def nd2toarbt(filename, images=None):
resolu = (reso_ref, reso_ref)

if Lnum >= 4 and Lnum <= 7:

# Band6
if Lnum == 7:
n_B6 = match_file(base, ".*B6_VCID_1.TIF")
Expand Down Expand Up @@ -2022,7 +2020,6 @@ def fcssm(
# Newer method of looping through the cloud objects/segments JS
# 16/12/2013
for cloud_type in s:

cld_area = cloud_type["Area"]
cld_label = cloud_type["Label"]

Expand Down Expand Up @@ -2294,7 +2291,6 @@ def mat_truecloud(x, y, h, A, B, C, omiga_par, omiga_per):


if __name__ == "__main__":

parser = argparse.ArgumentParser(
description="Computes the Fmask algorithm. Cloud, cloud shadow and Fmask combined (contains thecloud, cloud shadow and snow masks in a single array) are output to disk."
)
Expand Down
1 change: 0 additions & 1 deletion wagl/fmask_cloud_masking_wrapper.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
def fmask_cloud_mask(
mtl, null_mask=None, cloud_prob=None, wclr_max=None, sat_tag=None, aux_data=None
):

Lnum = int(sat_tag[-1:])
(_, _, _, _, _, _, _, _, fmask_byte, _, _, _, _, _, _, _) = _fmask.plcloud(
filename=mtl, mask=null_mask, num_Lst=Lnum, aux_data=aux_data or {}
Expand Down
2 changes: 0 additions & 2 deletions wagl/incident_exiting_angles.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@ def _incident_exiting_angles(
with h5py.File(satellite_solar_fname, "r") as sat_sol, h5py.File(
slope_aspect_fname, "r"
) as slp_asp, h5py.File(out_fname, "w") as out_fid:

grp1 = sat_sol[GroupName.SAT_SOL_GROUP.value]
grp2 = slp_asp[GroupName.SLP_ASP_GROUP.value]
if incident:
Expand Down Expand Up @@ -361,7 +360,6 @@ def _relative_azimuth_slope(
with h5py.File(incident_angles_fname, "r") as inci_fid, h5py.File(
exiting_angles_fname, "r"
) as exit_fid, h5py.File(out_fname, "w") as out_fid:

grp1 = inci_fid[GroupName.INCIDENT_GROUP.value]
grp2 = exit_fid[GroupName.EXITING_GROUP.value]
relative_azimuth_slope(grp1, grp2, out_fid, compression, filter_opts)
Expand Down
3 changes: 1 addition & 2 deletions wagl/interpolation.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -228,7 +228,7 @@ def __interpolate_grid_inner(grid, eval_func, depth, origin, shape):
interpolate_block(origin, shape, eval_func, grid)
else:
blocks = subdivide(origin, shape)
for (kUL, _, _, kLR) in blocks.values():
for kUL, _, _, kLR in blocks.values():
block_shape = (kLR[0] - kUL[0] + 1, kLR[1] - kUL[1] + 1)
__interpolate_grid_inner(grid, eval_func, depth - 1, kUL, block_shape)

Expand Down Expand Up @@ -442,7 +442,6 @@ def _interpolate(
) as comp, h5py.File(ancillary_fname, "r") as anc, h5py.File(
out_fname, "w"
) as out_fid:

grp1 = anc[GroupName.ANCILLARY_GROUP.value]
grp2 = sat_sol[GroupName.SAT_SOL_GROUP.value]
grp3 = comp[GroupName.COEFFICIENTS_GROUP.value]
Expand Down
1 change: 0 additions & 1 deletion wagl/land_sea.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,6 @@ def get_land_sea_mask(

# read the land/sea data
with rio.open(utmDataPath) as ds:

# get the gridded box for the full dataset extent
landSeaDataGGB = GriddedGeoBox.from_dataset(ds)

Expand Down
Loading