From 180ef0512d9c50dc1da5cdd00f8570cf9b2f8327 Mon Sep 17 00:00:00 2001 From: Jeremy Hooke Date: Wed, 12 Jul 2023 20:31:13 +1000 Subject: [PATCH 1/9] Fix/Stabilise black CI build The unversioned black build is failing. black is already specified (and frozen) in pre-commit, so we should re-use that. I've also updated the pre-commit config to modernise and match our other repos. --- .circleci/config.yml | 2 +- .github/workflows/black.yml | 11 --------- .github/workflows/flake8.yml | 20 ---------------- .github/workflows/pre-commit.yaml | 11 +++++++++ .pre-commit-config.yaml | 26 +++++++++++---------- docker-compose.yml | 2 +- env_tests/test_modtran_json_format.py | 4 ---- pyproject.toml | 13 +++++++++++ tests/test_read_subset.py | 1 - tests/test_reformat_tp5.py | 2 -- tests/test_tiling.py | 8 +++---- wagl/acquisition/__init__.py | 1 - wagl/acquisition/sentinel.py | 1 - wagl/ancillary.py | 3 --- wagl/brdf.py | 1 - wagl/cloud_shadow_masking.py | 2 -- wagl/fmask_cloud_masking.py | 4 ---- wagl/fmask_cloud_masking_wrapper.py | 1 - wagl/incident_exiting_angles.py | 2 -- wagl/interpolation.py | 3 +-- wagl/land_sea.py | 1 - wagl/land_sea_masking.py | 1 - wagl/modtran.py | 9 ------- wagl/modtran_profile_json.py | 2 -- wagl/multifile_workflow.py | 1 - wagl/reflectance.py | 1 - wagl/scripts/test_dsm.py | 2 -- wagl/scripts/test_exiting_angles.py | 2 -- wagl/scripts/test_incident_angles.py | 2 -- wagl/scripts/test_relative_slope.py | 1 - wagl/scripts/test_satellite_solar_angles.py | 20 ---------------- wagl/scripts/test_slope_aspect.py | 2 -- wagl/scripts/test_terrain_shadow_masks.py | 4 ---- wagl/slope_aspect.py | 1 - wagl/standardise.py | 4 ---- wagl/temperature.py | 1 - wagl/terrain_shadow_masks.py | 3 --- wagl/tiling.py | 6 +++-- wagl/vincenty.py | 1 - 39 files changed, 49 insertions(+), 133 deletions(-) delete mode 100644 .github/workflows/black.yml delete mode 100644 .github/workflows/flake8.yml create mode 100644 .github/workflows/pre-commit.yaml diff --git a/.circleci/config.yml b/.circleci/config.yml index bcb7e7cb..66745988 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -32,7 +32,7 @@ jobs: paths: - /root/miniconda key: v1-dependencies-{{ checksum ".circleci/environment.yml" }} - + # run tests! - run: name: run tests diff --git a/.github/workflows/black.yml b/.github/workflows/black.yml deleted file mode 100644 index 0cc97619..00000000 --- a/.github/workflows/black.yml +++ /dev/null @@ -1,11 +0,0 @@ -name: Lint-black - -on: [push, pull_request] - -jobs: - lint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - - uses: psf/black@stable diff --git a/.github/workflows/flake8.yml b/.github/workflows/flake8.yml deleted file mode 100644 index cc9b41e4..00000000 --- a/.github/workflows/flake8.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: Lint-flake8 - -on: [push, pull_request] - -jobs: - flake8_py3: - runs-on: ubuntu-latest - steps: - - name: Setup Python - uses: actions/setup-python@v1 - - name: Checkout PyTorch - uses: actions/checkout@master - - name: Install flake8 - run: pip install flake8 - - name: Run flake8 - uses: suo/flake8-github-action@releases/v1 - with: - checkName: 'flake8_py3' # NOTE: this needs to be the same as the job name - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/pre-commit.yaml b/.github/workflows/pre-commit.yaml new file mode 100644 index 00000000..313a2c72 --- /dev/null +++ b/.github/workflows/pre-commit.yaml @@ -0,0 +1,11 @@ +name: pre-commit + +on: [push, pull_request] + +jobs: + pre-commit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v3 + - uses: pre-commit/action@v3.0.0 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 35957d26..3a8985d5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,18 +2,20 @@ # See https://pre-commit.com/hooks.html for more hooks repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.4.0 + rev: v4.4.0 hooks: - - id: trailing-whitespace - - id: end-of-file-fixer + # These create too noisy a diff, as they weren't historically run. + # - id: trailing-whitespace + # - id: end-of-file-fixer - id: check-yaml - id: check-added-large-files -- repo: https://github.com/psf/black - rev: 19.3b0 - hooks: - - id: black - language_version: python3 -- repo: https://gitlab.com/pycqa/flake8 - rev: '' # pick a git hash / tag to point to - hooks: - - id: flake8 +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.0.277 + hooks: + - id: ruff + args: [ --fix, --exit-non-zero-on-fix ] +- repo: https://github.com/psf/black + rev: 23.7.0 + hooks: + - id: black + language_version: python3 diff --git a/docker-compose.yml b/docker-compose.yml index 80db46e4..224c78da 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,4 +4,4 @@ services: wagl: build: . volumes: - - ./:/build/wagl \ No newline at end of file + - ./:/build/wagl diff --git a/env_tests/test_modtran_json_format.py b/env_tests/test_modtran_json_format.py index 77a50aea..6811dec9 100644 --- a/env_tests/test_modtran_json_format.py +++ b/env_tests/test_modtran_json_format.py @@ -26,7 +26,6 @@ class JsonTest(unittest.TestCase): """ def test_midlat_summer_albedo(self): - """ Test the mid latitude summer albedo configuration. """ @@ -56,7 +55,6 @@ def test_midlat_summer_albedo(self): self.assertTrue(d == data) def test_tropical_albedo(self): - """ Test the tropical albedo configuration. """ @@ -87,7 +85,6 @@ def test_tropical_albedo(self): self.assertTrue(d == data) def test_thermal_channel(self): - """ Test surface brightness configuration. """ @@ -95,5 +92,4 @@ def test_thermal_channel(self): if __name__ == "__main__": - unittest.main() diff --git a/pyproject.toml b/pyproject.toml index b5fc2a88..af7a5b19 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,19 @@ [build-system] requires = ["setuptools>=42", "wheel", "numpy", "setuptools_scm[toml]>=3.4"] +[tool.ruff] +target-version = "py38" + +line-length = 90 +# Which checkers to enable? +select = [ + "E", # pycodestyle + "F", # pyflakes + # "UP", # pyupgrade + # "N", # pep8-naming +] + + [tool.black] line-length = 90 target-version = ['py36', 'py37', 'py38'] diff --git a/tests/test_read_subset.py b/tests/test_read_subset.py index 11f03797..76e227c7 100644 --- a/tests/test_read_subset.py +++ b/tests/test_read_subset.py @@ -15,7 +15,6 @@ class TestReadSubset(unittest.TestCase): - img, geobox = ut.create_test_image((1200, 1500)) img[:] = 1 diff --git a/tests/test_reformat_tp5.py b/tests/test_reformat_tp5.py index c9b74407..680a180a 100644 --- a/tests/test_reformat_tp5.py +++ b/tests/test_reformat_tp5.py @@ -29,7 +29,6 @@ def test_midlat_summer_albdo0(self): ref_albedo = "".join(src.readlines()) with tempfile.TemporaryDirectory() as tmpdir: - out_fname = pjoin(tmpdir, "test-midlat-summer.tp5") with open(out_fname, "w") as src: kwargs = { @@ -66,7 +65,6 @@ def test_midlat_summer_transmittance(self): ref_trans = "".join(src.readlines()) with tempfile.TemporaryDirectory() as tmpdir: - out_fname = pjoin(tmpdir, "test-midlat-summer-trans.tp5") with open(out_fname, "w") as src: kwargs = { diff --git a/tests/test_tiling.py b/tests/test_tiling.py index c1992597..77e2c309 100644 --- a/tests/test_tiling.py +++ b/tests/test_tiling.py @@ -72,20 +72,20 @@ def test_boundary_2(self): def test_exception_1(self): """Test empty image:""" - for (samples, lines, xtile, ytile) in self.exception_input1: + for samples, lines, xtile, ytile in self.exception_input1: tiles_list = list(generate_tiles(samples, lines, xtile, ytile)) self.assertEqual(tiles_list, [], "Expected an empty tile list.") def test_exception_2(self): """Test empty tiles:""" - for (samples, lines, xtile, ytile) in self.exception_input2: + for samples, lines, xtile, ytile in self.exception_input2: self.assertRaises( ZeroDivisionError, generate_tiles, samples, lines, xtile, ytile ) def do_test(self, test_input): """Check sizes and coverage for a list of test input.""" - for (samples, lines, xtile, ytile) in test_input: + for samples, lines, xtile, ytile in test_input: tiles_list = list(generate_tiles(samples, lines, xtile, ytile)) self.check_sizes(xtile, ytile, tiles_list) @@ -126,7 +126,7 @@ def check_tiling(self, samples, lines, tiles_list): # keyword flags. This confuses pylint. # pylint: enable=unbalanced-tuple-unpacking - for (tag, flat_index) in zip(unique, unique_indices): + for tag, flat_index in zip(unique, unique_indices): index = numpy.unravel_index(flat_index, (lines, samples)) self.assertGreater(tag, 0, "Hole in coverage detected at " + repr(index)) self.assertIn(tag, tag_dict, "Tile overlap detected at " + repr(index)) diff --git a/wagl/acquisition/__init__.py b/wagl/acquisition/__init__.py index 7a973452..2448d123 100644 --- a/wagl/acquisition/__init__.py +++ b/wagl/acquisition/__init__.py @@ -510,7 +510,6 @@ def acquisitions_s2_sinergise(pathname): acqs = [] for band_id in band_configurations: - # If it is a configured B-format transform it to the correct format if re.match("[0-9].?", band_id): band_name = "B{}".format(band_id.zfill(2)) diff --git a/wagl/acquisition/sentinel.py b/wagl/acquisition/sentinel.py index 52f59fb2..7d321209 100644 --- a/wagl/acquisition/sentinel.py +++ b/wagl/acquisition/sentinel.py @@ -69,7 +69,6 @@ def __init__( band_id="1", metadata=None, ): - super(Sentinel2Acquisition, self).__init__( pathname, uri, diff --git a/wagl/ancillary.py b/wagl/ancillary.py index 95f448f0..2e3e49e5 100644 --- a/wagl/ancillary.py +++ b/wagl/ancillary.py @@ -167,7 +167,6 @@ def _collect_ancillary( with h5py.File(satellite_solar_fname, "r") as fid, h5py.File( out_fname, "w" ) as out_fid: - sat_sol_grp = fid[GroupName.SAT_SOL_GROUP.value] collect_ancillary( container, @@ -288,7 +287,6 @@ def collect_ancillary( check_interpolation_sample_geometry(container, group, grp_name) for grp_name in container.supported_groups ): - coord_dset[:] = default_interpolation_grid(acquisition, vertices, boxline_dataset) attach_table_attributes(coord_dset, title="Coordinator", attrs=attrs) @@ -796,7 +794,6 @@ def get_water_vapour(acquisition, water_vapour_dict, scale_factor=0.1, tolerance datafile = pjoin(water_vapour_path, filename) if os.path.isfile(datafile): - with h5py.File(datafile, "r") as fid: index = read_h5_table(fid, "INDEX") diff --git a/wagl/brdf.py b/wagl/brdf.py index e6be795a..f54f0ebc 100644 --- a/wagl/brdf.py +++ b/wagl/brdf.py @@ -469,7 +469,6 @@ def get_brdf_data( # Determine if we're being forced into fallback if os.path.isdir(brdf_primary_path): - # Compare the scene date and MODIS BRDF start date to select the # BRDF data root directory. # Scene dates outside this range are to use the fallback data diff --git a/wagl/cloud_shadow_masking.py b/wagl/cloud_shadow_masking.py index 9de5001e..31d14fde 100644 --- a/wagl/cloud_shadow_masking.py +++ b/wagl/cloud_shadow_masking.py @@ -522,7 +522,6 @@ def stdev(b1, b2, b3, b4, b5, b7): i = 1 for lr in lapse_rates: - cheight = cloud_height(ctherm, surface_temp=surfaceTemp, lapse_rate=lr) d = shadow_length(cheight, rad_elev) @@ -556,7 +555,6 @@ def stdev(b1, b2, b3, b4, b5, b7): i = 1 for lr in lapse_rates: - cheight = cloud_height(ctherm, surface_temp=surfaceTemp, lapse_rate=lr) shad_length = shadow_length(cheight, rad_elev) diff --git a/wagl/fmask_cloud_masking.py b/wagl/fmask_cloud_masking.py index 9e79e02f..8a219ba4 100644 --- a/wagl/fmask_cloud_masking.py +++ b/wagl/fmask_cloud_masking.py @@ -413,7 +413,6 @@ def match_file(dir_path, pattern): - # Ignore filenames starting with '.' character, e.g. rsync work files. for f in sorted([x for x in os.listdir(dir_path) if not x.startswith(".")]): @@ -1020,7 +1019,6 @@ def nd2toarbt(filename, images=None): resolu = (reso_ref, reso_ref) if Lnum >= 4 and Lnum <= 7: - # Band6 if Lnum == 7: n_B6 = match_file(base, ".*B6_VCID_1.TIF") @@ -2022,7 +2020,6 @@ def fcssm( # Newer method of looping through the cloud objects/segments JS # 16/12/2013 for cloud_type in s: - cld_area = cloud_type["Area"] cld_label = cloud_type["Label"] @@ -2294,7 +2291,6 @@ def mat_truecloud(x, y, h, A, B, C, omiga_par, omiga_per): if __name__ == "__main__": - parser = argparse.ArgumentParser( description="Computes the Fmask algorithm. Cloud, cloud shadow and Fmask combined (contains thecloud, cloud shadow and snow masks in a single array) are output to disk." ) diff --git a/wagl/fmask_cloud_masking_wrapper.py b/wagl/fmask_cloud_masking_wrapper.py index 04d3bf53..a6daf088 100644 --- a/wagl/fmask_cloud_masking_wrapper.py +++ b/wagl/fmask_cloud_masking_wrapper.py @@ -7,7 +7,6 @@ def fmask_cloud_mask( mtl, null_mask=None, cloud_prob=None, wclr_max=None, sat_tag=None, aux_data=None ): - Lnum = int(sat_tag[-1:]) (_, _, _, _, _, _, _, _, fmask_byte, _, _, _, _, _, _, _) = _fmask.plcloud( filename=mtl, mask=null_mask, num_Lst=Lnum, aux_data=aux_data or {} diff --git a/wagl/incident_exiting_angles.py b/wagl/incident_exiting_angles.py index 3f004eef..31b62085 100644 --- a/wagl/incident_exiting_angles.py +++ b/wagl/incident_exiting_angles.py @@ -32,7 +32,6 @@ def _incident_exiting_angles( with h5py.File(satellite_solar_fname, "r") as sat_sol, h5py.File( slope_aspect_fname, "r" ) as slp_asp, h5py.File(out_fname, "w") as out_fid: - grp1 = sat_sol[GroupName.SAT_SOL_GROUP.value] grp2 = slp_asp[GroupName.SLP_ASP_GROUP.value] if incident: @@ -361,7 +360,6 @@ def _relative_azimuth_slope( with h5py.File(incident_angles_fname, "r") as inci_fid, h5py.File( exiting_angles_fname, "r" ) as exit_fid, h5py.File(out_fname, "w") as out_fid: - grp1 = inci_fid[GroupName.INCIDENT_GROUP.value] grp2 = exit_fid[GroupName.EXITING_GROUP.value] relative_azimuth_slope(grp1, grp2, out_fid, compression, filter_opts) diff --git a/wagl/interpolation.py b/wagl/interpolation.py index 6babe273..fff7d436 100644 --- a/wagl/interpolation.py +++ b/wagl/interpolation.py @@ -228,7 +228,7 @@ def __interpolate_grid_inner(grid, eval_func, depth, origin, shape): interpolate_block(origin, shape, eval_func, grid) else: blocks = subdivide(origin, shape) - for (kUL, _, _, kLR) in blocks.values(): + for kUL, _, _, kLR in blocks.values(): block_shape = (kLR[0] - kUL[0] + 1, kLR[1] - kUL[1] + 1) __interpolate_grid_inner(grid, eval_func, depth - 1, kUL, block_shape) @@ -442,7 +442,6 @@ def _interpolate( ) as comp, h5py.File(ancillary_fname, "r") as anc, h5py.File( out_fname, "w" ) as out_fid: - grp1 = anc[GroupName.ANCILLARY_GROUP.value] grp2 = sat_sol[GroupName.SAT_SOL_GROUP.value] grp3 = comp[GroupName.COEFFICIENTS_GROUP.value] diff --git a/wagl/land_sea.py b/wagl/land_sea.py index 6c74efe5..e72f384f 100644 --- a/wagl/land_sea.py +++ b/wagl/land_sea.py @@ -56,7 +56,6 @@ def get_land_sea_mask( # read the land/sea data with rio.open(utmDataPath) as ds: - # get the gridded box for the full dataset extent landSeaDataGGB = GriddedGeoBox.from_dataset(ds) diff --git a/wagl/land_sea_masking.py b/wagl/land_sea_masking.py index 5351305b..7c218332 100644 --- a/wagl/land_sea_masking.py +++ b/wagl/land_sea_masking.py @@ -118,7 +118,6 @@ def set_land_sea_bit( pqaResult, ancillary_path="/g/data/v10/eoancillarydata/Land_Sea_Rasters", ): - mask, md = calc_land_sea_mask(gridded_geo_box, ancillary_path) bit_index = pq_const.land_sea pqaResult.set_mask(mask, bit_index) diff --git a/wagl/modtran.py b/wagl/modtran.py index 6a06771e..728a3c30 100644 --- a/wagl/modtran.py +++ b/wagl/modtran.py @@ -89,7 +89,6 @@ def _format_json( ) as lon_lat_fid, h5py.File(ancillary_fname, "r") as anc_fid, h5py.File( out_fname, "w" ) as fid: - grp1 = anc_fid[GroupName.ANCILLARY_GROUP.value] grp2 = sat_sol_fid[GroupName.SAT_SOL_GROUP.value] grp3 = lon_lat_fid[GroupName.LON_LAT_GROUP.value] @@ -175,9 +174,7 @@ def format_json( acqs = [a for a in acquisitions if a.band_type == BandType.REFLECTIVE] for p in range(npoints): - for alb in Workflow.NBAR.albedos: - input_data = { "name": POINT_ALBEDO_FMT.format(p=p, a=str(alb.value)), "water": water_vapour, @@ -222,7 +219,6 @@ def format_json( acqs = [a for a in acquisitions if a.band_type == BandType.THERMAL] for p in range(npoints): - atmos_profile = read_h5_table(ancillary_group, dname.format(p=p)) n_layers = atmos_profile.shape[0] + 6 @@ -288,7 +284,6 @@ def _run_modtran( with h5py.File(atmospheric_inputs_fname, "r") as atmos_fid, h5py.File( out_fname, "w" ) as fid: - atmos_grp = atmos_fid[GroupName.ATMOSPHERIC_INPUTS_GRP.value] run_modtran( acquisitions, @@ -454,7 +449,6 @@ def _calculate_coefficients( with h5py.File(atmosheric_results_fname, "r") as atmos_fid, h5py.File( out_fname, "w" ) as fid: - results_group = atmos_fid[GroupName.ATMOSPHERIC_RESULTS_GRP.value] calculate_coefficients(results_group, fid, compression, filter_opts) @@ -666,7 +660,6 @@ def coefficients( nbar = sbt = None if channel_data is not None: - # calculate transmittance using channel data columns = [v.value for v in Workflow.NBAR.atmos_coefficients] @@ -802,7 +795,6 @@ def _get_solar_angles(tp6_fname): line, "*SINGLE SCATTER SOLAR PATH GEOMETRY " "TABLE FOR MULTIPLE SCATTERING*", ): - break else: raise ValueError( @@ -849,7 +841,6 @@ def read_modtran_channel(chn_fname, tp6_fname, acquisition, albedo): nbands = response.index.get_level_values("band_name").unique().shape[0] if albedo == Albedos.ALBEDO_TH: - upward_radiation = pd.read_csv( chn_fname, skiprows=5, header=None, delim_whitespace=True, nrows=nbands ) diff --git a/wagl/modtran_profile_json.py b/wagl/modtran_profile_json.py index b1814253..3ba8cfd2 100644 --- a/wagl/modtran_profile_json.py +++ b/wagl/modtran_profile_json.py @@ -40,7 +40,6 @@ def midlat_summer_albedo( filter_function, binary, ): - """ MODTRAN 6.0.1 input: 'json' format template for mid latitude summer albedo @@ -169,7 +168,6 @@ def tropical_albedo( filter_function, binary, ): - """ MODTRAN 6.0.1 input: 'json' format template for tropical albedo diff --git a/wagl/multifile_workflow.py b/wagl/multifile_workflow.py index dee3215e..ae763554 100644 --- a/wagl/multifile_workflow.py +++ b/wagl/multifile_workflow.py @@ -312,7 +312,6 @@ def run(self): ) with luigi.LocalTarget(target).open("w") as src: - # Thermal processing has two input configurations for modtran_input in json_data[key]["MODTRAN"]: modtran_input["MODTRANINPUT"]["SPECTRAL"]["FILTNM"] = pjoin( diff --git a/wagl/reflectance.py b/wagl/reflectance.py index f1368f8d..05e22373 100644 --- a/wagl/reflectance.py +++ b/wagl/reflectance.py @@ -60,7 +60,6 @@ def _calculate_reflectance( ) as fid_anc, h5py.File( out_fname, "w" ) as fid: - grp1 = fid_interp[GroupName.INTERP_GROUP.value] grp2 = fid_sat_sol[GroupName.SAT_SOL_GROUP.value] grp3 = fid_slp_asp[GroupName.SLP_ASP_GROUP.value] diff --git a/wagl/scripts/test_dsm.py b/wagl/scripts/test_dsm.py index 2a7cb8f7..49aa42a0 100644 --- a/wagl/scripts/test_dsm.py +++ b/wagl/scripts/test_dsm.py @@ -36,7 +36,6 @@ def test_dsm(self): with h5py.File(self.reference_fname, "r") as reference_fid, h5py.File( self.test_fname, "r" ) as test_fid: - ref_dset = reference_fid["dsm"] test_dset = test_fid["dsm"] @@ -49,7 +48,6 @@ def test_smoothed_dsm(self): with h5py.File(self.reference_fname, "r") as reference_fid, h5py.File( self.test_fname, "r" ) as test_fid: - ref_dset = reference_fid["dsm-smoothed"] test_dset = test_fid["dsm-smoothed"] diff --git a/wagl/scripts/test_exiting_angles.py b/wagl/scripts/test_exiting_angles.py index 7a45e405..f9ff00e5 100644 --- a/wagl/scripts/test_exiting_angles.py +++ b/wagl/scripts/test_exiting_angles.py @@ -36,7 +36,6 @@ def test_exiting_angle(self): with h5py.File(self.reference_fname, "r") as reference_fid, h5py.File( self.test_fname, "r" ) as test_fid: - ref_dset = reference_fid["exiting"] test_dset = test_fid["exiting"] @@ -49,7 +48,6 @@ def test_azimuthal_exiting_angle(self): with h5py.File(self.reference_fname, "r") as reference_fid, h5py.File( self.test_fname, "r" ) as test_fid: - ref_dset = reference_fid["azimuthal-exiting"] test_dset = test_fid["azimuthal-exiting"] diff --git a/wagl/scripts/test_incident_angles.py b/wagl/scripts/test_incident_angles.py index 6baab69f..5a14c4eb 100644 --- a/wagl/scripts/test_incident_angles.py +++ b/wagl/scripts/test_incident_angles.py @@ -36,7 +36,6 @@ def test_incident_angle(self): with h5py.File(self.reference_fname, "r") as reference_fid, h5py.File( self.test_fname, "r" ) as test_fid: - ref_dset = reference_fid["incident"] test_dset = test_fid["incident"] @@ -49,7 +48,6 @@ def test_azimuthal_incident_angle(self): with h5py.File(self.reference_fname, "r") as reference_fid, h5py.File( self.test_fname, "r" ) as test_fid: - ref_dset = reference_fid["azimuthal-incident"] test_dset = test_fid["azimuthal-incident"] diff --git a/wagl/scripts/test_relative_slope.py b/wagl/scripts/test_relative_slope.py index 0581a72a..b33b67fe 100644 --- a/wagl/scripts/test_relative_slope.py +++ b/wagl/scripts/test_relative_slope.py @@ -35,7 +35,6 @@ def test_relative_slope(self): with h5py.File(self.reference_fname, "r") as reference_fid, h5py.File( self.test_fname, "r" ) as test_fid: - ref_dset = reference_fid["relative-slope"] test_dset = test_fid["relative-slope"] diff --git a/wagl/scripts/test_satellite_solar_angles.py b/wagl/scripts/test_satellite_solar_angles.py index 36c51452..9e508d7c 100644 --- a/wagl/scripts/test_satellite_solar_angles.py +++ b/wagl/scripts/test_satellite_solar_angles.py @@ -44,7 +44,6 @@ def test_satellite_view(self): with h5py.File(self.reference_fname, "r") as reference_fid, h5py.File( self.test_fname, "r" ) as test_fid: - ref_dset = reference_fid["satellite-view"] test_dset = test_fid["satellite-view"] @@ -57,7 +56,6 @@ def test_satellite_azimuth(self): with h5py.File(self.reference_fname, "r") as reference_fid, h5py.File( self.test_fname, "r" ) as test_fid: - ref_dset = reference_fid["satellite-azimuth"] test_dset = test_fid["satellite-azimuth"] @@ -70,7 +68,6 @@ def test_solar_zenith(self): with h5py.File(self.reference_fname, "r") as reference_fid, h5py.File( self.test_fname, "r" ) as test_fid: - ref_dset = reference_fid["solar-zenith"] test_dset = test_fid["solar-zenith"] @@ -83,7 +80,6 @@ def test_solar_azimuth(self): with h5py.File(self.reference_fname, "r") as reference_fid, h5py.File( self.test_fname, "r" ) as test_fid: - ref_dset = reference_fid["solar-azimuth"] test_dset = test_fid["solar-azimuth"] @@ -96,7 +92,6 @@ def test_relative_azimuth(self): with h5py.File(self.reference_fname, "r") as reference_fid, h5py.File( self.test_fname, "r" ) as test_fid: - ref_dset = reference_fid["relative-azimuth"] test_dset = test_fid["relative-azimuth"] @@ -109,7 +104,6 @@ def test_time_array(self): with h5py.File(self.reference_fname, "r") as reference_fid, h5py.File( self.test_fname, "r" ) as test_fid: - ref_dset = reference_fid["acquisition-time"] test_dset = test_fid["acquisition-time"] @@ -123,7 +117,6 @@ def test_centreline_dataset(self): with h5py.File(self.reference_fname, "r") as reference_fid, h5py.File( self.test_fname, "r" ) as test_fid: - ref_data = read_h5_table(reference_fid, "centreline") test_data = read_h5_table(test_fid, "centreline") @@ -136,7 +129,6 @@ def test_centreline_row_index(self): with h5py.File(self.reference_fname, "r") as reference_fid, h5py.File( self.test_fname, "r" ) as test_fid: - ref_data = reference_fid["centreline"]["row_index"] test_data = test_fid["centreline"]["row_index"] @@ -149,7 +141,6 @@ def test_centreline_col_index(self): with h5py.File(self.reference_fname, "r") as reference_fid, h5py.File( self.test_fname, "r" ) as test_fid: - ref_data = reference_fid["centreline"]["col_index"] test_data = test_fid["centreline"]["col_index"] @@ -162,7 +153,6 @@ def test_centreline_n_pixels(self): with h5py.File(self.reference_fname, "r") as reference_fid, h5py.File( self.test_fname, "r" ) as test_fid: - ref_data = reference_fid["centreline"]["n_pixels"] test_data = test_fid["centreline"]["n_pixels"] @@ -175,7 +165,6 @@ def test_centreline_latitude(self): with h5py.File(self.reference_fname, "r") as reference_fid, h5py.File( self.test_fname, "r" ) as test_fid: - ref_data = reference_fid["centreline"]["latitude"] test_data = test_fid["centreline"]["latitude"] @@ -188,7 +177,6 @@ def test_centreline_longitude(self): with h5py.File(self.reference_fname, "r") as reference_fid, h5py.File( self.test_fname, "r" ) as test_fid: - ref_data = reference_fid["centreline"]["longitude"] test_data = test_fid["centreline"]["longitude"] @@ -202,7 +190,6 @@ def test_boxline_dataset(self): with h5py.File(self.reference_fname, "r") as reference_fid, h5py.File( self.test_fname, "r" ) as test_fid: - ref_data = read_h5_table(reference_fid, "boxline") test_data = read_h5_table(test_fid, "boxline") @@ -215,7 +202,6 @@ def test_boxline_row_index(self): with h5py.File(self.reference_fname, "r") as reference_fid, h5py.File( self.test_fname, "r" ) as test_fid: - ref_data = reference_fid["boxline"]["row_index"] test_data = test_fid["boxline"]["row_index"] @@ -228,7 +214,6 @@ def test_boxline_bisection_index(self): with h5py.File(self.reference_fname, "r") as reference_fid, h5py.File( self.test_fname, "r" ) as test_fid: - ref_data = reference_fid["boxline"]["bisection_index"] test_data = test_fid["boxline"]["bisection_index"] @@ -241,7 +226,6 @@ def test_boxline_start_index(self): with h5py.File(self.reference_fname, "r") as reference_fid, h5py.File( self.test_fname, "r" ) as test_fid: - ref_data = reference_fid["boxline"]["start_index"] test_data = test_fid["boxline"]["start_index"] @@ -254,7 +238,6 @@ def test_boxline_end_index(self): with h5py.File(self.reference_fname, "r") as reference_fid, h5py.File( self.test_fname, "r" ) as test_fid: - ref_data = reference_fid["boxline"]["end_index"] test_data = test_fid["boxline"]["end_index"] @@ -268,7 +251,6 @@ def test_coordinator_dataset(self): with h5py.File(self.reference_fname, "r") as reference_fid, h5py.File( self.test_fname, "r" ) as test_fid: - ref_data = read_h5_table(reference_fid, "coordinator") test_data = read_h5_table(test_fid, "coordinator") @@ -281,7 +263,6 @@ def test_coordinator_row_index(self): with h5py.File(self.reference_fname, "r") as reference_fid, h5py.File( self.test_fname, "r" ) as test_fid: - ref_data = reference_fid["coordinator"]["row_index"] test_data = test_fid["coordinator"]["row_index"] @@ -294,7 +275,6 @@ def test_coordinator_col_index(self): with h5py.File(self.reference_fname, "r") as reference_fid, h5py.File( self.test_fname, "r" ) as test_fid: - ref_data = reference_fid["coordinator"]["col_index"] test_data = test_fid["coordinator"]["col_index"] diff --git a/wagl/scripts/test_slope_aspect.py b/wagl/scripts/test_slope_aspect.py index 2c254cf0..19184221 100644 --- a/wagl/scripts/test_slope_aspect.py +++ b/wagl/scripts/test_slope_aspect.py @@ -36,7 +36,6 @@ def test_slope(self): with h5py.File(self.reference_fname, "r") as reference_fid, h5py.File( self.test_fname, "r" ) as test_fid: - ref_dset = reference_fid["slope"] test_dset = test_fid["slope"] @@ -49,7 +48,6 @@ def test_aspect(self): with h5py.File(self.reference_fname, "r") as reference_fid, h5py.File( self.test_fname, "r" ) as test_fid: - ref_dset = reference_fid["aspect"] test_dset = test_fid["aspect"] diff --git a/wagl/scripts/test_terrain_shadow_masks.py b/wagl/scripts/test_terrain_shadow_masks.py index e92c66f9..cc99c4a3 100644 --- a/wagl/scripts/test_terrain_shadow_masks.py +++ b/wagl/scripts/test_terrain_shadow_masks.py @@ -39,7 +39,6 @@ def test_self_shadow(self): with h5py.File(self.reference_fname, "r") as reference_fid, h5py.File( self.test_fname, "r" ) as test_fid: - ref_dset = reference_fid["self-shadow"] test_dset = test_fid["self-shadow"] @@ -52,7 +51,6 @@ def test_cast_shadow_sun(self): with h5py.File(self.reference_fname, "r") as reference_fid, h5py.File( self.test_fname, "r" ) as test_fid: - ref_dset = reference_fid["cast-shadow-sun"] test_dset = test_fid["cast-shadow-sun"] @@ -65,7 +63,6 @@ def test_cast_shadow_satellite(self): with h5py.File(self.reference_fname, "r") as reference_fid, h5py.File( self.test_fname, "r" ) as test_fid: - ref_dset = reference_fid["cast-shadow-satellite"] test_dset = test_fid["cast-shadow-satellite"] @@ -78,7 +75,6 @@ def test_combined_shadow(self): with h5py.File(self.reference_fname, "r") as reference_fid, h5py.File( self.test_fname, "r" ) as test_fid: - ref_dset = reference_fid["combined-shadow"] test_dset = test_fid["combined-shadow"] diff --git a/wagl/slope_aspect.py b/wagl/slope_aspect.py index 161deb17..0b5de6ea 100644 --- a/wagl/slope_aspect.py +++ b/wagl/slope_aspect.py @@ -29,7 +29,6 @@ def _slope_aspect_arrays( NBAR workflow. """ with h5py.File(dsm_fname, "r") as dsm_fid, h5py.File(out_fname, "w") as fid: - dsm_grp = dsm_fid[GroupName.ELEVATION_GROUP.value] slope_aspect_arrays(acquisition, dsm_grp, buffer_distance, fid, compression) diff --git a/wagl/standardise.py b/wagl/standardise.py index 281a7152..338a87b0 100644 --- a/wagl/standardise.py +++ b/wagl/standardise.py @@ -220,7 +220,6 @@ def card4l( ) if workflow in (Workflow.STANDARD, Workflow.NBAR): - # DEM log.info("DEM-retriveal") get_dsm( @@ -375,7 +374,6 @@ def card4l( log.info("Radiative-Transfer", point=point, albedo=albedo.value) with tempfile.TemporaryDirectory() as tmpdir: - prepare_modtran(acqs, point, [albedo], tmpdir) point_dir = pjoin(tmpdir, POINT_FMT.format(p=point)) @@ -387,7 +385,6 @@ def card4l( json_dict = json_data[key] if albedo == Albedos.ALBEDO_TH: - json_dict["MODTRAN"][0]["MODTRANINPUT"]["SPECTRAL"][ "FILTNM" ] = "%s/%s" % ( @@ -402,7 +399,6 @@ def card4l( ) else: - json_dict["MODTRAN"][0]["MODTRANINPUT"]["SPECTRAL"][ "FILTNM" ] = "%s/%s" % ( diff --git a/wagl/temperature.py b/wagl/temperature.py index 4832227c..fcb2da19 100644 --- a/wagl/temperature.py +++ b/wagl/temperature.py @@ -36,7 +36,6 @@ def _surface_brightness_temperature( with h5py.File(bilinear_fname, "r") as interp_fid, h5py.File( ancillary_fname, "r" ) as fid_anc, h5py.File(out_fname, "w") as fid: - grp1 = interp_fid[GroupName.INTERP_GROUP.value] surface_brightness_temperature(acquisition, grp1, fid, compression, filter_opts) diff --git a/wagl/terrain_shadow_masks.py b/wagl/terrain_shadow_masks.py index 749411d8..70592b43 100644 --- a/wagl/terrain_shadow_masks.py +++ b/wagl/terrain_shadow_masks.py @@ -37,7 +37,6 @@ def _self_shadow( with h5py.File(incident_angles_fname, "r") as fid_incident, h5py.File( exiting_angles_fname, "r" ) as fid_exiting, h5py.File(out_fname, "w") as fid: - grp1 = fid_incident[GroupName.INCIDENT_GROUP.value] grp2 = fid_exiting[GroupName.EXITING_GROUP.value] self_shadow(grp1, grp2, fid, compression, filter_opts) @@ -302,7 +301,6 @@ def _calculate_cast_shadow( with h5py.File(dsm_fname, "r") as dsm_fid, h5py.File( satellite_solar_angles_fname, "r" ) as fid_sat_sol, h5py.File(out_fname, "w") as fid: - grp1 = dsm_fid[GroupName.ELEVATION_GROUP.value] grp2 = fid_sat_sol[GroupName.SAT_SOL_GROUP.value] calculate_cast_shadow( @@ -538,7 +536,6 @@ def _combine_shadow( ) as fid_sun, h5py.File(cast_shadow_satellite_fname, "r") as fid_sat, h5py.File( out_fname, "w" ) as fid: - grp1 = fid_self[GroupName.SHADOW_GROUP.value] grp2 = fid_sun[GroupName.SHADOW_GROUP.value] grp3 = fid_sat[GroupName.SHADOW_GROUP.value] diff --git a/wagl/tiling.py b/wagl/tiling.py index fba4db64..0111f62e 100644 --- a/wagl/tiling.py +++ b/wagl/tiling.py @@ -144,7 +144,8 @@ def __init__( (a - img).sum() == 0 >>> len(tiles) 100 - >>> outds = TiledOutput('test_tiled_output_2D', samples=a.shape[1], lines=a.shape[0]) # noqa: E501 + >>> l, s = a.shape + >>> outds = TiledOutput('test_tiled_output_2D', samples=s, lines=l) >>> outds.closed False >>> for tile in tiles: @@ -166,7 +167,8 @@ def __init__( >>> a = numpy.random.randint(0, 256, (10, 100, 100)).astype('uint8') outds.closed >>> tiles = generate_tiles(a.shape[2], a.shape[1], 10, 10) - >>> outds = TiledOutput('test_tiled_output_3D', samples=a.shape[2], lines=a.shape[1], bands=a.shape[0]) # noqa: E501 + >>> b, l, s = a.shape + >>> outds = TiledOutput('test_tiled_output_3D', samples=s, lines=l, bands=b) for tile in tiles: ys, ye = tile[0] xs, xe = tile[1] diff --git a/wagl/vincenty.py b/wagl/vincenty.py index c141ecc3..cd715ccf 100644 --- a/wagl/vincenty.py +++ b/wagl/vincenty.py @@ -268,7 +268,6 @@ def vinc_dist(f, a, phi1, lembda1, phi2, lembda2): or lembda != 0 and abs((last_lembda - lembda) / lembda) > 1.0e-9 ): - sqr_sin_sigma = pow(math.cos(U2) * math.sin(lembda), 2) + pow( ( math.cos(U1) * math.sin(U2) From 70ac0007c5b6d11b0255f8c2ebcfea5eb11f3788 Mon Sep 17 00:00:00 2001 From: Jeremy Hooke Date: Fri, 14 Jul 2023 12:31:49 +1000 Subject: [PATCH 2/9] Use per-file ignore rather than a global disable --- .pre-commit-config.yaml | 2 +- pyproject.toml | 30 +++++++++++++++++++++++++++++- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3a8985d5..89a79738 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,7 +10,7 @@ repos: - id: check-yaml - id: check-added-large-files - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.0.277 + rev: v0.0.278 hooks: - id: ruff args: [ --fix, --exit-non-zero-on-fix ] diff --git a/pyproject.toml b/pyproject.toml index af7a5b19..3e816d3e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,14 +5,41 @@ requires = ["setuptools>=42", "wheel", "numpy", "setuptools_scm[toml]>=3.4"] target-version = "py38" line-length = 90 + # Which checkers to enable? select = [ "E", # pycodestyle "F", # pyflakes # "UP", # pyupgrade - # "N", # pep8-naming + "N", # pep8-naming ] +[tool.ruff.per-file-ignores] + +# These historically don't follow pep8 naming. +"tests/test_acquisition.py" = ["N802", "N803", "N806", "N816", "N817"] +"tests/test_blrb.py" = ["N802", "N803", "N806", "N816", "N817"] +"tests/test_geobox.py" = ["N802", "N803", "N806", "N816", "N817"] +"tests/test_lon_lat.py" = ["N802", "N803", "N806", "N816", "N817"] +"tests/test_random_locations.py" = ["N802", "N803", "N806", "N816", "N817"] +"tests/test_read_subset.py" = ["N802", "N803", "N806", "N816", "N817"] +"tests/test_vincenty.py" = ["N802", "N803", "N806", "N816", "N817"] +"wagl/acquisition/__init__.py" = ["N802", "N803", "N806", "N816", "N817"] +"wagl/brdf.py" = ["N802", "N803", "N806", "N816", "N817"] +"wagl/fmask_cloud_masking_wrapper.py" = ["N802", "N803", "N806", "N816", "N817"] +"wagl/geobox.py" = ["N802", "N803", "N806", "N816", "N817"] +"wagl/interpolation.py" = ["N802", "N803", "N806", "N816", "N817"] +"wagl/land_sea_masking.py" = ["N802", "N803", "N806", "N816", "N817"] +"wagl/land_sea.py" = ["N802", "N803", "N806", "N816", "N817"] +"wagl/logs.py" = ["N802", "N803", "N806", "N816", "N817"] +"wagl/modtran.py" = ["N802", "N803", "N806", "N816", "N817"] +"wagl/pq.py" = ["N802", "N803", "N806", "N816", "N817"] +"wagl/reflectance.py" = ["N802", "N803", "N806", "N816", "N817"] +"wagl/standardise.py" = ["N802", "N803", "N806", "N816", "N817"] +"wagl/temperature.py" = ["N802", "N803", "N806", "N816", "N817"] +"wagl/unittesting_tools.py" = ["N802", "N803", "N806", "N816", "N817"] +"wagl/vincenty.py" = ["N802", "N803", "N806", "N816", "N817"] + [tool.black] line-length = 90 @@ -40,3 +67,4 @@ exclude = ''' [tool.setuptools_scm] write_to = "wagl/version.py" version_scheme = "post-release" + From f3c43f688935f5499cbbb2d0a4433c5b15a062c7 Mon Sep 17 00:00:00 2001 From: Jeremy Hooke Date: Fri, 14 Jul 2023 13:26:57 +1000 Subject: [PATCH 3/9] Need a py38-supporting scipy version Scipy 1.11 now requires Py3.9+ --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 4dd53782..7ef27d85 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,3 +14,4 @@ numexpr==2.7.3 structlog==21.1.0 PyYAML==5.4.1 pytest==6.2.4 +scipy==1.10.1 From 8af91f48e3e8ca97b8a969414369fa14d5ef4628 Mon Sep 17 00:00:00 2001 From: Jeremy Hooke Date: Fri, 14 Jul 2023 13:28:30 +1000 Subject: [PATCH 4/9] Install scipy via condaforge for stability It will be built to match Python/etc versions. --- Dockerfile | 1 + requirements.txt | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index eee03dc0..0eaaa036 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,6 +28,7 @@ RUN chmod +x /root/miniconda.sh && /root/miniconda.sh -b -f -p conda # GDAL 3.1 is being used because https://gdal.org/api/python.html#usage RUN conda install -c conda-forge \ gdal==3.1.4 \ + scipy \ python-fmask==0.5.5 WORKDIR ${WAGL_DIR} diff --git a/requirements.txt b/requirements.txt index 7ef27d85..4dd53782 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,4 +14,3 @@ numexpr==2.7.3 structlog==21.1.0 PyYAML==5.4.1 pytest==6.2.4 -scipy==1.10.1 From 1c9ae2319cd2098a91059429d5d1be3a482a1600 Mon Sep 17 00:00:00 2001 From: Jeremy Hooke Date: Fri, 14 Jul 2023 13:47:28 +1000 Subject: [PATCH 5/9] Update conda version --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 0eaaa036..0024d6f0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,7 +22,7 @@ RUN ln -s $(which gfortran-10) $(which gfortran-10 | sed 's/\(.*\)\/\gfortran-10 WORKDIR ${BUILD_DIR} # Bump this when newer versions of python are required -ADD https://repo.continuum.io/miniconda/Miniconda3-py38_4.8.2-Linux-x86_64.sh /root/miniconda.sh +ADD https://repo.continuum.io/miniconda/Miniconda3-py38_23.5.2-Linux-x86_64.sh /root/miniconda.sh RUN chmod +x /root/miniconda.sh && /root/miniconda.sh -b -f -p conda # GDAL 3.1 is being used because https://gdal.org/api/python.html#usage From ceea3f387d64f2a7b9625dc1b2757d302a8f5782 Mon Sep 17 00:00:00 2001 From: Jeremy Hooke Date: Fri, 14 Jul 2023 13:51:58 +1000 Subject: [PATCH 6/9] Allow current library versions. --- Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 0024d6f0..42fc11d7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -35,8 +35,11 @@ WORKDIR ${WAGL_DIR} ADD . ./ # Install dependencies required for unittests -RUN pip install -r requirements.txt +# RUN pip install -r requirements.txt RUN pip install git+https://github.com/sixy6e/idl-functions.git#egg=master +RUN pip install -e . +# Print the deps used. +RUN pip freeze # include basic details for diagnostics RUN which python; python --version; From 9a0b95734a417db85ee3bc569f3d97938f556ab2 Mon Sep 17 00:00:00 2001 From: Jeremy Hooke Date: Fri, 14 Jul 2023 13:55:20 +1000 Subject: [PATCH 7/9] Use actual miniconda version --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 42fc11d7..9b953d25 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,7 +22,7 @@ RUN ln -s $(which gfortran-10) $(which gfortran-10 | sed 's/\(.*\)\/\gfortran-10 WORKDIR ${BUILD_DIR} # Bump this when newer versions of python are required -ADD https://repo.continuum.io/miniconda/Miniconda3-py38_23.5.2-Linux-x86_64.sh /root/miniconda.sh +ADD https://repo.anaconda.com/miniconda/Miniconda3-py38_23.5.2-0-Linux-x86_64.sh /root/miniconda.sh RUN chmod +x /root/miniconda.sh && /root/miniconda.sh -b -f -p conda # GDAL 3.1 is being used because https://gdal.org/api/python.html#usage From d51f66316f88ac018998b10f05b2c7af55dc23d6 Mon Sep 17 00:00:00 2001 From: Jeremy Hooke Date: Fri, 14 Jul 2023 15:08:16 +1000 Subject: [PATCH 8/9] Original conda version --- Dockerfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9b953d25..19596d0b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,7 +22,7 @@ RUN ln -s $(which gfortran-10) $(which gfortran-10 | sed 's/\(.*\)\/\gfortran-10 WORKDIR ${BUILD_DIR} # Bump this when newer versions of python are required -ADD https://repo.anaconda.com/miniconda/Miniconda3-py38_23.5.2-0-Linux-x86_64.sh /root/miniconda.sh +ADD https://repo.continuum.io/miniconda/Miniconda3-py38_4.8.2-Linux-x86_64.sh /root/miniconda.sh RUN chmod +x /root/miniconda.sh && /root/miniconda.sh -b -f -p conda # GDAL 3.1 is being used because https://gdal.org/api/python.html#usage @@ -35,10 +35,8 @@ WORKDIR ${WAGL_DIR} ADD . ./ # Install dependencies required for unittests -# RUN pip install -r requirements.txt RUN pip install git+https://github.com/sixy6e/idl-functions.git#egg=master RUN pip install -e . -# Print the deps used. RUN pip freeze # include basic details for diagnostics From 45cc897b1a9692def87e367f7540ee58f6e0a65a Mon Sep 17 00:00:00 2001 From: Jeremy Hooke Date: Fri, 14 Jul 2023 15:19:50 +1000 Subject: [PATCH 9/9] Certain libraries need matching versions --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 19596d0b..d24bb814 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,8 +28,8 @@ RUN chmod +x /root/miniconda.sh && /root/miniconda.sh -b -f -p conda # GDAL 3.1 is being used because https://gdal.org/api/python.html#usage RUN conda install -c conda-forge \ gdal==3.1.4 \ - scipy \ - python-fmask==0.5.5 + python-fmask==0.5.5 \ + scipy pandas scikit-image WORKDIR ${WAGL_DIR} ADD . ./