Skip to content

Added operations for the spectrum class #20

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
fnemina wants to merge 6 commits into
base: dev
Choose a base branch
from
Open

Added operations for the spectrum class #20

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
c321d14
Added operations for the spectrum class
fnemina May 26, 2022
27aea21
Spectral convolution
fnemina Sep 12, 2022
21ef24c
Implemented the abilty to force the reader
fnemina Sep 12, 2022
d2c55cb
Implemented savgol filter
fnemina Sep 12, 2022
0afcc97
Improved general reader
fnemina Sep 12, 2022
1decf68
Improving subseting of wavelength ranges.
fnemina Sep 16, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,8 @@ __pycache__*
*~
*.pyc
/dist/
/build/
/.vscode/
/*.egg-info
*.ipynb_checkpoints*
*.ipynb_checkpoints*
.DS_Store
2 changes: 2 additions & 0 deletions MANIFEST.in
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
# we include the rsr
include specdal/rsr/*.nc
1 change: 1 addition & 0 deletions setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
author_email='ylee546@wisc.edu',
license='MIT',
packages=['specdal', 'specdal.utils'],
include_package_data=True,
install_requires=['numpy', 'pandas', 'matplotlib', 'scipy'],
zip_safe=False,
classifiers=[
Expand Down
43 changes: 38 additions & 5 deletions specdal/collection.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,14 +93,14 @@ class Collection(object):
"""
Represents a dataset consisting of a collection of spectra
"""
def __init__(self, name, directory=None, spectra=None,
measure_type='pct_reflect', metadata=None):
def __init__(self, name, directory=None, spectra=None, ext=[".asd", ".sed", ".sig"],
measure_type='pct_reflect', metadata=None, reader=None):
self.name = name
self.spectra = spectra
self.measure_type = measure_type
self.metadata = metadata
if directory:
self.read(directory, measure_type)
self.read(directory, measure_type, ext=ext, reader=reader)
@property
def spectra(self):
"""
Expand Down Expand Up @@ -187,7 +187,7 @@ def __contains__(self, item):
# reader
def read(self, directory, measure_type='pct_reflect',
ext=[".asd", ".sed", ".sig"], recursive=False,
verbose=False):
verbose=False, reader=None):
"""
read all files in a path matching extension
"""
Expand All @@ -204,7 +204,7 @@ def read(self, directory, measure_type='pct_reflect',
filepath = os.path.join(dirpath, f)
spectrum = Spectrum(name=f_name, filepath=filepath,
measure_type=measure_type,
verbose=verbose)
verbose=verbose, reader=reader)
self.append(spectrum)
##################################################
# wrapper around spectral operations
Expand Down Expand Up @@ -318,4 +318,37 @@ def std(self, append=False):
if append:
self.append(spectrum)
return spectrum
##################################################
# method for computing the values for a specific satellite

def getSatellite(self, satellite="Aqua", sensor="MODIS", rsr_path = __file__.replace("/collection.py","/rsr/")):
c_tmp = Collection(name=self.name, metadata={})
c_tmp.metadata["satellite"] = satellite
c_tmp.metadata["sensor"] = sensor
# compute reflectance by bande
size_compute = len(self.spectra)
i = 1
# We iterate over all spectra to compute the reflectance per band
for spectra_tmp in self.spectra:
# we print current spectra
print(f"Spectra {i}/{size_compute}.", end="\r")
c_tmp.append(spectra_tmp.getSatellite(satellite, sensor, rsr_path))
i+=1

return c_tmp

def savgol_filter(self, window_length, polyorder, deriv=0,
delta=1.0, axis=-1, mode='interp', cval=0.0):
self.metadata["savgol_window_length"] = window_length
self.metadata["savgol_polyorder"] = polyorder

# We iterate over all spectra
for spectra_tmp in self.spectra:
spectra_tmp.savgol_filter(window_length,
polyorder, deriv, delta, axis, mode, cval)

def walevength_range(self, wlmin=350, wlmax=2500, dtype=None):
# We iterate over all spectra
for spectra_tmp in self.spectra:
spectra_tmp.measurement = spectra_tmp.measurement.loc[wlmin:wlmax]
spectra_tmp.metadata['wavelength_range'] = (wlmin, wlmax)
15 changes: 15 additions & 0 deletions specdal/operator.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
# wavelength as index and measurement as values
import pandas as pd
import numpy as np
from scipy.signal import savgol_filter
from .utils.misc import get_monotonic_series

################################################################################
Expand Down Expand Up @@ -114,6 +115,20 @@ def vector_normalize(series):
'''
return series / np.sqrt(np.sum(series ** 2))

################################################################################
# vector_normalize: divide the series such that 2-norm(series) == 1
def savgol(series, window_length, polyorder, deriv=0,
delta=1.0, axis=-1, mode='interp', cval=0.0):
'''
Savitzky-Golay filter using scipy implementation
'''
smooth = savgol_filter(series.values,
window_length, polyorder, deriv,
delta, axis, mode, cval)
smooth = pd.Series(data=smooth, index=series.index)
smooth.index.name = "wavelength"
return smooth

################################################################################
# DataFrame operations (collection of spectra)
################################################################################
Expand Down
24 changes: 21 additions & 3 deletions specdal/reader.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,19 +1,25 @@
# readers.py provides functions to read spectrum files for data and
# metadata.

import string
import pandas as pd
import numpy as np
from os.path import abspath, expanduser, splitext
from collections import OrderedDict
from .utils.reader_utils import *

def read(filepath, read_data=True, read_metadata=True, verbose=False):
def read(filepath, read_data=True, read_metadata=True, verbose=False, reader=None):
"""
Calls the appropriate reader based on file extension
"""
SUPPORTED_READERS = {'.asd':read_asd , '.sig':read_sig , '.sed':read_sed }
ext = splitext(filepath)[1]
reader = SUPPORTED_READERS[ext]
# Added option to force reader
if reader == None:
ext = splitext(filepath)[1]
reader = SUPPORTED_READERS[ext]
else:
reader = SUPPORTED_READERS[reader]

return reader(abspath(expanduser(filepath)), read_data,
read_metadata, verbose)

Expand Down Expand Up @@ -193,12 +199,24 @@ def read_asd(filepath, read_data=True, read_metadata=True, verbose=False):
gps_flags2 = gps_struct[9:11] # unpack this into bits
gps_satellites = gps_struct[11:16]
gps_filler = gps_struct[16:18]
# date
time_lock = 160
second = int.from_bytes(binconts[160:(160 + 2)], byteorder='little')
minute = int.from_bytes(binconts[160+2:160+2+2], byteorder='little')
hour = int.from_bytes(binconts[160+4:160+4+2], byteorder='little')
day = int.from_bytes(binconts[160+6:160+6+2], byteorder='little')
month = int.from_bytes(binconts[160+8:160+8+2], byteorder='little')+1
year = int.from_bytes(binconts[160+10:160+10+2], byteorder='little')+1900
# Dark current
dc = int.from_bytes(binconts[181:181+1], byteorder='little')
# metadata
metadata['integration_time'] = integration_time
metadata['measurement_type'] = spectrum_type
metadata['gps_time_tgt'] = gps_timestamp
metadata['gps_time_ref'] = None
metadata['wavelength_range'] = (wavestart, wavestop)
metadata['measurement_date'] = f"{year:04}-{month:02}-{day:02} {hour:02}:{minute:02}:{second:02}"
metadata['dark_current'] = dc
# metadata['splice'] = (splice1, splice2)
# metadata['resolution'] = wavestep
return data, metadata
Binary file added specdal/rsr/adeos_octs_RSR.nc
View file
Open in desktop
Binary file not shown.
Binary file added specdal/rsr/aqua_modis_RSR.nc
View file
Open in desktop
Binary file not shown.
Binary file added specdal/rsr/aviris_RSR.nc
View file
Open in desktop
Binary file not shown.
Binary file added specdal/rsr/coms_goci_RSR.nc
View file
Open in desktop
Binary file not shown.
Binary file added specdal/rsr/envisat_meris_RSR.nc
View file
Open in desktop
Binary file not shown.
Binary file added specdal/rsr/gcomc_sgli_RSR.nc
View file
Open in desktop
Binary file not shown.
Binary file added specdal/rsr/irs-p4_ocm_RSR.nc
View file
Open in desktop
Binary file not shown.
Binary file added specdal/rsr/iss_hico_RSR.nc
View file
Open in desktop
Binary file not shown.
Binary file added specdal/rsr/jpss-1_viirs_RSR.nc
View file
Open in desktop
Binary file not shown.
Binary file added specdal/rsr/landsat-8_oli_RSR.nc
View file
Open in desktop
Binary file not shown.
Binary file added specdal/rsr/nimbus-7_czcs_RSR.nc
View file
Open in desktop
Binary file not shown.
Binary file added specdal/rsr/oceansat-2_ocm-2_RSR.nc
View file
Open in desktop
Binary file not shown.
Binary file added specdal/rsr/orbview-2_seawifs_RSR.nc
View file
Open in desktop
Binary file not shown.
Binary file added specdal/rsr/pace_oci_RSR.nc
View file
Open in desktop
Binary file not shown.
Binary file added specdal/rsr/s3a_olci_RSR.nc
View file
Open in desktop
Binary file not shown.
Binary file added specdal/rsr/s3b_olci_RSR.nc
View file
Open in desktop
Binary file not shown.
Binary file added specdal/rsr/seahawk1_hawkeye_RSR.nc
View file
Open in desktop
Binary file not shown.
Binary file added specdal/rsr/sentinel-2a_msi_RSR.nc
View file
Open in desktop
Binary file not shown.
Binary file added specdal/rsr/sentinel-2b_msi_RSR.nc
View file
Open in desktop
Binary file not shown.
Binary file added specdal/rsr/suomi-npp_viirs_RSR.nc
View file
Open in desktop
Binary file not shown.
Binary file added specdal/rsr/terra_misr_RSR.nc
View file
Open in desktop
Binary file not shown.
Binary file added specdal/rsr/terra_modis_RSR.nc
View file
Open in desktop
Binary file not shown.
Loading