From 0df91aa8aa63a2dbbdc8670328e88f18635263f9 Mon Sep 17 00:00:00 2001 From: "Charles Graham, SWT" Date: Mon, 12 Aug 2024 14:06:10 -0500 Subject: [PATCH 01/17] Update ignore for this branch --- .gitignore | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index fe4672f..9a50205 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,11 @@ test_priv/* .vscode *.swp tests/actual -tests/output \ No newline at end of file +tests/output + +.pypirc +*.egg-info +dist +build +_build +.DS_Store \ No newline at end of file From 7a6ff0ff9a05d4fdd70a547a2654503464dc5211 Mon Sep 17 00:00:00 2001 From: "Charles Graham, SWT" Date: Mon, 12 Aug 2024 14:24:21 -0500 Subject: [PATCH 02/17] Handle invalid user input (no input file), ensure context manager is used for file to prevent open file descriptors --- __main__.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/__main__.py b/__main__.py index 8758ea1..7ad8a9b 100644 --- a/__main__.py +++ b/__main__.py @@ -3,7 +3,7 @@ from repgen.report import Report from repgen.util import filterAddress -version = "5.0.5" +version = "5.1.6" # setup base time, ex # default formats @@ -33,7 +33,7 @@ def parseArgs(): parser.print_help() exit(2) - return parser.parse_known_args()[0] + return parser # https://stackoverflow.com/a/52014520 def parse_var(s): @@ -92,8 +92,9 @@ def parse_vars(items): "EST": "EST5EDT", } -if __name__ == "__main__": - config = parseArgs() +def main(): + parser = parseArgs() + config = parser.parse_known_args()[0] kwargs = parse_vars(config.set)[0] if config.show_ver == True: @@ -105,7 +106,6 @@ def parse_vars(items): kwargs.pop("IN", None) # This doesn't need to be visible to the report definition kwargs.pop("REPORT", None) kwargs.pop("FILE", None) - (host, path) = filterAddress(config.host) (althost, altpath) = filterAddress(config.alternate) @@ -124,14 +124,21 @@ def parse_vars(items): # set some of the default values Value(1, host=host, path=path, tz=tz, ucformat=config.compat, timeout=config.timeout, althost=althost, altpath=altpath, dbofc=config.office, **kwargs) + report_data = None # read the report file if report_file == '-': report_file = sys.stdin.name f = sys.stdin else: - f = open(report_file, 'rt') - report_data = f.read() - f.close() + if not report_file: + parser.print_help() + print(f"\nError: No report file specified. You must specify a report file with the -i option\n") + sys.exit(1) + elif not os.path.exists(report_file): + print(f"\nError: Report file {report_file} does not exist\n") + sys.exit(1) + with open(report_file, 'rt') as f: + report_data = f.read() base_date = kwargs.get("DATE", config.base_date) base_time = kwargs.get("TIME", config.base_time) @@ -204,3 +211,6 @@ def parse_vars(items): mask = os.umask(0) os.chmod(out_file, 0o666 & (~mask)) os.umask(mask) + +if __name__ == "__main__": + main() \ No newline at end of file From 878d33787fd0ad14fe63bc93a39de183ebc1d9cd Mon Sep 17 00:00:00 2001 From: "Charles Graham, SWT" Date: Mon, 12 Aug 2024 14:43:14 -0500 Subject: [PATCH 03/17] Setup package format, add dev script --- .gitignore | 3 +- __main__.py | 217 +-------------------------------------------- repgen/__main__.py | 216 ++++++++++++++++++++++++++++++++++++++++++++ scripts/dev.bat | 4 + setup.py | 44 +++++++++ 5 files changed, 269 insertions(+), 215 deletions(-) create mode 100644 repgen/__main__.py create mode 100644 scripts/dev.bat create mode 100644 setup.py diff --git a/.gitignore b/.gitignore index 9a50205..2e98d2a 100644 --- a/.gitignore +++ b/.gitignore @@ -15,4 +15,5 @@ tests/output dist build _build -.DS_Store \ No newline at end of file +.DS_Store +venv \ No newline at end of file diff --git a/__main__.py b/__main__.py index 7ad8a9b..427f525 100644 --- a/__main__.py +++ b/__main__.py @@ -1,216 +1,5 @@ -import sys,time,datetime,pytz,tempfile,shutil,os -from repgen.data.value import Value -from repgen.report import Report -from repgen.util import filterAddress - -version = "5.1.6" - -# setup base time, ex -# default formats -def parseArgs(): - import argparse - parser=argparse.ArgumentParser() - _z = os.environ.get("TZ", None) # Get TZ from environment, if specified - dt = datetime.datetime.now().astimezone() - _d = dt.strftime("%d%b%Y") - _t = dt.strftime("%H%M") - parser.add_argument( '-V', '--version',dest='show_ver',action='store_true',default=False, help="print version number") - parser.add_argument( '-i', '--in', dest='in_file', help="INput report file", metavar="REPFILE" ) - parser.add_argument( '-o', '--out', dest='out_file', default="-", help="OUTput file with filled report", metavar="REPOUTPUT") - parser.add_argument( '-f', '--file', dest='data_file', default=None, help="Variable data file", metavar="DATAFILE" ) - parser.add_argument( '-d', '--date', dest='base_date', default=_d, help="base date for data", metavar="DDMMMYYYY" ) - parser.add_argument( '-t', '--time', dest='base_time', default=_t, help="base time for data", metavar="HHMM") - parser.add_argument( '-z', '--tz', dest='tz', default=_z, help="default timezone; equivalent to `TZ=timezone`", metavar='Time Zone Name') - parser.add_argument( '-O', '--office', dest='office', default=None, help="default office to use if not specified in report; equivalent to `DBOFC=OFFICE_ID`", metavar='OFFICE_ID') - parser.add_argument( '-a', '--address', dest='host', default='localhost', help="location for data connections; equivalent to `DB=hostname:port/path`", metavar='IP_or_hostname:port[/basepath]') - parser.add_argument( '-A', '--alternate', dest='alternate', default=None, help="alternate location for data connections, if the primary is unavailable (only for CDA)", metavar='IP_or_hostname:port[/basepath]') - parser.add_argument( '-c', '--compatibility', dest='compat', action="store_true", default=False, help="repgen4 compatibility; case-insensitive labels") - parser.add_argument( '--timeout', dest='timeout', type=float, default=None, help="Socket timeout, in seconds" ) - # This provides repgen4 style KEY=VALUE argument passing on the command-line - parser.add_argument( 'set', default=[], help="Additional key=value pairs. e.g. `DBTZ=UTC DBOFC=HEC`", metavar="KEY=VALUE", nargs="*" ) - - if len(sys.argv) == 1: - parser.print_help() - exit(2) - - return parser - -# https://stackoverflow.com/a/52014520 -def parse_var(s): - """ - Parse a key, value pair, separated by '=' - That's the reverse of ShellArgs. - - On the command line (argparse) a declaration will typically look like: - foo=hello - or - foo="hello world" - """ - items = s.split('=') - key = items[0].strip() # we remove blanks around keys, as is logical - if len(items) > 1: - # rejoin the rest: - value = '='.join(items[1:]) - return (key, value) - -def parse_vars(items): - """ - Parse a series of key-value pairs and return a dictionary and - a success boolean for whether each item was successfully parsed. - """ - count = 0 - d = {} - for item in items: - if "=" in item: - split_string = item.split("=") - d[split_string[0].strip().upper()] = split_string[1].strip() - count += 1 - else: - print(f"Error: Invalid argument provided - {item}") - - return d, count == len(items) - -# Pytz doesn't know all the aliases and abbreviations -# This works for Pacific, but untested in other locations that don't use DST. -TIMEZONE_ALIASES = { - "Pacific Standard Time": "PST8PDT", - "Pacific Daylight Time": "PST8PDT", - "Mountain Standard Time": "MST7MDT", - "Mountain Daylight Time": "MST7MDT", - "Central Standard Time": "CST6CDT", - "Central Daylight Time": "CST6CDT", - "Eastern Standard Time": "EST5EDT", - "Eastern Daylight Time": "EST5EDT", - - "PST": "PST8PDT", - "PDT": "PST8PDT", - "MST": "MST7MDT", - "MDT": "MST7MDT", - "CST": "CST6CDT", - "CDT": "CST6CDT", - "EDT": "EST5EDT", - "EST": "EST5EDT", -} - -def main(): - parser = parseArgs() - config = parser.parse_known_args()[0] - kwargs = parse_vars(config.set)[0] - - if config.show_ver == True: - print(version) - sys.exit(0) - - report_file = kwargs.get("IN", config.in_file) - out_file = kwargs.get("REPORT", config.out_file) - kwargs.pop("IN", None) # This doesn't need to be visible to the report definition - kwargs.pop("REPORT", None) - kwargs.pop("FILE", None) - (host, path) = filterAddress(config.host) - (althost, altpath) = filterAddress(config.alternate) - - tz = None - - if config.tz: - tz = pytz.timezone(config.tz) - - if not tz: - # Default to the system timezone - # Convert system timezone name to pytz compatible name - # This might fail if TIMEZONE_ALIASES is missing an entry, in which case, using the --tz argument will skip this - tz = str(datetime.datetime.now().astimezone().tzinfo) - tz = pytz.timezone(TIMEZONE_ALIASES.get(tz, tz)) - - # set some of the default values - Value(1, host=host, path=path, tz=tz, ucformat=config.compat, timeout=config.timeout, althost=althost, altpath=altpath, dbofc=config.office, **kwargs) - - report_data = None - # read the report file - if report_file == '-': - report_file = sys.stdin.name - f = sys.stdin - else: - if not report_file: - parser.print_help() - print(f"\nError: No report file specified. You must specify a report file with the -i option\n") - sys.exit(1) - elif not os.path.exists(report_file): - print(f"\nError: Report file {report_file} does not exist\n") - sys.exit(1) - with open(report_file, 'rt') as f: - report_data = f.read() - - base_date = kwargs.get("DATE", config.base_date) - base_time = kwargs.get("TIME", config.base_time) - delta = datetime.timedelta() - - if base_time == "2400": - base_time = "0000" - delta = datetime.timedelta(days=1) - - _t = datetime.datetime.strptime(base_date + " " + base_time , "%d%b%Y %H%M" ) + delta - - print( repr(_t), file=sys.stderr ) - - basedate = _t - - local_vars = {} - # Read data file input - data_file = kwargs.get("FILE", config.data_file) - if data_file: - f_d = open(config.data_file) - key = None - prefix = "" - - # This processes data file inputs, and converts ^a variables to _a. - # Format of this file is: - # ^ - # a - # Some Value - # b - # Another value - for line in f_d.readlines(): - line = line.strip() - if line == "^": - prefix = "_" - elif not key: - key = prefix + line - else: - # Check to see if the read in value is really a number, and convert it if so - val = line.strip('"') if '=' not in line else line - try: - if '.' in val: - val = float(val) - else: - val = int(val) - except (TypeError, ValueError): - pass - local_vars[key] = val - key = None - - f_d.close() - - # exec the definitions - report = Report(report_data, report_file, config.compat) - report.run(basedate, local_vars) - output = None - tmpname = None - - if out_file == "-": - output = sys.stdout - else: - fd,tmpname = tempfile.mkstemp(text=True, prefix="repgen-") - output = os.fdopen(fd,"wt") - - # build the report - report.fill_report(output) - - if out_file != "-": - output.close() - shutil.move(tmpname,out_file) - mask = os.umask(0) - os.chmod(out_file, 0o666 & (~mask)) - os.umask(mask) +# Forward to repgen.__main__ for backwards compatibility +import runpy if __name__ == "__main__": - main() \ No newline at end of file + runpy.run_module("repgen.__main__", run_name="__main__") \ No newline at end of file diff --git a/repgen/__main__.py b/repgen/__main__.py new file mode 100644 index 0000000..cce0ca6 --- /dev/null +++ b/repgen/__main__.py @@ -0,0 +1,216 @@ +import sys,datetime,pytz,tempfile,shutil,os +from repgen.data.value import Value +from repgen.report import Report +from repgen.util import filterAddress + +version = "5.1.6" + +# setup base time, ex +# default formats +def parseArgs(): + import argparse + parser=argparse.ArgumentParser() + _z = os.environ.get("TZ", None) # Get TZ from environment, if specified + dt = datetime.datetime.now().astimezone() + _d = dt.strftime("%d%b%Y") + _t = dt.strftime("%H%M") + parser.add_argument( '-V', '--version',dest='show_ver',action='store_true',default=False, help="print version number") + parser.add_argument( '-i', '--in', dest='in_file', help="INput report file", metavar="REPFILE" ) + parser.add_argument( '-o', '--out', dest='out_file', default="-", help="OUTput file with filled report", metavar="REPOUTPUT") + parser.add_argument( '-f', '--file', dest='data_file', default=None, help="Variable data file", metavar="DATAFILE" ) + parser.add_argument( '-d', '--date', dest='base_date', default=_d, help="base date for data", metavar="DDMMMYYYY" ) + parser.add_argument( '-t', '--time', dest='base_time', default=_t, help="base time for data", metavar="HHMM") + parser.add_argument( '-z', '--tz', dest='tz', default=_z, help="default timezone; equivalent to `TZ=timezone`", metavar='Time Zone Name') + parser.add_argument( '-O', '--office', dest='office', default=None, help="default office to use if not specified in report; equivalent to `DBOFC=OFFICE_ID`", metavar='OFFICE_ID') + parser.add_argument( '-a', '--address', dest='host', default='localhost', help="location for data connections; equivalent to `DB=hostname:port/path`", metavar='IP_or_hostname:port[/basepath]') + parser.add_argument( '-A', '--alternate', dest='alternate', default=None, help="alternate location for data connections, if the primary is unavailable (only for CDA)", metavar='IP_or_hostname:port[/basepath]') + parser.add_argument( '-c', '--compatibility', dest='compat', action="store_true", default=False, help="repgen4 compatibility; case-insensitive labels") + parser.add_argument( '--timeout', dest='timeout', type=float, default=None, help="Socket timeout, in seconds" ) + # This provides repgen4 style KEY=VALUE argument passing on the command-line + parser.add_argument( 'set', default=[], help="Additional key=value pairs. e.g. `DBTZ=UTC DBOFC=HEC`", metavar="KEY=VALUE", nargs="*" ) + + if len(sys.argv) == 1: + parser.print_help() + exit(2) + + return parser + +# https://stackoverflow.com/a/52014520 +def parse_var(s): + """ + Parse a key, value pair, separated by '=' + That's the reverse of ShellArgs. + + On the command line (argparse) a declaration will typically look like: + foo=hello + or + foo="hello world" + """ + items = s.split('=') + key = items[0].strip() # we remove blanks around keys, as is logical + if len(items) > 1: + # rejoin the rest: + value = '='.join(items[1:]) + return (key, value) + +def parse_vars(items): + """ + Parse a series of key-value pairs and return a dictionary and + a success boolean for whether each item was successfully parsed. + """ + count = 0 + d = {} + for item in items: + if "=" in item: + split_string = item.split("=") + d[split_string[0].strip().upper()] = split_string[1].strip() + count += 1 + else: + print(f"Error: Invalid argument provided - {item}") + + return d, count == len(items) + +# Pytz doesn't know all the aliases and abbreviations +# This works for Pacific, but untested in other locations that don't use DST. +TIMEZONE_ALIASES = { + "Pacific Standard Time": "PST8PDT", + "Pacific Daylight Time": "PST8PDT", + "Mountain Standard Time": "MST7MDT", + "Mountain Daylight Time": "MST7MDT", + "Central Standard Time": "CST6CDT", + "Central Daylight Time": "CST6CDT", + "Eastern Standard Time": "EST5EDT", + "Eastern Daylight Time": "EST5EDT", + + "PST": "PST8PDT", + "PDT": "PST8PDT", + "MST": "MST7MDT", + "MDT": "MST7MDT", + "CST": "CST6CDT", + "CDT": "CST6CDT", + "EDT": "EST5EDT", + "EST": "EST5EDT", +} + +def main(): + parser = parseArgs() + config = parser.parse_known_args()[0] + kwargs = parse_vars(config.set)[0] + + if config.show_ver == True: + print(version) + sys.exit(0) + + report_file = kwargs.get("IN", config.in_file) + out_file = kwargs.get("REPORT", config.out_file) + kwargs.pop("IN", None) # This doesn't need to be visible to the report definition + kwargs.pop("REPORT", None) + kwargs.pop("FILE", None) + (host, path) = filterAddress(config.host) + (althost, altpath) = filterAddress(config.alternate) + + tz = None + + if config.tz: + tz = pytz.timezone(config.tz) + + if not tz: + # Default to the system timezone + # Convert system timezone name to pytz compatible name + # This might fail if TIMEZONE_ALIASES is missing an entry, in which case, using the --tz argument will skip this + tz = str(datetime.datetime.now().astimezone().tzinfo) + tz = pytz.timezone(TIMEZONE_ALIASES.get(tz, tz)) + + # set some of the default values + Value(1, host=host, path=path, tz=tz, ucformat=config.compat, timeout=config.timeout, althost=althost, altpath=altpath, dbofc=config.office, **kwargs) + + report_data = None + # read the report file + if report_file == '-': + report_file = sys.stdin.name + f = sys.stdin + else: + if not report_file: + parser.print_help() + print(f"\nError: No report file specified. You must specify a report file with the -i option\n") + sys.exit(1) + elif not os.path.exists(report_file): + print(f"\nError: Report file {report_file} does not exist\n") + sys.exit(1) + with open(report_file, 'rt') as f: + report_data = f.read() + + base_date = kwargs.get("DATE", config.base_date) + base_time = kwargs.get("TIME", config.base_time) + delta = datetime.timedelta() + + if base_time == "2400": + base_time = "0000" + delta = datetime.timedelta(days=1) + + _t = datetime.datetime.strptime(base_date + " " + base_time , "%d%b%Y %H%M" ) + delta + + print( repr(_t), file=sys.stderr ) + + basedate = _t + + local_vars = {} + # Read data file input + data_file = kwargs.get("FILE", config.data_file) + if data_file: + f_d = open(config.data_file) + key = None + prefix = "" + + # This processes data file inputs, and converts ^a variables to _a. + # Format of this file is: + # ^ + # a + # Some Value + # b + # Another value + for line in f_d.readlines(): + line = line.strip() + if line == "^": + prefix = "_" + elif not key: + key = prefix + line + else: + # Check to see if the read in value is really a number, and convert it if so + val = line.strip('"') if '=' not in line else line + try: + if '.' in val: + val = float(val) + else: + val = int(val) + except (TypeError, ValueError): + pass + local_vars[key] = val + key = None + + f_d.close() + + # exec the definitions + report = Report(report_data, report_file, config.compat) + report.run(basedate, local_vars) + output = None + tmpname = None + + if out_file == "-": + output = sys.stdout + else: + fd,tmpname = tempfile.mkstemp(text=True, prefix="repgen-") + output = os.fdopen(fd,"wt") + + # build the report + report.fill_report(output) + + if out_file != "-": + output.close() + shutil.move(tmpname,out_file) + mask = os.umask(0) + os.chmod(out_file, 0o666 & (~mask)) + os.umask(mask) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/dev.bat b/scripts/dev.bat new file mode 100644 index 0000000..e9ddea5 --- /dev/null +++ b/scripts/dev.bat @@ -0,0 +1,4 @@ +@REM Sets up the repgen5 environment and loads the package in development mode + +python -m pip install -r requirements.txt +python -m pip install -e . \ No newline at end of file diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..478e7df --- /dev/null +++ b/setup.py @@ -0,0 +1,44 @@ +from setuptools import setup, find_packages +from repgen.__main__ import version + +with open("README.md", "r", encoding="utf-8") as fh: + long_description = fh.read() + +setup( + name="cwms-repgen", + version=version, + license="MIT", + author="USACE-HEC", + description="""This is a partial copy of HEC's (Hydrologic Engineering Center) repgen program. +The program creates fixed form text reports from a time series database, and textfiles.""", + long_description=long_description, + long_description_content_type="text/markdown", + url="https://github.com/USACE-WaterManagement/repgen5", + packages=find_packages(exclude=["tests", "docs"]), + classifiers=[ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + ], + python_requires=">=3.8", + install_requires=[ + "pytz", + "python-dateutil>=2.8.2", + ], + extras_require={ + "dev": [ + "sphinx", + "sphinx_rtd_theme", + "sphinx-copybutton", + "myst-parser", + "twine", + "wheel", + ], + }, + entry_points={ + "console_scripts": [ + "repgen5=repgen.__main__:main", + "repgen=repgen.__main__:main", + ], + }, +) From 35ccf5b7190ea7630e728463b25e45284f4cdc0f Mon Sep 17 00:00:00 2001 From: "Charles Graham, SWT" Date: Mon, 12 Aug 2024 14:43:35 -0500 Subject: [PATCH 04/17] Catch when a user enters IN=- with no stdin and other None report input handling --- repgen/__init__.py | 1 + repgen/report/report.py | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/repgen/__init__.py b/repgen/__init__.py index e69de29..2cc2870 100644 --- a/repgen/__init__.py +++ b/repgen/__init__.py @@ -0,0 +1 @@ +REPGEN_DOCS_URL="https://repgen5.readthedocs.io/" \ No newline at end of file diff --git a/repgen/report/report.py b/repgen/report/report.py index 5885e62..db00180 100644 --- a/repgen/report/report.py +++ b/repgen/report/report.py @@ -1,5 +1,6 @@ -import sys,time,datetime,pytz,tempfile,shutil,os,operator,calendar,re +import sys, datetime, os from repgen.data.value import Value +from repgen import REPGEN_DOCS_URL try: # Relativedelta supports months and years, but is external library from dateutil.relativedelta import relativedelta as timedelta @@ -129,6 +130,8 @@ def _validate_report(self, report: str) -> None: ValueError: If the report does not contain a '#FORM'/'#ENDFORM' or '#FORMFILE' tag. """ # TODO: What else should we be validating in a report? + if not report: + raise ValueError(f"\n\tReport is empty. See the documentation for how to create a report.\n\n\tCtrl + Click => {REPGEN_DOCS_URL}") # Pick one or the other, not both if ("#FORM\n" in report or "#ENDFORM\n" in report) and "#FORMFILE" in report: raise ValueError("\n\tReport contains both a #FORM/#ENDFORM and a #FORMFILE tag. You must choose ONE.\n") From 40dbcb2602a72fefa53ec84204757269f8dc44b5 Mon Sep 17 00:00:00 2001 From: "Charles Graham, SWT" Date: Mon, 12 Aug 2024 15:13:48 -0500 Subject: [PATCH 05/17] Add DB types currently available as an internal catch for when an unknown type is provided --- repgen/data/value.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/repgen/data/value.py b/repgen/data/value.py index 4e447c2..636eaab 100644 --- a/repgen/data/value.py +++ b/repgen/data/value.py @@ -57,6 +57,15 @@ class Value: "use_alternate": False, # Alternate server in use, if primary is unavailable } + DB_TYPES = [ + "CDA", + "SPKJSON", + "JSON", + "GENTS", + "TEXT", + "FILE", + "DSS" + ] # This isn't thread safe, not an issue yet though since repgen isn't multithreaded. _conn = None @@ -280,12 +289,12 @@ def processDateTime(value, key, extra_part=None): self.type = "TIMESERIES" self.values = [ ] # will be a tuple of (time stamp, value, quality ) - + if self.dbtype is None: raise ValueError("you must enter a scalar quantity if you aren't specifying a data source") # TODO: Remove this at some point? # Conversion with a warning to change the dbtype from radar to CDA for rebrand - elif self.dbtype.upper() == "radar": + elif self.dbtype.upper() == "RADAR": print("\n\tWARNING: Update from dbtype=\"RADAR\" to dbtype=\"CDA\"") self.dbtype = "CDA" elif self.dbtype.upper() == "FILE": @@ -586,7 +595,7 @@ def parse_slice(value): elif self.dbtype.upper() == "DSS": raise Exception("DSS retrieval is not currently implemented") else: - raise Exception(f"\n\n\t{self.dbtype.upper()} is not supported!\n\tAvailable options are:\n\t\t {', '.join(self.DB_OPTIONS)}\n") + raise Exception(f"\n\n\t{self.dbtype.upper()} is not supported!\n\tAvailable options are:\n\t\t {', '.join(self.DB_TYPES)}\n") # math functions def __add__( self, other ): From caa50c6a0e9e342ff2e242b9485c9f3929be4dba Mon Sep 17 00:00:00 2001 From: "Charles Graham, SWT" Date: Mon, 12 Aug 2024 15:27:18 -0500 Subject: [PATCH 06/17] Fix bug where warning was presented but script did not continue for dbtype/rebrand update --- repgen/data/value.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/repgen/data/value.py b/repgen/data/value.py index 636eaab..1fcb1ef 100644 --- a/repgen/data/value.py +++ b/repgen/data/value.py @@ -289,14 +289,11 @@ def processDateTime(value, key, extra_part=None): self.type = "TIMESERIES" self.values = [ ] # will be a tuple of (time stamp, value, quality ) - - if self.dbtype is None: - raise ValueError("you must enter a scalar quantity if you aren't specifying a data source") - # TODO: Remove this at some point? - # Conversion with a warning to change the dbtype from radar to CDA for rebrand - elif self.dbtype.upper() == "RADAR": + if self.dbtype.upper() == "RADAR": print("\n\tWARNING: Update from dbtype=\"RADAR\" to dbtype=\"CDA\"") self.dbtype = "CDA" + if self.dbtype is None: + raise ValueError("you must enter a scalar quantity if you aren't specifying a data source") elif self.dbtype.upper() == "FILE": pass elif self.dbtype.upper() == "COPY": From 9192e3df1b71e6be5a6e306652dd8cb2705b45d8 Mon Sep 17 00:00:00 2001 From: "Charles Graham, SWT" Date: Mon, 12 Aug 2024 15:28:35 -0500 Subject: [PATCH 07/17] Add default endpoint, check if prod (national) is provided and remind user to specify the office --- repgen/__init__.py | 3 ++- repgen/__main__.py | 11 ++++++++--- tests/various/tsid/testfiles/swt.keystone.frm | 1 - 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/repgen/__init__.py b/repgen/__init__.py index 2cc2870..ef334f1 100644 --- a/repgen/__init__.py +++ b/repgen/__init__.py @@ -1 +1,2 @@ -REPGEN_DOCS_URL="https://repgen5.readthedocs.io/" \ No newline at end of file +REPGEN_DOCS_URL="https://repgen5.readthedocs.io/" +PROD_CDA_HOST = "https://cwms-data.usace.army.mil/cwms-data" \ No newline at end of file diff --git a/repgen/__main__.py b/repgen/__main__.py index cce0ca6..447345e 100644 --- a/repgen/__main__.py +++ b/repgen/__main__.py @@ -2,7 +2,7 @@ from repgen.data.value import Value from repgen.report import Report from repgen.util import filterAddress - +from repgen import PROD_CDA_HOST version = "5.1.6" # setup base time, ex @@ -22,7 +22,7 @@ def parseArgs(): parser.add_argument( '-t', '--time', dest='base_time', default=_t, help="base time for data", metavar="HHMM") parser.add_argument( '-z', '--tz', dest='tz', default=_z, help="default timezone; equivalent to `TZ=timezone`", metavar='Time Zone Name') parser.add_argument( '-O', '--office', dest='office', default=None, help="default office to use if not specified in report; equivalent to `DBOFC=OFFICE_ID`", metavar='OFFICE_ID') - parser.add_argument( '-a', '--address', dest='host', default='localhost', help="location for data connections; equivalent to `DB=hostname:port/path`", metavar='IP_or_hostname:port[/basepath]') + parser.add_argument( '-a', '--address', dest='host', default=PROD_CDA_HOST, help="location for data connections; equivalent to `DB=hostname:port/path`", metavar='IP_or_hostname:port[/basepath]') parser.add_argument( '-A', '--alternate', dest='alternate', default=None, help="alternate location for data connections, if the primary is unavailable (only for CDA)", metavar='IP_or_hostname:port[/basepath]') parser.add_argument( '-c', '--compatibility', dest='compat', action="store_true", default=False, help="repgen4 compatibility; case-insensitive labels") parser.add_argument( '--timeout', dest='timeout', type=float, default=None, help="Socket timeout, in seconds" ) @@ -108,7 +108,12 @@ def main(): kwargs.pop("FILE", None) (host, path) = filterAddress(config.host) (althost, altpath) = filterAddress(config.alternate) - + if host == filterAddress(PROD_CDA_HOST)[0]: + if not config.office: + raise ValueError(f'''\n\n + Attempted to query Production CDA ({PROD_CDA_HOST}) + \n\tYou must specify an OFFICE ID with the -O argument when using the default CDA host / National CDA instance i.e. + \n\t{' '.join(sys.argv)} -O NAE\n\n''') tz = None if config.tz: diff --git a/tests/various/tsid/testfiles/swt.keystone.frm b/tests/various/tsid/testfiles/swt.keystone.frm index 009bfef..d557128 100644 --- a/tests/various/tsid/testfiles/swt.keystone.frm +++ b/tests/various/tsid/testfiles/swt.keystone.frm @@ -83,7 +83,6 @@ stor = Value( UNDEF="~~", DBUNITS="ac-ft", ) -print(stor.values) # ============================================================================== # Dates # ============================================================================== From 1554d4fcc90a5cc0f12d24cc02cccfb6825133cd Mon Sep 17 00:00:00 2001 From: "Charles Graham, SWT" Date: Mon, 12 Aug 2024 17:05:28 -0500 Subject: [PATCH 08/17] Initial level class, testing --- repgen/__init__.py | 5 +- repgen/data/__init__.py | 3 +- repgen/data/levels.py | 51 +++++++++++++++++++ repgen/data/value.py | 4 +- setup.py | 1 + tests/various/tsid/testfiles/swt.keystone.frm | 1 + 6 files changed, 62 insertions(+), 3 deletions(-) create mode 100644 repgen/data/levels.py diff --git a/repgen/__init__.py b/repgen/__init__.py index ef334f1..47f6d59 100644 --- a/repgen/__init__.py +++ b/repgen/__init__.py @@ -1,2 +1,5 @@ REPGEN_DOCS_URL="https://repgen5.readthedocs.io/" -PROD_CDA_HOST = "https://cwms-data.usace.army.mil/cwms-data" \ No newline at end of file +PROD_CDA_HOST = "https://cwms-data.usace.army.mil/cwms-data" + +# Time in seconds before a request should timeout +REQUEST_TIMEOUT_SECONDS = 30 \ No newline at end of file diff --git a/repgen/data/__init__.py b/repgen/data/__init__.py index 6f61403..323b42e 100644 --- a/repgen/data/__init__.py +++ b/repgen/data/__init__.py @@ -1 +1,2 @@ -from . value import Value \ No newline at end of file +from . value import Value +from . levels import Level \ No newline at end of file diff --git a/repgen/data/levels.py b/repgen/data/levels.py new file mode 100644 index 0000000..314120b --- /dev/null +++ b/repgen/data/levels.py @@ -0,0 +1,51 @@ +# stdlib +from urllib.parse import quote +# third party +import requests +import sys +import os + +# custom +from repgen import PROD_CDA_HOST, REQUEST_TIMEOUT_SECONDS + +class Level: + @staticmethod + def getLevelById(levelId: str, office: str, effectiveDate: str, timeZone: str = "", unit: str = "", *args, **kwargs) -> dict: + ''' + Get levels for a site from the CDA API. + + Args: + levelId (str): Specifies the requested location level. + office (str): Specifies the office of the Location Level to be returned. + effectiveDate (str): Specifies the effective date of Location Level to be returned. + timeZone (str, optional): Specifies the time zone of the values of the effective date field (unless otherwise specified), as well as the time zone of any times in the response. If this field is not specified, the default time zone of UTC shall be used. + unit (str, optional): Desired unit for the values retrieved. + + Returns: + LevelResponse: A structured response containing location level details. + ''' + + if not levelId: + raise ValueError("levelId is required: i.e. levelId='KEYS.Elev.Inst.0.Top of Conservation'") + else: + # Encode the levelId for % and / characters that are not allowed in the URL + levelId = quote(levelId) + params = {"effective-date": effectiveDate, "timezone": timeZone, "unit": unit, "office": office, **kwargs} + # Remove None values + params = {k: v for k, v in params.items() if v is not None} + # Encode only the values + encoded_params = {k: quote(str(v)) for k, v in params.items()} + print(encoded_params) + print(f"{PROD_CDA_HOST}/levels/{levelId}") + response = requests.get(f"{PROD_CDA_HOST}/levels/{levelId}" , params=encoded_params, timeout=REQUEST_TIMEOUT_SECONDS) + if response.status_code == 200: + return response.json() + else: + raise Exception(f"Error getting levels for levelId {levelId} and office {office}: {response.status_code}") + +if __name__ == "__main__": + # Invoke this script for testing with: (to make sure to import the top level repgen package) + # python -m repgen.data.levels + + print("NOTE: This script should ONLY be called directly for testing purposes.") + print(Level.getLevelById(levelId="KEYS.Elev.Inst.0.Top of Conservation", office="SWT", effectiveDate="2024-06-21", timeZone="America/Chicago", unit="ft")) \ No newline at end of file diff --git a/repgen/data/value.py b/repgen/data/value.py index 1fcb1ef..fa48ccd 100644 --- a/repgen/data/value.py +++ b/repgen/data/value.py @@ -8,6 +8,8 @@ from ssl import SSLError import re from repgen.util import extra_operator, filterAddress +from repgen.data.levels import Level + import signal try: @@ -34,7 +36,7 @@ def handler(signum, frame): if sys.platform != "win32": signal.signal(signal.SIGALRM, handler) -class Value: +class Value(Level): shared = { "picture" : "NNZ", "misstr" : "-M-", diff --git a/setup.py b/setup.py index 478e7df..02e10ea 100644 --- a/setup.py +++ b/setup.py @@ -24,6 +24,7 @@ install_requires=[ "pytz", "python-dateutil>=2.8.2", + "requests" ], extras_require={ "dev": [ diff --git a/tests/various/tsid/testfiles/swt.keystone.frm b/tests/various/tsid/testfiles/swt.keystone.frm index d557128..8274da1 100644 --- a/tests/various/tsid/testfiles/swt.keystone.frm +++ b/tests/various/tsid/testfiles/swt.keystone.frm @@ -93,6 +93,7 @@ stor = Value( # # Flow # # ============================================================================== flow_out = Value( + DBTYPE="CDA", DBPAR="Flow-Res Out", DBPTYP="Ave", DBINT="1Hour", From 0ad292b13cac7d38edf3b98b0ebe345fa5dd04ee Mon Sep 17 00:00:00 2001 From: "Charles Graham, SWT" Date: Mon, 12 Aug 2024 20:14:19 -0500 Subject: [PATCH 09/17] Tested working Levels class for primary GET endpoints --- repgen/data/levels.py | 149 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 138 insertions(+), 11 deletions(-) diff --git a/repgen/data/levels.py b/repgen/data/levels.py index 314120b..a2aa744 100644 --- a/repgen/data/levels.py +++ b/repgen/data/levels.py @@ -2,15 +2,24 @@ from urllib.parse import quote # third party import requests -import sys -import os +from requests.exceptions import HTTPError +from json.decoder import JSONDecodeError # custom from repgen import PROD_CDA_HOST, REQUEST_TIMEOUT_SECONDS +def printError(err, response): + print(f"HTTPError: {err}") + if err.response.status_code == 404: + print(f"HTTPError: 404 Not Found") + elif err.response.status_code == 400: + print(f"HTTPError: 400 Bad Request") + print(f"Response: {err.response.text}") + if response: + print(f"Response: {response.text}") class Level: @staticmethod - def getLevelById(levelId: str, office: str, effectiveDate: str, timeZone: str = "", unit: str = "", *args, **kwargs) -> dict: + def getLevelById(levelId: str, office: str, effectiveDate: str, unit: str = "", timeZone: str = "GMT", *args, **kwargs) -> dict: ''' Get levels for a site from the CDA API. @@ -23,29 +32,147 @@ def getLevelById(levelId: str, office: str, effectiveDate: str, timeZone: str = Returns: LevelResponse: A structured response containing location level details. + + Example: + Level.getLevelById( + levelId="KEYS.Elev.Inst.0.Top of Conservation", + office="SWT", + effectiveDate="2024-06-21T00:00:00", + timeZone="America/Chicago", + unit="ft" + ) ''' if not levelId: raise ValueError("levelId is required: i.e. levelId='KEYS.Elev.Inst.0.Top of Conservation'") + elif not office: + raise ValueError("office is required: i.e. office='SWT'") + elif not effectiveDate: + raise ValueError("effectiveDate is required: i.e. effectiveDate='2024-06-21T00:00:00'") + elif not unit: + raise ValueError("unit is required: i.e. unit='ft'") else: # Encode the levelId for % and / characters that are not allowed in the URL levelId = quote(levelId) - params = {"effective-date": effectiveDate, "timezone": timeZone, "unit": unit, "office": office, **kwargs} + params = {"effective-date": quote(effectiveDate), "timezone": quote(timeZone), "unit": unit, "office": office, **kwargs} # Remove None values params = {k: v for k, v in params.items() if v is not None} # Encode only the values - encoded_params = {k: quote(str(v)) for k, v in params.items()} - print(encoded_params) - print(f"{PROD_CDA_HOST}/levels/{levelId}") - response = requests.get(f"{PROD_CDA_HOST}/levels/{levelId}" , params=encoded_params, timeout=REQUEST_TIMEOUT_SECONDS) + # encoded_params = {k: quote(str(v)) for k, v in params.items()} + try: + response = requests.get(f"{PROD_CDA_HOST}/levels/{levelId}" , params=params, timeout=REQUEST_TIMEOUT_SECONDS) + print(f"[/levels/{levelId}]\t", response.url) + # Raise HTTPError if one occurred + response.raise_for_status() + except HTTPError as err: + printError(err, response) if response.status_code == 200: return response.json() else: - raise Exception(f"Error getting levels for levelId {levelId} and office {office}: {response.status_code}") + raise Exception(f"Error getting levels for levelId {levelId} and office {office}: {response.status_code}\n{response.text}") + + def getLevels(levelIdMask: str, + office: str, + unit: str = None, + datum: str = None, + begin: str = None, + end: str = None, + timeZone: str = None, + format: str = "json", + page: str = None, + pageSize = None, + *args, **kwargs) -> dict: + ''' + Get levels for a site from the CDA API. + + Args: + levelIdMask (str): pecifies the name(s) of the location level(s) whose data is to be included in the response. Uses * for all. + office (str): Specifies the owning office of the location level(s) whose data is to be included in the response. If this field is not specified, matching location level information from all offices shall be returned. + unit (str, optional): Specifies the unit or unit system of the response. Valid values for the unit field are: + 1. EN (DEFAULT) - Specifies English unit system. Location level values will be in the default English units for their parameters. + 2. SI - Specifies the SI unit system. Location level values will be in the default SI units for their parameters. + 3. Other - Any unit returned in the response to the units URI request that is appropriate for the requested parameters. + datum (str, optional):pecifies the elevation datum of the response. This field affects only elevation location levels. Valid values for this field are: + 1. NAVD88 - The elevation values will in the specified or default units above the NAVD-88 datum. + 2. NGVD29 - The elevation values will be in the specified or default units above the NGVD-29 datum. + begin (str, optional): Specifies the start of the time window for data to be included in the response. If this field is not specified, any required time window begins 24 hours prior to the specified or default end time. + end (str, optional): Specifies the end of the time window for data to be included in the response. If this field is not specified, any required time window ends at the current time + timeZone (str, optional): Specifies the time zone of the values of the begin and end fields (unless otherwise specified), as well as the time zone of any times in the response. If this field is not specified, the default time zone of UTC shall be used. + format (str, optional): Specifies the encoding format of the response. Requests specifying an Accept header:application/json;version=2 must not include this field. Valid format field values for this URI are: + 1. tab + 2. csv + 3. xml + 4. wml2 (only if name field is specified) + 5. json (default) + page (str, optional): This identifies where in the request you are. This is an opaque value, and can be obtained from the 'next-page' value in the response. + pageSize (str, optional): How many entries per page returned. Default 100. + + Returns: + dict: A structured response containing location level details. + + Example: + Level.getLevels( + levelIdMask="KEYS.Elev.Inst.0.*", + office="SWT", + effectiveDate="2024-06-21T00:00:00", + timeZone="America/Chicago", + unit="ft" + ) + ''' + + if not levelIdMask: + raise ValueError("levelIdMask is required: i.e. levelIdMask='KEYS.Elev.Inst.0.*'") + elif not office: + raise ValueError("office is required: i.e. office='SWT'") + params = { + "level-id-mask": levelIdMask, + "office": office, + "unit": unit, + "datum": datum, + "begin": begin, + "end": end, + "timezone": timeZone, + "format": format, + "page": page, + "page-size": pageSize, + **kwargs + } + # Remove None values + params = {k: v for k, v in params.items() if v is not None} + # Encode only the values + # encoded_params = {k: quote(str(v)) for k, v in params.items()} + try: + response = requests.get(f"{PROD_CDA_HOST}/levels" , + headers={ + "Accept": "*/*" + }, + params=params, + timeout=REQUEST_TIMEOUT_SECONDS) + print("[/levels]\t", response.url) + # Raise HTTPError if one occurred + response.raise_for_status() + except HTTPError as err: + printError(err, response) + + if response.status_code == 200: + try: + return response.json() + except JSONDecodeError as err: + print(f"JSONDecodeError: {err}") + print(f"Response: {response.text}") + else: + raise Exception(f"Error getting levels for levelIdMask {levelIdMask} and office {office}: {response.status_code}\n{response.text}") if __name__ == "__main__": # Invoke this script for testing with: (to make sure to import the top level repgen package) # python -m repgen.data.levels - print("NOTE: This script should ONLY be called directly for testing purposes.") - print(Level.getLevelById(levelId="KEYS.Elev.Inst.0.Top of Conservation", office="SWT", effectiveDate="2024-06-21", timeZone="America/Chicago", unit="ft")) \ No newline at end of file + print("WARNING: This script should ONLY be called directly for testing purposes.") + print("TEST: Level.getLevelById") + print(Level.getLevelById(levelId="KEYS.Elev.Inst.0.Top of Conservation", office="SWT", effectiveDate="2024-06-21T00:00:00", timeZone="America/Chicago", unit="ft")) + print("TEST: Level.getLevels") + print(Level.getLevels(levelIdMask="KEYS.Elev.Inst.0.Top of Conservation", + begin="2024-06-11T00:00:00-05:00", + end="2024-06-12T00:00:00-05:00", + timeZone="America/Chicago", + office="SWT")) \ No newline at end of file From 19fbea4bb131e1a167566daf359d487d2a333eef Mon Sep 17 00:00:00 2001 From: "Charles Graham, SWT" Date: Mon, 12 Aug 2024 21:20:52 -0500 Subject: [PATCH 10/17] Rename to LevelsApi to match CWMSjs --- repgen/data/__init__.py | 2 +- repgen/data/value.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/repgen/data/__init__.py b/repgen/data/__init__.py index 323b42e..7158cf1 100644 --- a/repgen/data/__init__.py +++ b/repgen/data/__init__.py @@ -1,2 +1,2 @@ from . value import Value -from . levels import Level \ No newline at end of file +from . levels import LevelsApi \ No newline at end of file diff --git a/repgen/data/value.py b/repgen/data/value.py index fa48ccd..826b475 100644 --- a/repgen/data/value.py +++ b/repgen/data/value.py @@ -8,7 +8,7 @@ from ssl import SSLError import re from repgen.util import extra_operator, filterAddress -from repgen.data.levels import Level +from repgen.data.levels import LevelsApi import signal @@ -36,7 +36,7 @@ def handler(signum, frame): if sys.platform != "win32": signal.signal(signal.SIGALRM, handler) -class Value(Level): +class Value(LevelsApi): shared = { "picture" : "NNZ", "misstr" : "-M-", From a424c0e28aa266646082b078c4f08e462572ddcc Mon Sep 17 00:00:00 2001 From: "Charles Graham, SWT" Date: Mon, 12 Aug 2024 21:21:02 -0500 Subject: [PATCH 11/17] Create shared session across all of repgen for requests lib --- repgen/__init__.py | 10 ++++++++- repgen/data/levels.py | 48 +++++++++++++++++++++++++++++++++++-------- 2 files changed, 48 insertions(+), 10 deletions(-) diff --git a/repgen/__init__.py b/repgen/__init__.py index 47f6d59..9df7d9c 100644 --- a/repgen/__init__.py +++ b/repgen/__init__.py @@ -1,5 +1,13 @@ +import requests + REPGEN_DOCS_URL="https://repgen5.readthedocs.io/" PROD_CDA_HOST = "https://cwms-data.usace.army.mil/cwms-data" # Time in seconds before a request should timeout -REQUEST_TIMEOUT_SECONDS = 30 \ No newline at end of file +REQUEST_TIMEOUT_SECONDS = 30 +# The MAX number of times to retry a request +MAX_RETRIES = 3 +# The amount of time to wait before retrying a request in seconds +BACKOFF_FACTOR = 1 +# Create a session for all requests +session = requests.Session() diff --git a/repgen/data/levels.py b/repgen/data/levels.py index a2aa744..ddfecd3 100644 --- a/repgen/data/levels.py +++ b/repgen/data/levels.py @@ -1,14 +1,35 @@ # stdlib from urllib.parse import quote # third party -import requests +from repgen import session from requests.exceptions import HTTPError +from requests import Response from json.decoder import JSONDecodeError +from requests.adapters import HTTPAdapter +from urllib3.util.retry import Retry # custom -from repgen import PROD_CDA_HOST, REQUEST_TIMEOUT_SECONDS +from repgen import PROD_CDA_HOST, REQUEST_TIMEOUT_SECONDS, \ + MAX_RETRIES, BACKOFF_FACTOR -def printError(err, response): +_retry_strategy = Retry( + total=MAX_RETRIES, + status_forcelist=[429, 500, 502, 503, 504], + method_whitelist=["GET"], + backoff_factor=BACKOFF_FACTOR, +) + +session.mount("https://", HTTPAdapter(max_retries=_retry_strategy)) + + +def printError(err: HTTPError, response: Response) -> None: + ''' + Prints the error and response if present + + Args: + err (HTTPError): The HTTPError object. + response (Response): The Response object. + ''' print(f"HTTPError: {err}") if err.response.status_code == 404: print(f"HTTPError: 404 Not Found") @@ -17,7 +38,8 @@ def printError(err, response): print(f"Response: {err.response.text}") if response: print(f"Response: {response.text}") -class Level: +class LevelsApi: + @staticmethod def getLevelById(levelId: str, office: str, effectiveDate: str, unit: str = "", timeZone: str = "GMT", *args, **kwargs) -> dict: ''' @@ -54,13 +76,15 @@ def getLevelById(levelId: str, office: str, effectiveDate: str, unit: str = "", else: # Encode the levelId for % and / characters that are not allowed in the URL levelId = quote(levelId) - params = {"effective-date": quote(effectiveDate), "timezone": quote(timeZone), "unit": unit, "office": office, **kwargs} + params = {"effective-date": effectiveDate, "timezone": quote(timeZone), "unit": unit, "office": office, **kwargs} # Remove None values params = {k: v for k, v in params.items() if v is not None} # Encode only the values # encoded_params = {k: quote(str(v)) for k, v in params.items()} try: - response = requests.get(f"{PROD_CDA_HOST}/levels/{levelId}" , params=params, timeout=REQUEST_TIMEOUT_SECONDS) + response = session.get(f"{PROD_CDA_HOST}/levels/{levelId}" , + params=params, + timeout=REQUEST_TIMEOUT_SECONDS) print(f"[/levels/{levelId}]\t", response.url) # Raise HTTPError if one occurred response.raise_for_status() @@ -71,6 +95,7 @@ def getLevelById(levelId: str, office: str, effectiveDate: str, unit: str = "", else: raise Exception(f"Error getting levels for levelId {levelId} and office {office}: {response.status_code}\n{response.text}") + @staticmethod def getLevels(levelIdMask: str, office: str, unit: str = None, @@ -142,7 +167,7 @@ def getLevels(levelIdMask: str, # Encode only the values # encoded_params = {k: quote(str(v)) for k, v in params.items()} try: - response = requests.get(f"{PROD_CDA_HOST}/levels" , + response = session.get(f"{PROD_CDA_HOST}/levels" , headers={ "Accept": "*/*" }, @@ -169,9 +194,14 @@ def getLevels(levelIdMask: str, print("WARNING: This script should ONLY be called directly for testing purposes.") print("TEST: Level.getLevelById") - print(Level.getLevelById(levelId="KEYS.Elev.Inst.0.Top of Conservation", office="SWT", effectiveDate="2024-06-21T00:00:00", timeZone="America/Chicago", unit="ft")) + print(LevelsApi.getLevelById( + levelId="KEYS.Elev.Inst.0.Top of Conservation", + office="SWT", + effectiveDate="2024-06-21T00:00:00", + timeZone="America/Chicago", + unit="ft")) print("TEST: Level.getLevels") - print(Level.getLevels(levelIdMask="KEYS.Elev.Inst.0.Top of Conservation", + print(LevelsApi.getLevels(levelIdMask="KEYS.Elev.Inst.0.Top of Conservation", begin="2024-06-11T00:00:00-05:00", end="2024-06-12T00:00:00-05:00", timeZone="America/Chicago", From d8cec21fc92a73f6255726f8e82516089595e0c6 Mon Sep 17 00:00:00 2001 From: "Charles Graham, SWT" Date: Mon, 12 Aug 2024 22:22:03 -0500 Subject: [PATCH 12/17] Removed allowed methods, CDA can handle that and it changes between versions of python --- repgen/data/levels.py | 1 - 1 file changed, 1 deletion(-) diff --git a/repgen/data/levels.py b/repgen/data/levels.py index ddfecd3..84b8afc 100644 --- a/repgen/data/levels.py +++ b/repgen/data/levels.py @@ -15,7 +15,6 @@ _retry_strategy = Retry( total=MAX_RETRIES, status_forcelist=[429, 500, 502, 503, 504], - method_whitelist=["GET"], backoff_factor=BACKOFF_FACTOR, ) From 6b8200eff598c4059a018a42826a42b2b462d600 Mon Sep 17 00:00:00 2001 From: "Charles Graham, SWT" Date: Mon, 12 Aug 2024 22:22:24 -0500 Subject: [PATCH 13/17] Start testing a level value in a form --- repgen/__init__.py | 5 + repgen/data/value.py | 364 ++++++++++--------- tests/various/level/testfiles/swt.levels.frm | 20 + 3 files changed, 221 insertions(+), 168 deletions(-) create mode 100644 tests/various/level/testfiles/swt.levels.frm diff --git a/repgen/__init__.py b/repgen/__init__.py index 9df7d9c..18ff25a 100644 --- a/repgen/__init__.py +++ b/repgen/__init__.py @@ -11,3 +11,8 @@ BACKOFF_FACTOR = 1 # Create a session for all requests session = requests.Session() + +CDA_DATE_FORMAT = "%Y-%m-%dT%H:%M:%S" + +# Specifies the allowed unit system of the level response +CDA_UNIT_SYSTEMS = ["SI", "EN", "Other"] \ No newline at end of file diff --git a/repgen/data/value.py b/repgen/data/value.py index 826b475..9715bd8 100644 --- a/repgen/data/value.py +++ b/repgen/data/value.py @@ -9,7 +9,7 @@ import re from repgen.util import extra_operator, filterAddress from repgen.data.levels import LevelsApi - +from repgen import CDA_DATE_FORMAT, CDA_UNIT_SYSTEMS import signal try: @@ -43,7 +43,7 @@ class Value(LevelsApi): "undef" : "-?-", "missdta" : -901, "missing": "MISSOK", # How to treat missing values - + "levelid": None, # shared and updated between calls "host" : None, # ip address/hostname or file name "dbtype" : None, # file or spkjson @@ -58,6 +58,7 @@ class Value(LevelsApi): "ucformat": None, # Upper-case date format (repgen4 compatibility) "use_alternate": False, # Alternate server in use, if primary is unavailable } + dbunits = None DB_TYPES = [ "CDA", @@ -188,6 +189,7 @@ def processDateTime(value, key, extra_part=None): pending_date = None # update the shared keywords + print(kwargs) for key in kwargs: value = kwargs[key] # If the value is wrapped in quotes, it's most likely wrong (possibly value was read from a file where it had quotes). @@ -288,8 +290,13 @@ def processDateTime(value, key, extra_part=None): return elif len(args) > 0: raise ValueError("Only 1 non named value is allowed") - - self.type = "TIMESERIES" + + # If the levelId is set, go for that first + print('dbtype', self.dbtype) + if self.levelid: + self.type = "LEVEL" + else: + self.type = "TIMESERIES" self.values = [ ] # will be a tuple of (time stamp, value, quality ) if self.dbtype.upper() == "RADAR": print("\n\tWARNING: Update from dbtype=\"RADAR\" to dbtype=\"CDA\"") @@ -363,7 +370,6 @@ def parse_slice(value): elif self.dbtype.upper() == "SPKJSON": import json, http.client as httplib, urllib.parse as urllib - fmt = "%d-%b-%Y %H%M" tz = self.dbtz units= self.dbunits ts_name = ".".join( (self.dbloc, self.dbpar, self.dbptyp, self.dbint, self.dbdur, self.dbver) ) @@ -372,13 +378,13 @@ def parse_slice(value): start = self.start.astimezone(tz) end = self.end.astimezone(tz) - sys.stderr.write("Getting %s from %s to %s in tz %s, with units %s\n" % (ts_name,start.strftime(fmt),end.strftime(fmt),str(tz),units)) + sys.stderr.write("Getting %s from %s to %s in tz %s, with units %s\n" % (ts_name,start.strftime(CDA_DATE_FORMAT),end.strftime(CDA_DATE_FORMAT),str(tz),units)) query = "/fcgi-bin/get_ts.py?" params = urllib.urlencode( { "site": ts_name, "units": units, - "start_time": start.strftime(fmt), - "end_time": end.strftime(fmt), + "start_time": start.strftime(CDA_DATE_FORMAT), + "end_time": end.strftime(CDA_DATE_FORMAT), "tz": str(tz) }) try: @@ -416,181 +422,203 @@ def parse_slice(value): except Exception as err: print( repr(err) + " : " + str(err), file=sys.stderr ) elif self.dbtype.upper() in ["JSON", "CDA"]: - import json, http.client as httplib, urllib.parse as urllib - - #fmt = "%d-%b-%Y %H%M" - fmt = "%Y-%m-%dT%H:%M:%S" + print(self.start, self.end) tz = self.dbtz units = self.dbunits - ts_name = ".".join( (self.dbloc, self.dbpar, self.dbptyp, str(self.dbint), str(self.dbdur), self.dbver) ) - - if self.start is None or self.end is None: - return - - # Loop until we fetch some data, if missing is NOMISS - retry_count = 10 # Go back at most this many weeks + 1 - sstart = tz.normalize(tz.localize(self.start)) if self.start.tzinfo is None else self.start - send = tz.normalize(tz.localize(self.end)) if self.end.tzinfo is None else self.end - - path = self.path if not Value.shared["use_alternate"] else self.altpath - host = self.host if not Value.shared["use_alternate"] else self.althost - headers = { 'Accept': "application/json;version=2" } - - while(retry_count > 0): - # Convert time to destination timezone - # Should this actually convert the time to the destination time zone (astimezone), or simply swap the TZ (replace)? - # 'astimezone' is be the "proper" behavior, but 'replace' mimics repgen_4. - # This should *not* be a naive datetime + + if self.start and self.end: + sstart = tz.normalize(tz.localize(self.start)) if self.start.tzinfo is None else self.start + send = tz.normalize(tz.localize(self.end)) if self.end.tzinfo is None else self.end assert sstart.tzinfo is not None, "Naive datetime; start time should contain timezone" assert send.tzinfo is not None, "Naive datetime; end time should contain timezone" start = sstart.astimezone(tz) end = send.astimezone(tz) - - params = urllib.urlencode( { - "name": ts_name, - "unit": units, - "begin": start.strftime(fmt), - "end": end.strftime(fmt), - "office": self.dbofc if self.dbofc is not None else "", - "timezone": str(tz), - "pageSize": -1, # always fetch all results - }) - - sys.stderr.write("Getting %s from %s to %s in tz %s, with units %s\n" % (ts_name,start.strftime(fmt),end.strftime(fmt),str(tz),units)) + print("type", self.type) + if self.type == "TIMESERIES": + if self.start is None or self.end is None: + return + import json, http.client as httplib, urllib.parse as urllib - try: - data = None - retry_until_alternate = 3 - while retry_until_alternate > 0: - retry_until_alternate -= 1 - if path is None: - path = "" + ts_name = ".".join( (self.dbloc, self.dbpar, self.dbptyp, str(self.dbint), str(self.dbdur), self.dbver) ) + + # Loop until we fetch some data, if missing is NOMISS + retry_count = 10 # Go back at most this many weeks + 1 + + path = self.path if not Value.shared["use_alternate"] else self.altpath + host = self.host if not Value.shared["use_alternate"] else self.althost + headers = { 'Accept': "application/json;version=2" } + + while(retry_count > 0): + # Convert time to destination timezone + # Should this actually convert the time to the destination time zone (astimezone), or simply swap the TZ (replace)? + # 'astimezone' is be the "proper" behavior, but 'replace' mimics repgen_4. + # This should *not* be a naive datetime + + params = urllib.urlencode( { + "name": ts_name, + "unit": units, + "begin": start.strftime(CDA_DATE_FORMAT), + "end": end.strftime(CDA_DATE_FORMAT), + "office": self.dbofc if self.dbofc is not None else "", + "timezone": str(tz), + "pageSize": -1, # always fetch all results + }) + + sys.stderr.write("Getting %s from %s to %s in tz %s, with units %s\n" % (ts_name,start.strftime(CDA_DATE_FORMAT),end.strftime(CDA_DATE_FORMAT),str(tz),units)) - query = f"/{path}/timeseries?" + try: + data = None + retry_until_alternate = 3 + while retry_until_alternate > 0: + retry_until_alternate -= 1 + if path is None: + path = "" - # The http(s) guess isn't perfect, but it's good enough. It's for display purposes only. - print("Fetching: %s" % ("https://" if host[-2:] == "43" else "http://") + host+query+params, file=sys.stderr) + query = f"/{path}/timeseries?" - try: - if sys.platform != "win32" and self.timeout: - # The SSL handshake can sometimes fail and hang indefinitely - # inflate the timeout slightly, so the socket has a chance to return a timeout error - # This is a failsafe to prevent a hung process - signal.alarm(int(self.timeout * 1.1) + 1) + # The http(s) guess isn't perfect, but it's good enough. It's for display purposes only. + print("Fetching: %s" % ("https://" if host[-2:] == "43" else "http://") + host+query+params, file=sys.stderr) - if Value._conn is None: - try: - from repgen.util.urllib2_tls import TLS1Connection - Value._conn = TLS1Connection( host, timeout=self.timeout, context=ssl_ctx ) - Value._conn.request("GET", "/{path}" ) - except SSLError as err: - print(type(err).__name__ + " : " + str(err), file=sys.stderr) - print("Falling back to non-SSL", file=sys.stderr) - # SSL not supported (could be standalone instance) - Value._conn = httplib.HTTPConnection( host, timeout=self.timeout ) - Value._conn.request("GET", "/{path}" ) - - # Test if the connection is valid - Value._conn.getresponse().read() - - Value._conn.request("GET", query+params, None, headers ) - r1 = Value._conn.getresponse() - - # getresponse can also hang sometimes, so keep alarm active until after we fetch the response - if sys.platform != "win32" and self.timeout: - signal.alarm(0) # disable the alarm - - # Grab the charset from the headers, and decode the response using that if set - # HTTP default charset is iso-8859-1 for text (RFC 2616), and utf-8 for JSON (RFC 4627) - parts = r1.getheader("Content-Type").split(";") - charset = "iso-8859-1" if parts[0].startswith("text") else "utf-8" # Default charset - - if len(parts) > 1: - for prop in parts: - prop_parts = prop.split("=") - if len(prop_parts) > 1 and prop_parts[0].lower() == "charset": - charset = prop_parts[1] - - data = r1.read().decode(charset) - - if r1.status == 200: - break - - print("HTTP Error " + str(r1.status) + ": " + data, file=sys.stderr) - if r1.status == 404: - json.loads(data) - # We don't care about the actual error, just if it's valid JSON - # Valid JSON means it was a CDA response, so we treat it as a valid response, and won't retry. - break - except (httplib.NotConnected, httplib.ImproperConnectionState, httplib.BadStatusLine, ValueError, OSError) as e: - print(f"Error fetching: {e}", file=sys.stderr) - if retry_until_alternate == 0 and self.althost is not None and host != self.althost: - print("Trying alternate server", file=sys.stderr) - Value.shared["use_alternate"] = True - (host, path) = (self.althost, self.altpath) - Value._conn = None - retry_until_alternate = 3 - else: - print("Reconnecting to server and trying again", file=sys.stderr) - ttime.sleep(3) - try: - Value._conn.close() - except: - pass - Value._conn = None - continue + try: + if sys.platform != "win32" and self.timeout: + # The SSL handshake can sometimes fail and hang indefinitely + # inflate the timeout slightly, so the socket has a chance to return a timeout error + # This is a failsafe to prevent a hung process + signal.alarm(int(self.timeout * 1.1) + 1) + + if Value._conn is None: + try: + from repgen.util.urllib2_tls import TLS1Connection + Value._conn = TLS1Connection( host, timeout=self.timeout, context=ssl_ctx ) + Value._conn.request("GET", "/{path}" ) + except SSLError as err: + print(type(err).__name__ + " : " + str(err), file=sys.stderr) + print("Falling back to non-SSL", file=sys.stderr) + # SSL not supported (could be standalone instance) + Value._conn = httplib.HTTPConnection( host, timeout=self.timeout ) + Value._conn.request("GET", "/{path}" ) + + # Test if the connection is valid + Value._conn.getresponse().read() + + Value._conn.request("GET", query+params, None, headers ) + r1 = Value._conn.getresponse() + + # getresponse can also hang sometimes, so keep alarm active until after we fetch the response + if sys.platform != "win32" and self.timeout: + signal.alarm(0) # disable the alarm + + # Grab the charset from the headers, and decode the response using that if set + # HTTP default charset is iso-8859-1 for text (RFC 2616), and utf-8 for JSON (RFC 4627) + parts = r1.getheader("Content-Type").split(";") + charset = "iso-8859-1" if parts[0].startswith("text") else "utf-8" # Default charset + + if len(parts) > 1: + for prop in parts: + prop_parts = prop.split("=") + if len(prop_parts) > 1 and prop_parts[0].lower() == "charset": + charset = prop_parts[1] + + data = r1.read().decode(charset) + + if r1.status == 200: + break + + print("HTTP Error " + str(r1.status) + ": " + data, file=sys.stderr) + if r1.status == 404: + json.loads(data) + # We don't care about the actual error, just if it's valid JSON + # Valid JSON means it was a CDA response, so we treat it as a valid response, and won't retry. + break + except (httplib.NotConnected, httplib.ImproperConnectionState, httplib.BadStatusLine, ValueError, OSError) as e: + print(f"Error fetching: {e}", file=sys.stderr) + if retry_until_alternate == 0 and self.althost is not None and host != self.althost: + print("Trying alternate server", file=sys.stderr) + Value.shared["use_alternate"] = True + (host, path) = (self.althost, self.altpath) + Value._conn = None + retry_until_alternate = 3 + else: + print("Reconnecting to server and trying again", file=sys.stderr) + ttime.sleep(3) + try: + Value._conn.close() + except: + pass + Value._conn = None + continue - data_dict = None + data_dict = None - try: - data_dict = json.loads(data) - except json.JSONDecodeError as err: - print(str(err), file=sys.stderr) - print(repr(data), file=sys.stderr) - - # get the depth - prev_t = 0 - #print repr(data_dict) - - if data_dict.get("total", 0) > 0: - for d in data_dict["values"]: - _t = float(d[0])/1000.0 # json returns times in javascript time, milliseconds since epoch, convert to unix time of seconds since epoch - _dt = datetime.datetime.fromtimestamp(_t,pytz.utc) - _dt = _dt.astimezone(self.dbtz) - #_dt = _dt.replace(tzinfo=self.tz) - #print("_dt: %s" % repr(_dt)) - #print _dt - if d[1] is not None: - #print("Reading value: %s" % d[1]) - _v = float(d[1]) # does not currently implement text operations - else: - _v = None - _q = int(d[2]) - self.values.append( ( _dt,_v,_q ) ) - else: - print("No values were fetched.", file=sys.stderr) - - if self.ismissing(): - if self.missing == "NOMISS": - sstart = sstart - timedelta(weeks=1) - retry_count = retry_count - 1 - continue - - if self.time: - self.type = "SCALAR" - if self.missing == "NOMISS": - # Get the last one, in case we fetched extra because of NOMISS - for v in reversed(self.values): - if v is not None and v[1] is not None: - self.value = v[1] - break - elif len(self.values) > 0: - self.value = self.values[-1][1] + try: + data_dict = json.loads(data) + except json.JSONDecodeError as err: + print(str(err), file=sys.stderr) + print(repr(data), file=sys.stderr) + + # get the depth + prev_t = 0 + #print repr(data_dict) + + if data_dict.get("total", 0) > 0: + for d in data_dict["values"]: + _t = float(d[0])/1000.0 # json returns times in javascript time, milliseconds since epoch, convert to unix time of seconds since epoch + _dt = datetime.datetime.fromtimestamp(_t,pytz.utc) + _dt = _dt.astimezone(self.dbtz) + #_dt = _dt.replace(tzinfo=self.tz) + #print("_dt: %s" % repr(_dt)) + #print _dt + if d[1] is not None: + #print("Reading value: %s" % d[1]) + _v = float(d[1]) # does not currently implement text operations + else: + _v = None + _q = int(d[2]) + self.values.append( ( _dt,_v,_q ) ) + else: + print("No values were fetched.", file=sys.stderr) + + if self.ismissing(): + if self.missing == "NOMISS": + sstart = sstart - timedelta(weeks=1) + retry_count = retry_count - 1 + continue + + if self.time: + self.type = "SCALAR" + if self.missing == "NOMISS": + # Get the last one, in case we fetched extra because of NOMISS + for v in reversed(self.values): + if v is not None and v[1] is not None: + self.value = v[1] + break + elif len(self.values) > 0: + self.value = self.values[-1][1] + + except Exception as err: + print( repr(err) + " : " + str(err), file=sys.stderr ) + + break + + elif self.type == "LEVEL": + if units not in CDA_UNIT_SYSTEMS: + raise ValueError(f"\n\nInvalid unit system. Valid values for levels are {', '.join(CDA_UNIT_SYSTEMS)} i.e. dbunits=\"EN\"") + levelData = LevelsApi.getLevels( + levelIdMask=self.levelid, + begin=None if not self.start else self.start.strftime(CDA_DATE_FORMAT), + end=None if not self.end else self.end.strftime(CDA_DATE_FORMAT), + office=self.dbofc, + timeZone=str(tz), + pageSize=-1, + unit=units + ) + print(levelData) + for level in levelData["location-levels"]["location-levels"]: + print(level) + + self.type = "SCALAR" - except Exception as err: - print( repr(err) + " : " + str(err), file=sys.stderr ) - break elif self.dbtype.upper() == "DSS": raise Exception("DSS retrieval is not currently implemented") else: diff --git a/tests/various/level/testfiles/swt.levels.frm b/tests/various/level/testfiles/swt.levels.frm new file mode 100644 index 0000000..e23e38f --- /dev/null +++ b/tests/various/level/testfiles/swt.levels.frm @@ -0,0 +1,20 @@ +#FORM +Keystone Lake Levels + +%TOP_OF_DAM +%TOP_OF_CONSERVATION +%BOTTOM_OF_CONSERVATION + +#ENDFORM +#DEF + +toc_level =Value( + dbtype = "CDA", + levelId = "KEYS.Elev.Inst.0.Top of Conservation", + misstr = "-M-", + undef = "-?-", + dbunits = "EN", + ) +print(str(toc_level)) +#ENDDEF + \ No newline at end of file From cda7a0f7d6778d06ac201ed2e15ee14e0c4c5df8 Mon Sep 17 00:00:00 2001 From: Charlie Date: Tue, 20 Aug 2024 23:18:46 -0500 Subject: [PATCH 14/17] Cleanup print statements, think on how levels should print to the form --- repgen/data/value.py | 20 +++++++++----------- tests/various/level/testfiles/swt.levels.frm | 4 ++-- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/repgen/data/value.py b/repgen/data/value.py index 9715bd8..cf4993b 100644 --- a/repgen/data/value.py +++ b/repgen/data/value.py @@ -189,7 +189,6 @@ def processDateTime(value, key, extra_part=None): pending_date = None # update the shared keywords - print(kwargs) for key in kwargs: value = kwargs[key] # If the value is wrapped in quotes, it's most likely wrong (possibly value was read from a file where it had quotes). @@ -292,7 +291,6 @@ def processDateTime(value, key, extra_part=None): raise ValueError("Only 1 non named value is allowed") # If the levelId is set, go for that first - print('dbtype', self.dbtype) if self.levelid: self.type = "LEVEL" else: @@ -422,7 +420,6 @@ def parse_slice(value): except Exception as err: print( repr(err) + " : " + str(err), file=sys.stderr ) elif self.dbtype.upper() in ["JSON", "CDA"]: - print(self.start, self.end) tz = self.dbtz units = self.dbunits @@ -433,7 +430,6 @@ def parse_slice(value): assert send.tzinfo is not None, "Naive datetime; end time should contain timezone" start = sstart.astimezone(tz) end = send.astimezone(tz) - print("type", self.type) if self.type == "TIMESERIES": if self.start is None or self.end is None: return @@ -612,11 +608,14 @@ def parse_slice(value): pageSize=-1, unit=units ) - print(levelData) for level in levelData["location-levels"]["location-levels"]: - print(level) - - self.type = "SCALAR" + # print(level) + for _d, _v in level["values"]["segments"][0]["values"]: + _dt = datetime.datetime.strptime(_d, CDA_DATE_FORMAT + "%z") + _dt = _dt.astimezone(self.dbtz) + self.values.append( ( _dt,_v, 0 if _v is not None else 5 ) ) + print(self.values) + # self.type = "SCALAR" elif self.dbtype.upper() == "DSS": @@ -838,15 +837,16 @@ def __le__(self, other): return NotImplemented def __str__(self): + print(self.type, self.value, self.values) if self.type=="SCALAR": return self.format(self.value) else: return "Unable to process at this time" + def __repr__(self): return "" % (self.type,str(self.value),len(self.values),self.picture) def format(self,value): - #print repr(value) if self.ismissing(value) or isinstance(value, list): return self.misstr @@ -859,7 +859,6 @@ def format(self,value): prefix = self.picture[0:specifier_start] picture = re.search(r"%([0-9.,+-]*[bcdoxXneEfFgG%])", self.picture).group(1) suffix = self.picture[self.picture.index(picture) + len(picture):] - if isinstance(value, (Decimal,float)) and not math.isfinite(value): result = prefix + self.undef + suffix else: @@ -914,7 +913,6 @@ def pop(self): self.index = 0 self.index = self.index+1 try: - #print repr(self.values[self.index-1]) return self.format(self.values[self.index-1][1]) except IndexError: # If data is missing, just return the undefined string value diff --git a/tests/various/level/testfiles/swt.levels.frm b/tests/various/level/testfiles/swt.levels.frm index e23e38f..3681194 100644 --- a/tests/various/level/testfiles/swt.levels.frm +++ b/tests/various/level/testfiles/swt.levels.frm @@ -8,13 +8,13 @@ Keystone Lake Levels #ENDFORM #DEF -toc_level =Value( +TOP_OF_CONSERVATION =Value( dbtype = "CDA", levelId = "KEYS.Elev.Inst.0.Top of Conservation", misstr = "-M-", undef = "-?-", dbunits = "EN", ) -print(str(toc_level)) +print(str(TOP_OF_CONSERVATION)) #ENDDEF \ No newline at end of file From 52273b808965a91925a8318adb5ea802303e6d82 Mon Sep 17 00:00:00 2001 From: Charlie Date: Wed, 21 Aug 2024 00:34:47 -0500 Subject: [PATCH 15/17] Initial working state for levels. Piggybacking on how Timeseries are handled. Updated form for various scenarios. --- repgen/data/value.py | 16 +++--- repgen/util/debug.py | 57 ++++++++++++++++++++ tests/various/level/testfiles/swt.levels.frm | 30 ++++++++--- 3 files changed, 88 insertions(+), 15 deletions(-) create mode 100644 repgen/util/debug.py diff --git a/repgen/data/value.py b/repgen/data/value.py index cf4993b..41907a5 100644 --- a/repgen/data/value.py +++ b/repgen/data/value.py @@ -147,7 +147,7 @@ def processDateTime(value, key, extra_part=None): self.type="SCALAR" self.value = None self.values = [] - self.picture="%s" + self.picture = None # Normalize the keyword names to lowercase kwargs = {key.lower(): value for key, value in kwargs.items()} # Pop the TSID from the kwargs, so it doesn't get passed to the constructor/deep copy @@ -293,7 +293,13 @@ def processDateTime(value, key, extra_part=None): # If the levelId is set, go for that first if self.levelid: self.type = "LEVEL" + # Convert the remainder picture to the level format + # Leftover from the BASDATE amd other shared value calls + if self.picture == "%Y%b%d %H%M": + self.picture = "%5.2f" else: + if not self.picture: + self.picture = "%s" self.type = "TIMESERIES" self.values = [ ] # will be a tuple of (time stamp, value, quality ) if self.dbtype.upper() == "RADAR": @@ -609,13 +615,10 @@ def parse_slice(value): unit=units ) for level in levelData["location-levels"]["location-levels"]: - # print(level) for _d, _v in level["values"]["segments"][0]["values"]: _dt = datetime.datetime.strptime(_d, CDA_DATE_FORMAT + "%z") _dt = _dt.astimezone(self.dbtz) self.values.append( ( _dt,_v, 0 if _v is not None else 5 ) ) - print(self.values) - # self.type = "SCALAR" elif self.dbtype.upper() == "DSS": @@ -837,7 +840,6 @@ def __le__(self, other): return NotImplemented def __str__(self): - print(self.type, self.value, self.values) if self.type=="SCALAR": return self.format(self.value) else: @@ -908,7 +910,7 @@ def format(self,value): def pop(self): if self.type == "SCALAR": return self.format(self.value) - elif self.type == "TIMESERIES": + elif self.type in ["TIMESERIES", "LEVEL"]: if self.index is None: self.index = 0 self.index = self.index+1 @@ -930,7 +932,7 @@ def datatimes(self): Value.shared["dbtype"]=typ tmp.type = self.type - if self.type == "TIMESERIES": + if self.type in ["TIMESERIES", "LEVEL"]: for v in self.values: tmp.values.append( (v[0],v[0],v[2]) ) elif self.type == "SCALAR": diff --git a/repgen/util/debug.py b/repgen/util/debug.py new file mode 100644 index 0000000..a28387b --- /dev/null +++ b/repgen/util/debug.py @@ -0,0 +1,57 @@ +import functools + +# These are used to help trace where repgen is in the callstack +# They are not used in the code, and are used to help debug issues + + +def log_calls(func): + """ + Decorator to log function calls. + + Args: + func (callable): The function to be decorated. + + Returns: + callable: The decorated function. + + Example: + @log_calls + def my_function(arg1, arg2): + print(f'Calling my_function with args: {arg1} and {arg2}') + result = arg1 + arg2 + print(f'my_function returned {result}') + return result + """ + @functools.wraps(func) + def wrapper(*args, **kwargs): + print(f'Calling {func.__name__} with args: {args} and kwargs: {kwargs}') + result = func(*args, **kwargs) + print(f'{func.__name__} returned {result}') + return result + return wrapper + +def decorate_all_methods(cls): + ''' + Decorate all methods of a class with the log_calls decorator. + + Args: + cls (class): The class to be decorated. + + Returns: + class: The decorated class. + + Example: + @decorate_all_methods + class MyClass: + def my_method(self, arg1, arg2): + print(f'Calling my_method with args: {arg1} and {arg2}') + result = arg1 + arg2 + print(f'my_method returned {result}') + return result + ''' + for attribute_name in dir(cls): + attribute = getattr(cls, attribute_name) + if callable(attribute) and not attribute_name.startswith("__"): + wrapped = log_calls(attribute) + setattr(cls, attribute_name, wrapped) + return cls \ No newline at end of file diff --git a/tests/various/level/testfiles/swt.levels.frm b/tests/various/level/testfiles/swt.levels.frm index 3681194..fd9df64 100644 --- a/tests/various/level/testfiles/swt.levels.frm +++ b/tests/various/level/testfiles/swt.levels.frm @@ -1,20 +1,34 @@ #FORM Keystone Lake Levels -%TOP_OF_DAM -%TOP_OF_CONSERVATION -%BOTTOM_OF_CONSERVATION +Top of Dam %TOP_OF_DAM %TOD_DT +Top of Dam %TOP_OF_DAM %TOD_DT +Top of Dam %TOP_OF_DAM %TOD_DT +Top of Cons %TOP_OF_CONSERVATION %TOC_DT +Bottom of Cons %BOTTOM_OF_CONSERVATION %BOC_DT #ENDFORM #DEF +TOP_OF_DAM = Value( + dbtype = "CDA", + misstr = "-M-", + undef = "-?-", + levelId = "KEYS.Elev.Inst.0.Top of Dam", + dbunits = "EN", +) TOP_OF_CONSERVATION =Value( - dbtype = "CDA", levelId = "KEYS.Elev.Inst.0.Top of Conservation", - misstr = "-M-", - undef = "-?-", - dbunits = "EN", ) -print(str(TOP_OF_CONSERVATION)) + +BOTTOM_OF_CONSERVATION = Value( + levelId = "KEYS.Elev.Inst.0.Bottom of Conservation", + picture = "%5.3f", +) +TOD_DT = Value(TOP_OF_DAM.datatimes(), + PICTURE="%d%b%Y %K%M", +) +TOC_DT = Value(TOP_OF_CONSERVATION.datatimes()) +BOC_DT = Value(BOTTOM_OF_CONSERVATION.datatimes()) #ENDDEF \ No newline at end of file From ddbb200a69acfb3b35ec59d6c64589e54db099c9 Mon Sep 17 00:00:00 2001 From: "Charles Graham, SWT" Date: Wed, 21 Aug 2024 21:11:01 -0500 Subject: [PATCH 16/17] Add misc start/end dates to test --- tests/various/level/testfiles/swt.levels.frm | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/various/level/testfiles/swt.levels.frm b/tests/various/level/testfiles/swt.levels.frm index fd9df64..088e50b 100644 --- a/tests/various/level/testfiles/swt.levels.frm +++ b/tests/various/level/testfiles/swt.levels.frm @@ -19,10 +19,13 @@ TOP_OF_DAM = Value( TOP_OF_CONSERVATION =Value( levelId = "KEYS.Elev.Inst.0.Top of Conservation", + start=BASDATE.value.replace(month=6, day=21, year=2019, hour=8, minute=0, second=0, microsecond=0), + end=BASDATE.value.replace(hour=8, minute=0, second=0, microsecond=0), ) BOTTOM_OF_CONSERVATION = Value( levelId = "KEYS.Elev.Inst.0.Bottom of Conservation", + start=BASDATE.value.replace(month=6, day=21, year=2020, hour=8, minute=0, second=0, microsecond=0), picture = "%5.3f", ) TOD_DT = Value(TOP_OF_DAM.datatimes(), From e2647421110f3b2c9b2f0b3391f9aead24bb5eeb Mon Sep 17 00:00:00 2001 From: Charles Graham Date: Tue, 10 Sep 2024 14:12:35 -0500 Subject: [PATCH 17/17] Add initial checks for if a user has a Value as a TS and switches to a level and vice versa. --- .prettierrc | 9 +++++++++ repgen/data/value.py | 14 ++++++++++---- 2 files changed, 19 insertions(+), 4 deletions(-) create mode 100644 .prettierrc diff --git a/.prettierrc b/.prettierrc new file mode 100644 index 0000000..f507a13 --- /dev/null +++ b/.prettierrc @@ -0,0 +1,9 @@ +{ + "tabWidth": 4, + "useTabs": true, + "semi": true, + "singleQuote": true, + "trailingComma": "es5", + "printWidth": 80 + } + \ No newline at end of file diff --git a/repgen/data/value.py b/repgen/data/value.py index 41907a5..d964136 100644 --- a/repgen/data/value.py +++ b/repgen/data/value.py @@ -194,7 +194,14 @@ def processDateTime(value, key, extra_part=None): # If the value is wrapped in quotes, it's most likely wrong (possibly value was read from a file where it had quotes). if isinstance(value, str) and len(value) > 0 and value[0] == '"' and value[-1] == '"': value = value[1:-1] - + print("key", key, 'value', value) + if key in ["dbloc", "dbpar", "dbptyp","dbint","dbdur","dbver"]: + # Switch to timeseries when a user actively provides a TS parameter + if self.type == "LEVEL": self.type = "TIMESERIES" + elif key == "levelid": + # Switch to timeseries when a user actively provides a TS parameter + if self.type in ["SCALAR", "TIMESERIES"]: + self.type = "LEVEL" if (key == "tz" or key == "dbtz") and isinstance(value, string_types): value = pytz.timezone(value) elif key == "start" or key == "end" or key.endswith("time") or key.endswith("date"): @@ -203,7 +210,6 @@ def processDateTime(value, key, extra_part=None): value = value.values[0][0] else: value = value.value # internally we want the actual datetime - if key.endswith("time"): pending_time = value elif key.endswith("date"): @@ -291,8 +297,9 @@ def processDateTime(value, key, extra_part=None): raise ValueError("Only 1 non named value is allowed") # If the levelId is set, go for that first + print(self.levelid) + print(self.type) if self.levelid: - self.type = "LEVEL" # Convert the remainder picture to the level format # Leftover from the BASDATE amd other shared value calls if self.picture == "%Y%b%d %H%M": @@ -300,7 +307,6 @@ def processDateTime(value, key, extra_part=None): else: if not self.picture: self.picture = "%s" - self.type = "TIMESERIES" self.values = [ ] # will be a tuple of (time stamp, value, quality ) if self.dbtype.upper() == "RADAR": print("\n\tWARNING: Update from dbtype=\"RADAR\" to dbtype=\"CDA\"")