diff --git a/.gitignore b/.gitignore index 776a7183..8bc2be71 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ **/target tmp **/\.~lock* +scripts # Byte-compiled / optimized / DLL files **/__pycache__/ diff --git a/cwms/api.py b/cwms/api.py index 31afe02a..0784592e 100644 --- a/cwms/api.py +++ b/cwms/api.py @@ -26,6 +26,7 @@ the error. """ +import base64 import json import logging from json import JSONDecodeError @@ -188,20 +189,8 @@ def get_xml( Raises: ApiError: If an error response is return by the API. """ - - headers = {"Accept": api_version_text(api_version)} - response = SESSION.get(endpoint, params=params, headers=headers) - response.close() - - if response.status_code < 200 or response.status_code >= 300: - logging.error(f"CDA Error: response={response}") - raise ApiError(response) - - try: - return response.content.decode("utf-8") - except JSONDecodeError as error: - logging.error(f"Error decoding CDA response as xml: {error}") - return {} + # Wrap the primary get for backwards compatibility + return get(endpoint=endpoint, params=params, api_version=api_version) def get( @@ -209,7 +198,7 @@ def get( params: Optional[RequestParams] = None, *, api_version: int = API_VERSION, -) -> JSON: +) -> Any: """Make a GET request to the CWMS Data API. Args: @@ -228,17 +217,28 @@ def get( """ headers = {"Accept": api_version_text(api_version)} - response = SESSION.get(endpoint, params=params, headers=headers) - response.close() - if response.status_code < 200 or response.status_code >= 300: - logging.error(f"CDA Error: response={response}") - raise ApiError(response) - - try: - return cast(JSON, response.json()) - except JSONDecodeError as error: - logging.error(f"Error decoding CDA response as json: {error}") - return {} + with SESSION.get(endpoint, params=params, headers=headers) as response: + if not response.ok: + logging.error(f"CDA Error: response={response}") + raise ApiError(response) + try: + # Avoid case sensitivity issues with the content type header + content_type = response.headers.get("Content-Type", "").lower() + # Most CDA content is JSON + if "application/json" in content_type or not content_type: + return cast(JSON, response.json()) + # Use automatic charset detection with .text + if "text/plain" in content_type or "text/" in content_type: + return response.text + if content_type.startswith("image/"): + return base64.b64encode(response.content).decode("utf-8") + # Fallback for remaining content types + return response.content.decode("utf-8") + except JSONDecodeError as error: + logging.error( + f"Error decoding CDA response as JSON: {error} on line {error.lineno}\n\tFalling back to text" + ) + return response.text def get_with_paging( @@ -247,7 +247,7 @@ def get_with_paging( params: RequestParams, *, api_version: int = API_VERSION, -) -> JSON: +) -> Any: """Make a GET request to the CWMS Data API with paging. Args: @@ -312,12 +312,10 @@ def post( if isinstance(data, dict) or isinstance(data, list): data = json.dumps(data) - response = SESSION.post(endpoint, params=params, headers=headers, data=data) - response.close() - - if response.status_code < 200 or response.status_code >= 300: - logging.error(f"CDA Error: response={response}") - raise ApiError(response) + with SESSION.post(endpoint, params=params, headers=headers, data=data) as response: + if not response.ok: + logging.error(f"CDA Error: response={response}") + raise ApiError(response) def patch( @@ -346,16 +344,13 @@ def patch( """ headers = {"accept": "*/*", "Content-Type": api_version_text(api_version)} - if data is None: - response = SESSION.patch(endpoint, params=params, headers=headers) - else: - if isinstance(data, dict) or isinstance(data, list): - data = json.dumps(data) - response = SESSION.patch(endpoint, params=params, headers=headers, data=data) - response.close() - if response.status_code < 200 or response.status_code >= 300: - logging.error(f"CDA Error: response={response}") - raise ApiError(response) + + if data and isinstance(data, dict) or isinstance(data, list): + data = json.dumps(data) + with SESSION.patch(endpoint, params=params, headers=headers, data=data) as response: + if not response.ok: + logging.error(f"CDA Error: response={response}") + raise ApiError(response) def delete( @@ -379,8 +374,7 @@ def delete( """ headers = {"Accept": api_version_text(api_version)} - response = SESSION.delete(endpoint, params=params, headers=headers) - response.close() - if response.status_code < 200 or response.status_code >= 300: - logging.error(f"CDA Error: response={response}") - raise ApiError(response) + with SESSION.delete(endpoint, params=params, headers=headers) as response: + if not response.ok: + logging.error(f"CDA Error: response={response}") + raise ApiError(response) diff --git a/cwms/catalog/blobs.py b/cwms/catalog/blobs.py index 3e622697..d94758c2 100644 --- a/cwms/catalog/blobs.py +++ b/cwms/catalog/blobs.py @@ -1,29 +1,40 @@ +import base64 from typing import Optional import cwms.api as api from cwms.cwms_types import JSON, Data +from cwms.utils.checks import is_base64 +STORE_DICT = """data = { + "office-id": "SWT", + "id": "MYFILE_OR_BLOB_ID.TXT", + "description": "Your description here", + "media-type-id": "application/octet-stream", + "value": "STRING of content or BASE64_ENCODED_STRING" +} +""" -def get_blob(blob_id: str, office_id: str) -> Data: - """Get a single clob. + +def get_blob(blob_id: str, office_id: str) -> str: + """Get a single BLOB (Binary Large Object). Parameters ---------- blob_id: string - Specifies the id of the blob + Specifies the id of the blob. ALL blob ids are UPPERCASE. office_id: string Specifies the office of the blob. Returns ------- - cwms data type. data.json will return the JSON output and data.df will return a dataframe + str: the value returned based on the content-type it was stored with as a string """ endpoint = f"blobs/{blob_id}" params = {"office": office_id} response = api.get(endpoint, params, api_version=1) - return Data(response) + return str(response) def get_blobs( @@ -50,36 +61,39 @@ def get_blobs( endpoint = "blobs" params = {"office": office_id, "page-size": page_size, "like": blob_id_like} - response = api.get(endpoint, params, api_version=1) + response = api.get(endpoint, params, api_version=2) return Data(response, selector="blobs") def store_blobs(data: JSON, fail_if_exists: Optional[bool] = True) -> None: - """Create New Blob + f"""Create New Blob Parameters - ---------- - Data: JSON dictionary - JSON containing information of Blob to be updated - { - "office-id": "string", - "id": "string", - "description": "string", - "media-type-id": "string", - "value": "string" - } - fail_if_exists: Boolean - Create will fail if provided ID already exists. Default: true + ---------- + **Note**: The "id" field is automatically cast to uppercase. - Returns - ------- - None + Data: JSON dictionary + JSON containing information of Blob to be updated. + + {STORE_DICT} + fail_if_exists: Boolean + Create will fail if the provided ID already exists. Default: True + + Returns + ------- + None """ if not isinstance(data, dict): - raise ValueError("Cannot store a Blob without a JSON data dictionary") + raise ValueError( + f"Cannot store a Blob without a JSON data dictionary:\n{STORE_DICT}" + ) + + # Encode value if it's not already Base64-encoded + if "value" in data and not is_base64(data["value"]): + # Encode to bytes, then Base64, then decode to string for storing + data["value"] = base64.b64encode(data["value"].encode("utf-8")).decode("utf-8") endpoint = "blobs" params = {"fail-if-exists": fail_if_exists} - return api.post(endpoint, data, params, api_version=1) diff --git a/cwms/ratings/ratings.py b/cwms/ratings/ratings.py index f5b36650..b106bd62 100644 --- a/cwms/ratings/ratings.py +++ b/cwms/ratings/ratings.py @@ -15,7 +15,7 @@ def rating_current_effective_date(rating_id: str, office_id: str) -> Any: """Retrieve the most recent effective date for a specific rating id. Returns - datatime + Any the datetime of the most recent effective date for a rating id. If max effective date is not present for rating_id then None will be returned @@ -46,7 +46,7 @@ def get_current_rating( The owning office of the rating specifications. If no office is provided information from all offices will be returned rating_table_in_df: Bool, Optional Default = True - define if the independant and dependant variables should be stored as a dataframe + define if the independent and dependant variables should be stored as a dataframe Returns ------- Data : Data @@ -112,7 +112,7 @@ def get_ratings_xml( timezone: Optional[str] = None, method: Optional[str] = "EAGER", ) -> Any: - """Retrives ratings for a specific rating-id + """Retrieves ratings for a specific rating-id Parameters ---------- @@ -124,7 +124,7 @@ def get_ratings_xml( begin: datetime, optional the start of the time window for data to be included in the response. This is based on the effective date of the ratings end: datetime, optional - the end of the time window for data to be included int he reponse. This is based on the effective date of the ratings + the end of the time window for data to be included int he response. This is based on the effective date of the ratings timezone: the time zone of the values in the being and end fields if not specified UTC is used method: @@ -225,13 +225,13 @@ def rating_simple_df_to_json( active: Optional[bool] = True, ) -> JSON: """This function converts a dataframe to a json dictionary in the correct format to be posted using the store_ratings function. Can - only be used for simple ratings with a indenpendant and 1 dependant variable. + only be used for simple ratings with a independent and 1 dependant variable. Parameters ---------- data: pd.Dataframe Rating Table to be stored to an exiting rating specification and template. Can only have 2 columns ind and dep. ind - contained the indenpendant variable and dep contains the dependent variable. + contained the independent variable and dep contains the dependent variable. ind dep 0 9.62 0.01 1 9.63 0.01 @@ -249,7 +249,7 @@ def rating_simple_df_to_json( office_id: str the owning office of the rating units: str - units for both the independant and dependent variable seperated by ; i.e. ft;cfs or ft;ft. + units for both the independent and dependent variable separated by ; i.e. ft;cfs or ft;ft. effective_date: datetime, The effective date of the rating curve to be stored. transition_start_date: datetime Optional = None @@ -384,7 +384,7 @@ def delete_ratings( def store_rating(data: Any, store_template: Optional[bool] = True) -> None: - """Will create a new ratingset including template/spec and rating + """Will create a new rating-set including template/spec and rating Parameters ---------- @@ -403,7 +403,7 @@ def store_rating(data: Any, store_template: Optional[bool] = True) -> None: if not isinstance(data, dict) and xml_heading not in data: raise ValueError( - "Cannot store a timeseries without a JSON data dictionaryor in XML" + "Cannot store a timeseries without a JSON data dictionary or in XML" ) if xml_heading in data: diff --git a/cwms/ratings/ratings_spec.py b/cwms/ratings/ratings_spec.py index 684bbdb6..5d08abb7 100644 --- a/cwms/ratings/ratings_spec.py +++ b/cwms/ratings/ratings_spec.py @@ -8,7 +8,7 @@ def get_rating_spec(rating_id: str, office_id: str) -> Data: - """Retrives a single rating spec + """Retrieves a single rating spec Parameters ---------- @@ -37,7 +37,7 @@ def get_rating_specs( rating_id_mask: Optional[str] = None, page_size: int = 500000, ) -> Data: - """Retrives a list of rating specification + """Retrieves a list of rating specification Parameters ---------- @@ -45,7 +45,7 @@ def get_rating_specs( The owning office of the rating specifications. If no office is provided information from all offices will be returned rating-id-mask: string, optional - Posix regular expression that specifies the rating ids to be included in the reponce. If not specified all + Posix regular expression that specifies the rating ids to be included in the response. If not specified all rating specs shall be returned. page-size: int, optional, default is 5000000: Specifies the number of records to obtain in a single call. @@ -111,7 +111,7 @@ def rating_spec_df_to_xml(data: pd.DataFrame) -> str: Parameters ---------- data : pd_dataframe - pandas dataframe that contrains rating specification paramters + pandas dataframe that contains rating specification parameters should follow same formate the is returned from get_rating_spec function Returns ------- @@ -134,10 +134,10 @@ def rating_spec_df_to_xml(data: pd.DataFrame) -> str: {str(data.loc[0,'auto-migrate-extension']).lower()} """ - ind_rouding = data.loc[0, "independent-rounding-specs"] - if isinstance(ind_rouding, list): + ind_rounding = data.loc[0, "independent-rounding-specs"] + if isinstance(ind_rounding, list): i = 1 - for rounding in ind_rouding: + for rounding in ind_rounding: spec_xml = ( spec_xml + f"""\n {rounding['value']}""" diff --git a/cwms/timeseries/timeseries.py b/cwms/timeseries/timeseries.py index 4eef62a6..1e72e5f3 100644 --- a/cwms/timeseries/timeseries.py +++ b/cwms/timeseries/timeseries.py @@ -21,8 +21,8 @@ def get_multi_timeseries_df( Parameters ---------- - ts_ids: linst - a list of timeseries to get. If the timeseries is a verioned timeseries then serpeate the ts_id from the + ts_ids: list + a list of timeseries to get. If the timeseries is a versioned timeseries then separate the ts_id from the version_date using a :. Example "OMA.Stage.Inst.6Hours.0.Fcst-MRBWM-GRFT:2024-04-22 07:00:00-05:00". Make sure that the version date include the timezone offset if not in UTC. office_id: string @@ -163,7 +163,7 @@ def get_timeseries( not specified, any required time window ends at the current time. Any timezone information should be passed within the datetime object. If no timezone information is given, default will be UTC. - page_size: int, optional, default is 5000000: Sepcifies the number of records to obtain in + page_size: int, optional, default is 5000000: Specifies the number of records to obtain in a single call. version_date: datetime, optional, default is None Version date of time series values being requested. If this field is not specified and @@ -208,7 +208,7 @@ def timeseries_df_to_json( office_id: str, version_date: Optional[datetime] = None, ) -> JSON: - """This function converts a dataframe to a json dictionary in the correct format to be posted using the store_timeseries fucntion. + """This function converts a dataframe to a json dictionary in the correct format to be posted using the store_timeseries function. Parameters ---------- @@ -223,7 +223,7 @@ def timeseries_df_to_json( 2 2023-12-20T15:15:00.000-05:00 98.5 0 3 2023-12-20T15:30:00.000-05:00 98.5 0 ts_id: str - timeseried id:specified name of the timeseries to be posted to + timeseries id:specified name of the timeseries to be posted to office_id: str the owning office of the time series units: str @@ -280,7 +280,7 @@ def store_timeseries( ---------- data: JSON dictionary Time Series data to be stored. - create_as_ltrs: bool, optional, defualt is False + create_as_ltrs: bool, optional, default is False Flag indicating if timeseries should be created as Local Regular Time Series. store_rule: str, optional, default is None: The business rule to use when merging the incoming with existing data. Available values : diff --git a/cwms/utils/__init__.py b/cwms/utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cwms/utils/checks.py b/cwms/utils/checks.py new file mode 100644 index 00000000..e16e7235 --- /dev/null +++ b/cwms/utils/checks.py @@ -0,0 +1,10 @@ +import base64 + + +def is_base64(s: str) -> bool: + """Check if a string is Base64 encoded.""" + try: + decoded = base64.b64decode(s, validate=True) + return base64.b64encode(decoded).decode("utf-8") == s + except (ValueError, TypeError): + return False diff --git a/pyproject.toml b/pyproject.toml index 5e647bfc..922e27c3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,12 +1,14 @@ [tool.poetry] name = "cwms-python" repository = "https://github.com/HydrologicEngineeringCenter/cwms-python" -version = "0.6.3" + +version = "0.6.4" + packages = [ { include = "cwms" }, ] -description = "Corps water managerment systems (CWMS) REST API for Data Retrieval of USACE water data" +description = "Corps water management systems (CWMS) REST API for Data Retrieval of USACE water data" readme = "README.md" license = "LICENSE" keywords = ["USACE", "water data", "CWMS"] diff --git a/tests/api_test.py b/tests/api_test.py index 86c5e3e3..86d7450d 100644 --- a/tests/api_test.py +++ b/tests/api_test.py @@ -48,10 +48,3 @@ def test_api_headers(): version = api_version_text(api_version=2) assert version == "application/json;version=2" - - -def test_api_headers_invalid_version(): - """An exception should be raised if the API version is not valid.""" - - with pytest.raises(InvalidVersion): - version = api_version_text(api_version=3)