From 126359493c91c89cc89be1f4c5fcabf65c4305b8 Mon Sep 17 00:00:00 2001 From: Kieran Date: Mon, 3 Mar 2025 20:08:26 -0700 Subject: [PATCH 1/3] Added requests v2.32.3 to dependencies. --- Pipfile | 1 + pyproject.toml | 1 + 2 files changed, 2 insertions(+) diff --git a/Pipfile b/Pipfile index 8020d1b..020cb13 100644 --- a/Pipfile +++ b/Pipfile @@ -7,6 +7,7 @@ name = "pypi" python-dateutil = "==2.8.2" pydantic = "==2.0a4" aiohttp = "==3.8.4" +requests = "==2.32.3" [dev-packages] black = "*" diff --git a/pyproject.toml b/pyproject.toml index 5bde95c..eec67d0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,6 +9,7 @@ description = "An async Python client for working with Colorado Avalanche Inform readme = "README.md" requires-python = ">=3.10" dependencies = [ + "requests==2.32.3", "aiohttp==3.8.4", "pydantic==2.0a4", "python-dateutil==2.8.*", From 9f0daba6fa20023a90fa023694ca421a400b02a9 Mon Sep 17 00:00:00 2001 From: Kieran Date: Mon, 3 Mar 2025 20:29:22 -0700 Subject: [PATCH 2/3] Added SyncCaicClient class for synchronous API functionality --- src/caic_python/client.py | 598 +++++++++++++++++++++++++++++++++++++- 1 file changed, 596 insertions(+), 2 deletions(-) diff --git a/src/caic_python/client.py b/src/caic_python/client.py index d184e23..2606355 100644 --- a/src/caic_python/client.py +++ b/src/caic_python/client.py @@ -57,6 +57,7 @@ import time import typing +import requests import aiohttp import aiohttp.http import pydantic @@ -152,10 +153,10 @@ async def _get(self, url: str, params: dict | list | None = None) -> dict: try: resp = await self.session.get(url, params=params) - if resp.status >= 400: + if resp.status_code >= 400: error = await resp.text() raise errors.CaicRequestException( - f"Error status from CAIC: {resp.status} - {error}" + f"Error status from CAIC: {resp.status_code} - {error}" ) data = await resp.json() @@ -697,3 +698,596 @@ async def avy_forecast( LOGGER.error("Unable to decode forecast response: %s", str(err)) return ret + +class SyncCaicClient: + """A syncronous HTTP client for the CAIC API(s).""" + + def __init__(self) -> None: + self.headers = { + "User-Agent": f"caic-python/{__version__}" + } + self.session = requests.Session() + + def close(self) -> None: + """Close the underlying session.""" + self.session.close() + + def _get(self, url: str, params: dict | list | None = None) -> dict: + """ + Get a URL using this client. + + Meant to be ``CaicURLs`` agnostic, so pass in a full URL. + + Parameters + ---------- + url : str + The full URL, in other words, an attr of ``CaicURLs`` + an API endpoint. + params : dict | None, optional + Optional URL parameters to pass in - the CAIC + APIs rely on params, by default None. + + Returns + ------- + dict + The return from ``requests.Response.json`` if, this call, or + the HTTP request itself, did not throw an error. + + Raises + ------ + errors.CaicRequestException + For common HTTP errors, a >400 response status, + a ``requests.RequestException``, or a ``JSONDecodeError``. + """ + + data = {} + + try: + resp = self.session.get(url, params=params) + print("Response status:", resp.status_code) + if resp.status_code >= 400: + error = resp.text + raise errors.CaicRequestException( + f"Error status from CAIC: {resp.status_code} - {error}" + ) + + # print("Response JSON:", resp.json()) # Debugging + data = resp.json() + + except requests.RequestException as err: + raise errors.CaicRequestException( + f"Error connecting to CAIC: {err}" + ) from err + except JSONDecodeError as err: + raise errors.CaicRequestException( + f"Error decoding CAIC response: {err}" + ) from err + + else: + return data + + def _api_id_get( + self, obj_id: str, endpoint: str, resp_model: pydantic.BaseModel + ) -> models.FieldReport | None: + resp_data = self._get(f"{CaicURLs.API}/{endpoint}/{obj_id}.json") + + try: + return resp_model(**resp_data) + except pydantic.ValidationError as err: + LOGGER.warning( + "Error parsing '%s' response (ID: %s): %s", endpoint, obj_id, str(err) + ) + return None + + def _api_paginate_get( + self, page: int, per: int, uri: str, params: typing.Mapping | None = None + ) -> dict: + """ + A paginated get request to the CAIC API. + + Only supports ``CaicURLs.API`` endpoints - these seem to be the only + ones that need pagination. + + Parameters + ---------- + page : int + The page number to get from the API. + per : int + The number of items to get per page. + uri : str + The ``CaicURLs.API`` endpoint to get. + params : typing.Mapping | None, optional + Optional URL parameters to include with the get request. + Do not include ``page`` and ``per`` in ``params`` as these values + are overwritten/set by this method. By default None. + + Returns + ------- + dict | None + The API's JSON response as a dict. + + Raises + ------ + errors.CaicRequestException + If raised by ``_get``, + """ + + if params is None: + params = {} + + params["per"] = per + params["page"] = page + + print(CaicURLs.API + uri) + data = self._get(CaicURLs.API + uri, params=params) + + return data + + def _api_paginator( + self, + endpoint: str, + resp_model: pydantic.BaseModel, + params: dict | None, + per: int = 1000, + page_limit: int = 100, + retries: int = 2, + total_retries: int = 10, + ) -> list[pydantic.BaseModel]: + """ + Loop over ``_api_paginate_get`` until done, or conditions are met. + + WARNING: The ``/api/observation_reports`` doesn't give us pagination metadata. + This means it may be possible to get stuck in a loop if you intentionally + bypass safeguards in this method. + + Loop exit conditions: + - The total results from a given page are less than ``per`` - in other words, + we're on the last page. + - The TOTAL number of caught, logged, and then thrown out exceptions is + equal to ``total_retries``. + - The total number of pages retrieved by this method is equal to + ``page_limit``. + - If ``page_limit`` negative, this check is bypassed! + - If ``resp_model`` is ``models.V1AvyObsResponse``, there is page metadata. + The loop will exit when the metadata reports this method has retrieved + the last page. + - Keep in mind that the ``page_limit`` check described above + trumps this check. Users may have to increase or disable page + limit when calling this method on the ``api/avalanche_observations`` + endpoint. + + Parameters + ---------- + endpoint : str + The API endpoint to request. + resp_model : pydantic.BaseModel + The model used to cast the JSON body of each response to an object. + params : dict | None, optional + Optional parameters for the request, by default None. + per : int, optional + The number of items to request per page, by default 1000. + page_limit : int, optional + The maximum number of pages to get. Set this to a negative number + to disable this limit. By default 100. + DO NOT disable this limit if ``endpoint`` is ``/api/observation_reports``. + retries : int, optional + The number of retries on a given page before moving to the next page, + by default 2. + total_retries : int, optional + The total number of retries before this method quits, by default 10. + + Returns + ------- + list[pydantic.BaseModel] + A list of the ``pydantic.BaseModel`` objects defined by ``resp_model``. + An empty list may indicate no data or it may indicate that all requests + failed. + """ + + page = 1 + paginating = True + results = [] + retry_count = 0 + page_retries = 0 + + while paginating: + if retry_count == total_retries: + if not results: + LOGGER.critical("All queries failed!") + LOGGER.error("Reached the maximum number of query retries.") + break + + try: + resp = self._api_paginate_get(page, per, endpoint, params) + except Exception as err: # pylint: disable=W0718 + LOGGER.error( + "Failed to request the CAIC endpoint '%s': %s", endpoint, err + ) + if page_retries == retries: + page += 1 + retry_count += 1 + page_retries = 0 + else: + page_retries += 1 + retry_count += 1 + continue + + if len(resp) < per: + LOGGER.info("Got all the results for the query: %s", str(params)) + paginating = False + + try: + if resp_model == models.V1AvyResponse: + obj = resp_model(**resp) + else: + obj = [resp_model(**item) for item in resp] + except pydantic.ValidationError as err: + LOGGER.warning( + "Unable to validate response from the '%s' endpoint " + "(Page# %s - Query (%s)): %s", + endpoint, + page, + str(params), + str(err), + ) + # Can't find a way around the duplicate code here. + if page_retries == retries: + page += 1 + retry_count += 1 + page_retries = 0 + else: + page_retries += 1 + retry_count += 1 + continue + + if page == page_limit: + LOGGER.warning("Reached the page limit before all pages downloaded.") + paginating = False + + # Special handling for V1AvyObsResponse objects. + if resp_model == models.V1AvyResponse: + if obj.meta.current_page == obj.meta.total_pages: + paginating = False + # Just a sanity check to avoid infinite looping + elif page >= obj.meta.total_pages: + LOGGER.debug("Paginating mismatch") + paginating = False + + results.extend(obj.data) + + else: + results.extend(obj) + + page += 1 + + return results + + def _proxy_get( + self, proxy_endpoint: str, proxy_uri: str, proxy_params: dict + ) -> dict | list | None: + """Get a URL from the avalanche.state.co.us API proxy. + + This is an endpoint that can proxy to other CAIC APIs. + + Parameters + ---------- + proxy_endpoint : str + The actual endpoint of the proxy to request, the real URI in the + HTTP request. Can be any of ``ProxyEndpoints``. + proxy_uri : str + The URI used by the proxy in its request to the proxied API. + proxy_params : dict + URL parameters to pass in the request to the proxied API. + + Returns + ------- + dict | list + The response raw response, or None + """ + proxy_params_str = "&".join([f"{k}={v}" for k, v in proxy_params.items()]) + params = {"_api_proxy_uri": f"{proxy_uri}?{proxy_params_str}"} + return self._get(CaicURLs.HOME + proxy_endpoint, params=params) + + def avy_obs( + self, start: str, end: str, page_limit: int = 1000, ver1: bool = False + ) -> list[models.AvalancheObservation]: + """Query for avalanche observations on the CAIC website. + + Supports both the v1 and v2 APIs. FWIW, the website still uses v1 for this + particular endpoint. My guess is because it supports proper pagination + unlike v2, which supports pagination but does not return pagination + info in response objects so clients have to guess when they're done paging. + + ``start`` and ``end`` arguments should be in this format - ``YYYY-MM-DD HH:mm:ss``. + However, the return of a ``datetime.datetime.isoformat()`` call works as well. + + Parameters + ---------- + start : str + Query for avalanches observed after at this day (and time). + end : str + Query for avalanches observed before this day (and time). + page_limit : int, optional + Limit per page results to this amount, by default 1000. + ver1 : bool, optional + Use the v1 endpoint instead, not recommended, by default False. + + Returns + ------- + list[models.AvalancheObservation] + A list of all avalanche observations returned by the query. + """ + + if ver1: + endpoint = "/api/avalanche_observations" + model = models.V1AvyResponse + else: + endpoint = "/api/v2/avalanche_observations" + model = models.AvalancheObservation + + params = { + "observed_after": start, + "observed_before": end, + "t": str(int(time.time())), + } + + obs = self._api_paginator( + endpoint, + model, + params=params, + page_limit=page_limit, + ) + + return obs + + def field_reports( # pylint: disable=W0102 + self, + start: str, + end: str, + bc_zones: list[str] = [], + cracking_obs: list[str] = [], + collapsing_obs: list[str] = [], + query: str = "", + avy_seen: bool | None = None, + page_limit: int = 100, + ) -> list[models.FieldReport]: + """ + Search CAIC field reports. + + Replicates the search on the CAIC ``View Field Reports`` page. + + Parameters + ---------- + start : str + The start time for the query (observations after this date[+time]). + end : str + The end time for the query (observations before this date[+time]). + bc_zones : list[str], optional + One or more Backcountry Zones to limit the search results by. + Should be one of ``enums.BCZoneTitles``. By default []. + cracking_obs : list[str], optional + One or more Cracking Observations to limit the search results by. + Should be one of ``enums.CrackingObsNames``. By default []. + collapsing_obs : list[str], optional + One or more Collapsing Observations to limit the search results by. + Should be one of ``enums.CollapsingObsNames``. By default []. + query : str, optional + A query string to limit searches by, by default "". + avy_seen : bool | None, optional + Whether an avalanche was seen, or None to disable this search + modifier, by default None. + page_limit : int, optional + Limit the number of pages returned by the API. Must be at least 1 + or a value error is raised. By default 100. + + Returns + ------- + list[models.FieldReport] + All field reports returned by the search. + + Raises + ------ + ValueError + If ``page_limit`` is less than 1. + + """ + + if page_limit <= 0: + raise ValueError("A page_limit MUST be set for field_reports!") + + params = { + "r[backcountry_zone_title_in][]": list_to_plus_args(bc_zones), + "r[snowpack_observations_cracking_in]": list_to_plus_args(cracking_obs), + "r[snowpack_observations_collapsing_in][]": list_to_plus_args( + collapsing_obs + ), + "q": query, + "r[saw_avalanche_eq]": avy_seen, + "r[observed_at_gteq]": start, + "r[observed_at_lteq]": end, + "r[sorts][]": "observed_at+desc", # We'll hard code this. + } + + # Sanitize params + params = {k: v for k, v in params.items() if v not in (None, "")} + + obs = self._api_paginator( + "/api/v2/observation_reports", + models.FieldReport, + params=params, + page_limit=page_limit, + ) + + return obs + + def field_report(self, report_id: str) -> models.FieldReport | None: + """Get a single CAIC Feild Report (aka Observation Report) by UUID. + + Parameters + ---------- + report_id : str + The UUID of the field report to retrieve. + + Returns + ------- + models.FieldReport | None + The retrieved field report, or + None if there was an error validating response. + """ + + report = self._api_id_get( + report_id, CaicApiEndpoints.OBS_REPORT, models.FieldReport + ) + + return report + + def snowpack_observation( + self, obs_id: str + ) -> models.SnowpackObservation | None: + """Get a single snowpack observation by UUID. + + Parameters + ---------- + obs_id : str + The ID of the snowpack observation. + + Returns + ------- + models.SnowpackObservation | None + The retrieved snowpack observation, or + None if there was an error validating response. + """ + + report = self._api_id_get( + obs_id, CaicApiEndpoints.SNOWPACK_OBS, models.SnowpackObservation + ) + + return report + + def avy_observation(self, obs_id: str) -> models.AvalancheObservation | None: + """Get a single avalanche observation by UUID. + + Parameters + ---------- + obs_id : str + The UUID of the avalanche observation. + + Returns + ------- + models.AvalancheObservation | None + The retrieved avalanche observation, or + None if there was an error validating response. + """ + + report = self._api_id_get( + obs_id, CaicApiEndpoints.AVY_OBS, models.AvalancheObservation + ) + + return report + + def weather_observation( + self, obs_id: str + ) -> models.WeatherObservation | None: + """Get a single weather observation by UUID. + + Parameters + ---------- + obs_id : str + The UUID of the weather observation. + + Returns + ------- + models.WeatherObservation | None + The retrieved weather observation, or + None if there was an error validating response. + """ + report = self._api_id_get( + obs_id, CaicApiEndpoints.WEATHER_OBS, models.WeatherObservation + ) + + return report + + def bc_zone(self, zone_slug: str) -> models.BackcountryZone | None: + """Get a single backcountry zone by UUID. + + Parameters + ---------- + zone_slug : str + The zone's slug name. + + Returns + ------- + models.BackcountryZone | None + The backcountry zone, or + None if there was an error validating response. + """ + + report = self._api_id_get( + zone_slug, CaicApiEndpoints.ZONES, models.BackcountryZone + ) + + return report + + def highway_zone(self, zone_slug: str) -> models.HighwayZone | None: + """Get a single highway zone by UUID. + + Parameters + ---------- + zone_slug : str + The zone's slug name. + + Returns + ------- + models.HighwayZone | None + The highway zone, or + None if there was an error validating response. + """ + + report = self._api_id_get( + zone_slug, CaicApiEndpoints.ZONES, models.HighwayZone + ) + + return report + + def avy_forecast( + self, date: str + ) -> list[models.AvalancheForecast | models.RegionalDiscussionForecast]: + """Get the avalanche forecasts as they were on the given date. + + Forecasts cover the date given + the following two days. + + Parameters + ---------- + date : str + The date that the avalanche forecast was produced for. + + Returns + ------- + list[models.AvalancheForecast | models.RegionalDiscussionForecast] + A list of returned forecasts. The list should contain two types. + The localized forecast for detailed areas of CO, and the regional + discussion pieces that cover broader portions of the state. + """ + + params = {"datetime": date, "includeExpired": "true"} + resp = self._proxy_get( + proxy_endpoint=ProxyEndpoints.AVID, + proxy_uri="/products/all", + proxy_params=params, + ) + + ret = [] + + if resp: + try: + for item in resp: + if ( + isinstance(item, dict) + and item.get("type") == "avalancheforecast" + ): + ret.append(models.AvalancheForecast(**item)) + else: + ret.append(models.RegionalDiscussionForecast(**item)) + except pydantic.ValidationError as err: + LOGGER.error("Unable to decode forecast response: %s", str(err)) + + return ret \ No newline at end of file From 2df3bc61b3e856568f9cc300a249fa0ac4d74ec6 Mon Sep 17 00:00:00 2001 From: kieran-smith Date: Tue, 4 Mar 2025 10:36:20 -0700 Subject: [PATCH 3/3] Undoing accidental change to CaicClient --- src/caic_python/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/caic_python/client.py b/src/caic_python/client.py index 2606355..40ada2e 100644 --- a/src/caic_python/client.py +++ b/src/caic_python/client.py @@ -153,10 +153,10 @@ async def _get(self, url: str, params: dict | list | None = None) -> dict: try: resp = await self.session.get(url, params=params) - if resp.status_code >= 400: + if resp.status >= 400: error = await resp.text() raise errors.CaicRequestException( - f"Error status from CAIC: {resp.status_code} - {error}" + f"Error status from CAIC: {resp.status} - {error}" ) data = await resp.json()