From cd1666d0d546caedaf9837fbb4abb1d3774af369 Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Mon, 20 Nov 2023 06:58:56 -0500 Subject: [PATCH 1/5] Adding search by sdss_id and catalog_id under the query endpoint. --- python/valis/db/queries.py | 51 ++++++++++++++++++++++++++++++++++++ python/valis/routes/query.py | 15 ++++++++++- 2 files changed, 65 insertions(+), 1 deletion(-) diff --git a/python/valis/db/queries.py b/python/valis/db/queries.py index 857fb6f..966daaf 100644 --- a/python/valis/db/queries.py +++ b/python/valis/db/queries.py @@ -78,3 +78,54 @@ def cone_search(ra: Union[str, float], dec: Union[str, float], where(vizdb.SDSSidStacked.cone_search(ra, dec, radius, ra_col='ra_sdss_id', dec_col='dec_sdss_id')) + + +def get_targets_by_sdss_id(sdss_id: int) -> peewee.ModelSelect: + """ Perform a search for SDSS targets on vizdb.SDSSidStacked based on the sdss_id. + + Perform a search for SDSS targets using the peewee ORM in the + vizdb.SDSSidStacked table. We return the peewee ModelSelect + directly here so it can be easily combined with other queries, + if needed. + + In the route endpoint itself, remember to return wrap this in a list. + + Parameters + ---------- + sdss_id : int + the sdss_id + + Returns + ------- + peewee.ModelSelect + the ORM query + """ + + return vizdb.SDSSidStacked.select().where(vizdb.SDSSidStacked.sdss_id == sdss_id) + + +def get_targets_by_catalog_id(catalog_id: int) -> peewee.ModelSelect: + """ Perform a search for SDSS targets on vizdb.SDSSidStacked based on the catalog_id. + + Perform a search for SDSS targets using the peewee ORM in the + vizdb.SDSSidStacked table. We return the peewee ModelSelect + directly here so it can be easily combined with other queries, + if needed. + + In the route endpoint itself, remember to return wrap this in a list. + + Parameters + ---------- + catalog_id : int + the catalog_id + + Returns + ------- + peewee.ModelSelect + the ORM query + """ + + return vizdb.SDSSidStacked.select()\ + .join(vizdb.SDSSidFlat, on=(vizdb.SDSSidStacked.sdss_id == + vizdb.SDSSidFlat.sdss_id))\ + .where(vizdb.SDSSidFlat.catalogid == catalog_id) \ No newline at end of file diff --git a/python/valis/routes/query.py b/python/valis/routes/query.py index 8388f37..1f2b894 100644 --- a/python/valis/routes/query.py +++ b/python/valis/routes/query.py @@ -10,7 +10,7 @@ from valis.routes.base import Base from valis.db.db import get_pw_db from valis.db.models import SDSSidStackedBase -from valis.db.queries import cone_search +from valis.db.queries import cone_search, get_targets_by_sdss_id, get_targets_by_catalog_id class SearchCoordUnits(str, Enum): @@ -55,3 +55,16 @@ async def cone_search(self, """ Perform a cone search """ return list(cone_search(ra, dec, radius, units=units)) + + @router.get('/sdssid', summary='Perform a search of SDSS targets based on the sdss_id', + response_model=List[SDSSidStackedBase], dependencies=[Depends(get_pw_db)]) + async def sdss_id_search(self, sdss_id: Union[int, str] = Query(..., description='Value of sdss_id', example=47510284)): + """ Perform an sdss_id search """ + return list(get_targets_by_sdss_id(int(sdss_id))) + + + @router.get('/catalogid', summary='Perform a search of SDSS targets based on the catalog_id', + response_model=List[SDSSidStackedBase], dependencies=[Depends(get_pw_db)]) + async def catalog_id_search(self, catalog_id: Union[int, str] = Query(..., description='Value of catalog_id', example=7613823349)): + """ Perform a catalog_id search """ + return list(get_targets_by_catalog_id(int(catalog_id))) \ No newline at end of file From 84ee0409daf1a24b3afe011a26362a4b8ae3397c Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Thu, 7 Dec 2023 08:33:12 -0500 Subject: [PATCH 2/5] Return only target object in sdss_id search. --- python/valis/routes/query.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/python/valis/routes/query.py b/python/valis/routes/query.py index 51c8a91..06d9d8b 100644 --- a/python/valis/routes/query.py +++ b/python/valis/routes/query.py @@ -86,14 +86,15 @@ async def cone_search(self, return list(cone_search(ra, dec, radius, units=units)) - @router.get('/sdssid', summary='Perform a search of SDSS targets based on the sdss_id', - response_model=List[SDSSidStackedBase], dependencies=[Depends(get_pw_db)]) + @router.get('/sdssid', summary='Perform a search for an SDSS target based on the sdss_id', + response_model=Union[SDSSidStackedBase, dict], dependencies=[Depends(get_pw_db)]) async def sdss_id_search(self, sdss_id: Union[int, str] = Query(..., description='Value of sdss_id', example=47510284)): - """ Perform an sdss_id search """ - return list(get_targets_by_sdss_id(int(sdss_id))) + """ Perform an sdss_id search. Assumes a maximum of one target per sdss_id. Empty object returned when no match is found.""" + targets = get_targets_by_sdss_id(int(sdss_id)) + return targets[0] if len(targets) > 0 else {} - @router.get('/catalogid', summary='Perform a search of SDSS targets based on the catalog_id', + @router.get('/catalogid', summary='Perform a search for SDSS targets based on the catalog_id', response_model=List[SDSSidStackedBase], dependencies=[Depends(get_pw_db)]) async def catalog_id_search(self, catalog_id: Union[int, str] = Query(..., description='Value of catalog_id', example=7613823349)): """ Perform a catalog_id search """ From f091bf189899f05ba1d3153730f101e0af3b4ee7 Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Mon, 11 Dec 2023 12:39:42 -0500 Subject: [PATCH 3/5] Add sdss_id list endpoint --- python/valis/db/queries.py | 30 ++++++++++++++++++++++++++++++ python/valis/routes/query.py | 13 ++++++++++++- 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/python/valis/db/queries.py b/python/valis/db/queries.py index b245067..41c72a6 100644 --- a/python/valis/db/queries.py +++ b/python/valis/db/queries.py @@ -119,6 +119,36 @@ def cone_search(ra: Union[str, float], dec: Union[str, float], dec_col='dec_sdss_id')) +def get_paged_sdss_id_list(search_string: str = "", page_number: int = 1, items_per_page: int = 10) -> peewee.ModelSelect: + """ Return a paged list of sdss_id values. + + Return paginated and ordered sdss_id column values from the + vizdb.SDSSidStacked table using the peewee ORM. + We return the peewee ModelSelect + directly here so it can be easily combined with other queries, + if needed. + + Parameters + ---------- + search_string : str + String that matches the starting digits of the returned sdss_id values. If empty, no matching is applied. + page_number : int + Page number of the returned sdss_id values. + items_per_page : int + Number of sdss_id values displayed in the page. + + Returns + ------- + peewee.ModelSelect + the ORM query + """ + where_condition = vizdb.SDSSidStacked.sdss_id.cast('TEXT') ** (search_string + "%") if search_string else True + return vizdb.SDSSidStacked.select(vizdb.SDSSidStacked.sdss_id)\ + .where(where_condition)\ + .order_by(vizdb.SDSSidStacked.sdss_id)\ + .paginate(page_number, items_per_page) + + def get_targets_by_sdss_id(sdss_id: int) -> peewee.ModelSelect: """ Perform a search for SDSS targets on vizdb.SDSSidStacked based on the sdss_id. diff --git a/python/valis/routes/query.py b/python/valis/routes/query.py index ba04943..b1d9a96 100644 --- a/python/valis/routes/query.py +++ b/python/valis/routes/query.py @@ -13,7 +13,8 @@ from valis.db.models import SDSSidStackedBase, SDSSidPipesBase from valis.db.queries import (cone_search, append_pipes, carton_program_search, carton_program_list, carton_program_map, - get_targets_by_sdss_id, get_targets_by_catalog_id) + get_targets_by_sdss_id, get_targets_by_catalog_id, + get_paged_sdss_id_list) class SearchCoordUnits(str, Enum): @@ -100,6 +101,16 @@ async def catalog_id_search(self, catalog_id: Union[int, str] = Query(..., descr """ Perform a catalog_id search """ return list(get_targets_by_catalog_id(int(catalog_id))) + @router.get('/list/sdssid', summary='Return a paged list of sdss_id values', + response_model=List[int], dependencies=[Depends(get_pw_db)]) + async def sdss_id_list(self, search_string: str = Query(default="", description='String that matches the starting digits of the returned sdss_id values', example=""), + page_number: int = Query(..., description='Page number of the returned items', gt=0, example=1), + items_per_page: int = Query(..., description='Number of items displayed in a page', gt=0, example=10)): + """ Return an ordered and paged list of sdss_id values.""" + sdss_ids = get_paged_sdss_id_list(search_string, page_number, items_per_page) + return list(sdss_ids.scalars()) + + @router.get('/list/cartons', summary='Return a list of all cartons', response_model=list, dependencies=[Depends(get_pw_db)]) async def cartons(self, From 587441920b729cc5bf9dcf4f90c957867eb9be7f Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Fri, 16 Feb 2024 13:21:15 -0500 Subject: [PATCH 4/5] Use sdss_id ranges in query --- python/valis/db/queries.py | 18 ++++++++++++++++-- python/valis/routes/query.py | 4 ++-- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/python/valis/db/queries.py b/python/valis/db/queries.py index 41c72a6..e08ddd9 100644 --- a/python/valis/db/queries.py +++ b/python/valis/db/queries.py @@ -6,6 +6,7 @@ from typing import Union import peewee +from peewee import fn import astropy.units as u from astropy.coordinates import SkyCoord from sdssdb.peewee.sdss5db import vizdb @@ -119,7 +120,7 @@ def cone_search(ra: Union[str, float], dec: Union[str, float], dec_col='dec_sdss_id')) -def get_paged_sdss_id_list(search_string: str = "", page_number: int = 1, items_per_page: int = 10) -> peewee.ModelSelect: +def get_paged_sdss_id_list(search_integer: int, page_number: int = 1, items_per_page: int = 10) -> peewee.ModelSelect: """ Return a paged list of sdss_id values. Return paginated and ordered sdss_id column values from the @@ -142,7 +143,20 @@ def get_paged_sdss_id_list(search_string: str = "", page_number: int = 1, items_ peewee.ModelSelect the ORM query """ - where_condition = vizdb.SDSSidStacked.sdss_id.cast('TEXT') ** (search_string + "%") if search_string else True + + max_sdss_id = vizdb.SDSSidStacked.select(fn.MAX(vizdb.SDSSidStacked.sdss_id)).scalar() + + max_num_digits = len(str(max_sdss_id)) + num_search_digits = len(str(search_integer)) + max_i = max(0, max_num_digits - num_search_digits + 1) + + where_condition = (False) + for i in range(0, max_i): + min_id = int(search_integer * 10**(i)) + max_id = int((search_integer + 1) * 10**(i)) + where = ((vizdb.SDSSidStacked.sdss_id >= min_id) & (vizdb.SDSSidStacked.sdss_id < max_id)) + where_condition = where_condition | where + return vizdb.SDSSidStacked.select(vizdb.SDSSidStacked.sdss_id)\ .where(where_condition)\ .order_by(vizdb.SDSSidStacked.sdss_id)\ diff --git a/python/valis/routes/query.py b/python/valis/routes/query.py index b1d9a96..084fb1c 100644 --- a/python/valis/routes/query.py +++ b/python/valis/routes/query.py @@ -103,11 +103,11 @@ async def catalog_id_search(self, catalog_id: Union[int, str] = Query(..., descr @router.get('/list/sdssid', summary='Return a paged list of sdss_id values', response_model=List[int], dependencies=[Depends(get_pw_db)]) - async def sdss_id_list(self, search_string: str = Query(default="", description='String that matches the starting digits of the returned sdss_id values', example=""), + async def sdss_id_list(self, search_integer: int = Query(default="", description='Integer that matches the starting digits of the returned sdss_id values', ge=1, example=1), page_number: int = Query(..., description='Page number of the returned items', gt=0, example=1), items_per_page: int = Query(..., description='Number of items displayed in a page', gt=0, example=10)): """ Return an ordered and paged list of sdss_id values.""" - sdss_ids = get_paged_sdss_id_list(search_string, page_number, items_per_page) + sdss_ids = get_paged_sdss_id_list(search_integer, page_number, items_per_page) return list(sdss_ids.scalars()) From aae0b2dc499d6a190275a719bf5c13d8ad6bce29 Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Mon, 19 Feb 2024 06:39:07 -0500 Subject: [PATCH 5/5] simplify loop condition --- python/valis/db/queries.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/valis/db/queries.py b/python/valis/db/queries.py index e08ddd9..227afb4 100644 --- a/python/valis/db/queries.py +++ b/python/valis/db/queries.py @@ -148,7 +148,7 @@ def get_paged_sdss_id_list(search_integer: int, page_number: int = 1, items_per_ max_num_digits = len(str(max_sdss_id)) num_search_digits = len(str(search_integer)) - max_i = max(0, max_num_digits - num_search_digits + 1) + max_i = max_num_digits - num_search_digits + 1 where_condition = (False) for i in range(0, max_i):