From 424217db697dd6a23f04b2eeb318ced599b0f643 Mon Sep 17 00:00:00 2001 From: SpicyGarlicAlbacoreRoll Date: Tue, 9 Sep 2025 09:15:57 -0800 Subject: [PATCH 1/7] cfg loaded as part of main module rather than at query time. Change nisar static layer url schema --- src/SearchAPI/application/application.py | 54 ++++++++++++------------ 1 file changed, 26 insertions(+), 28 deletions(-) diff --git a/src/SearchAPI/application/application.py b/src/SearchAPI/application/application.py index 596ca89..03a5423 100644 --- a/src/SearchAPI/application/application.py +++ b/src/SearchAPI/application/application.py @@ -34,6 +34,9 @@ allow_headers=["*"], ) +cfg = load_config_maturity() +cmr_health = get_cmr_health(cfg['cmr_base'], cfg['cmr_health']) + @router.api_route("/services/search/param", methods=["GET", "POST", "HEAD"]) async def query_params(searchOptions: SearchOptsModel = Depends(process_search_request)): @@ -226,32 +229,30 @@ async def file_to_wkt(files: list[UploadFile]): headers=constants.DEFAULT_HEADERS ) -# @router.get('/redirect/{short_name}/{granule_id}') -# async def nisar_static_layer(environment: Literal['prod', 'test'], short_name: str, granule_id: str): -# """ -# environment: 'prod' or 'test' (whether to search the cmr prod or uat record) -# short_name: the CMR static layer collection short name to search -# granule_id: the granule id of the product to find the static layer for - -# returns: redirect to file url -# """ -# opts = asf.ASFSearchOptions() -# if environment == 'test': -# opts.host = asf.INTERNAL.CMR_HOST_UAT - -# try: -# granule = asf.search( -# granule_list=[granule_id], -# opts=opts -# )[0] -# except IndexError: -# raise HTTPException(status_code=400, detail=f'Unable to find static layer, provided scene named "{granule_id}" not found in CMR record') +# example: https://api.daac.asf.alaska.edu/services/redirect/NISAR_L2_STATIC/{granule_id}.h5 +@router.get('/services/redirect/{short_name}/{granule_id}') +async def nisar_static_layer(short_name: str, granule_id: str): + """ + short_name: the CMR static layer collection short name to search + granule_id: the granule id of the product to find the static layer for + + returns: redirect to file url + """ + opts = asf.ASFSearchOptions(host=cfg['cmr_base']) + + try: + granule = asf.search( + granule_list=[granule_id], + opts=opts + )[0] + except IndexError: + raise HTTPException(status_code=400, detail=f'Unable to find static layer, provided scene named "{granule_id}" not found in CMR record') -# static_layer = granule.get_static_layer(opts=asf.ASFSearchOptions(shortName=short_name)) -# if static_layer is None: -# raise HTTPException(status_code=500, detail=f'Static layer not found for scene named "{granule_id}"') + static_layer = granule.get_static_layer(opts=asf.ASFSearchOptions(shortName=short_name)) + if static_layer is None: + raise HTTPException(status_code=500, detail=f'Static layer not found for scene named "{granule_id}"') -# return RedirectResponse(static_layer.properties['url']) + return RedirectResponse(static_layer.properties['url']) def validate_wkt(wkt: str): @@ -281,14 +282,11 @@ async def health_check(): api_logger.info(exc) api_version = {'version': 'unknown'} - cfg = load_config_maturity() - cmr_health = get_cmr_health(cfg['cmr_base'], cfg['cmr_health']) - api_health = { 'ASFSearchAPI': { 'ok?': True, 'version': api_version['version'], - 'config': load_config_maturity() + 'config': cfg }, 'CMRSearchAPI': cmr_health } From 54a81ce0c1961ba9f5bbccf0e5c6b1c691469ebd Mon Sep 17 00:00:00 2001 From: SpicyGarlicAlbacoreRoll Date: Wed, 24 Sep 2025 15:51:13 -0800 Subject: [PATCH 2/7] chore: update README.md --- README.md | 154 +++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 147 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index c308a28..a6a8873 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,149 @@ # SearchAPI-v3 -- Login to aws console using Kion -- Find the region that has the VPC and note account number, vpc_id, subnet_ids and security_group. Subnet_ids should be a comma separated list -- Get temp cli credentials from Kion and add them to your shell -- Run CDK bootstrap in region with the VPC using cdk-bootstrap-example.sh and filling in the account number, vpc_id, subnet_ids and security_group -- If not already created, make an GitHubActionsOidcProvider using the cdk/oidc/oidc-provider.yml template -- Create OIDC role using cloudformation template cdk/oidc/github-actions-oidc.yml. For 'ActionsRoleName' parameter put 'SearchAPIActionsOIDCRole' -- Create a github environment with params using the same values from the CDK bootstrap. AWS_ACCOUNT_ID, SECURITY_GROUP, SUBNET_IDS, VPC_ID +SearchAPI-v3 is a wrapper around the [asf-search python module](https://github.com/asfadmin/Discovery-asf_search) using a serverless deployment with the FastAPI web framework and AWS lambda. + +### Main Endpoints + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
EndpointDescirptionMethods
`/`server configuration info`GET`
`/health`same as root `/``GET`
`/services/search/param`Search via any valid asf-search parameters`GET` `POST` `HEAD`
`/services/search/baseline`Create a baseline stack based off a given reference and optional dataset`GET` `POST` `HEAD`
+ +## Development + +### Branching + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
InstanceBranchDescription, Instructions, Notes
Featuresfeat-*Always branch off dev, for new features
Issuesbugfix-*Always branch off dev, for bugfixes
developmentdevBeginning integration testing happens here, deploys to test-staging deployment
testingtestOnly accepts merges from development, test deployment
production staging and integration testingprod-stagingOnly acepts merges from testing, deploys to prod-staging deployment
releaseprodonly accepts merges from prod-staging, release branch, prod deployment
+ +### Installation + +To install locally run the following in a terminal (we highly recommend installing using a virtual environment) +```bash +pip install -r requirements.txt +pip install . +``` + +To install test requirements, run +```bash +pip install -r tests/requirements.txt +``` + +### Running Locally + +To run the API locally run the following in a terminal +```bash +uvicorn src.SearchAPI.application:app --reload --port 8080 +``` +The api should now be available via your localhost at http://127.0.0.1:8080 and can be opened and queried with your browser or network tool of choice. + + + + +## Testing + +## Running the Test Suite Locally +After running the API (see `Running Locally` above), in order to run the test suite locally run the following: +```bash +pytest --api "http://127.0.0.1:8080" -n auto "tests/yml_tests/" +``` + +## Writing tests +Tests should be written to relevant subfolder & files in `/tests` + +The test suite uses the `pytest-automation` pytest plugin which allows us to define and re-use input for test cases in the yaml format. Test cases are written to files in `tests/yml_tests/`, and reusable resources for those tests `tests/yml_tests/Resources/`. + +```yaml + +tests: +- Test Nisar Product L1 RSLC: # this is a test case + product: NISAR_L1_PR_RSLC_087_039_D_114_2005_DHDH_A_20251102T222008_20251102T222017_T00407_N_P_J_001.yml # this file should be in `tests/yml_tests/Resources/`. See other yml files in the folder to see how you might structure the yml object + product_level: L1 + +- Test Nisar Product L2 GSLC: # this is another test case + product: NISAR_L2_PR_GSLC_087_039_D_112_2005_DHDH_A_20251102T221859_20251102T221935_T00407_N_F_J_001.yml + product_level: L2 +``` + +We can create the mapping from our yaml test cases in `tests/yml_tests/pytest-config.yml`, which will be used to call the desired python function in `tests/yml_tests/pytest-managers.py` + +In `tests/yml_tests/pytest-config.yml`: +```yaml +- For running ASFProduct tests: + required_keys: ['product', 'product_level'] # the keys the test case requires + method: test_NISARProduct # the python function in pytest-managers.py that will be called + required_in_title: Test Nisar Product # (OPTIONAL) will only run test cases defined with `Test Nisar Product` in the name, so the above two test cases would be run with our tests. +``` + + +In `tests/yml_tests/pytest-managers.py`: +```python +def test_new_endpoint(**args) -> None: # Must match the name in pytest-config.yml like above for `method` + test_new_endpoint(client=client, **args) +``` From 13975b99311ecd29759a4df3a03f3cf8c2366e15 Mon Sep 17 00:00:00 2001 From: SpicyGarlicAlbacoreRoll Date: Wed, 24 Sep 2025 15:52:58 -0800 Subject: [PATCH 3/7] chore: add pull request template --- .github/pull_request_template.md | 43 ++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 .github/pull_request_template.md diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..3ca960e --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,43 @@ +# Merge Requirements: +The following requirements must be met for your pull request to be considered for review & merging. Until these requirements are met please mark the pull request as a draft. + +## Purpose +Why is this pull request necessary? Provide a reference to a related issue in this repository that your pull request addresses (if applicable). + +## Description +A brief description of the changes proposed in the pull request. If there are any changes to packaging requirements please list them. If it's a new endpoint list: +- Supported HTTP methods +- input params +- output +- errors it can raise + +## Snippet +If the pull request provides a new feature, provide an example demonstrating the use-case(s) for this pull request (If applicable). + +For example, if you are adding a new endpoint, show how we might call it and what kind of output we could expect: +``` bash +curl 'http://127.0.0.1:8080/services/utils/useful_new_endpoint?param1=value1¶m2=value2' +``` + +If it modifies an existing endpoint (like a new output type for `/services/search/param`) show and example of what the output would look like. + +## Error/Warning/Regression Free +Your code runs without any unhandled errors, warnings, or regressions + +## Unit Tests + +You have added unit tests to the test suite see the [README Testing section](https://github.com/asfadmin/Discovery-SearchAPI-v3/tree/dev?tab=readme-ov-file#writing-tests) for an overview on adding tests to the test suite. + +## Target Merge Branch +Your pull request targets the `master` branch + + +*** + +### Checklist +- [ ] Purpose +- [ ] Description +- [ ] Snippet +- [ ] Error/Warning/Regression Free +- [ ] Unit Tests +- [ ] Target Merge Branch \ No newline at end of file From d66ca6888eeb42c12f1cf41c6a64311943d4d951 Mon Sep 17 00:00:00 2001 From: SpicyGarlicAlbacoreRoll Date: Wed, 24 Sep 2025 15:54:29 -0800 Subject: [PATCH 4/7] fix: change target merge branch in pr template --- .github/pull_request_template.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 3ca960e..79764a6 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -29,7 +29,7 @@ Your code runs without any unhandled errors, warnings, or regressions You have added unit tests to the test suite see the [README Testing section](https://github.com/asfadmin/Discovery-SearchAPI-v3/tree/dev?tab=readme-ov-file#writing-tests) for an overview on adding tests to the test suite. ## Target Merge Branch -Your pull request targets the `master` branch +Your pull request targets the `dev` branch *** From e0570422e2bce0284580c11a9ff0e19c1f4374a9 Mon Sep 17 00:00:00 2001 From: SpicyGarlicAlbacoreRoll Date: Wed, 24 Sep 2025 15:55:59 -0800 Subject: [PATCH 5/7] fix: capitalized pull request template file --- .github/pull_request_template.md | 43 -------------------------------- 1 file changed, 43 deletions(-) delete mode 100644 .github/pull_request_template.md diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md deleted file mode 100644 index 79764a6..0000000 --- a/.github/pull_request_template.md +++ /dev/null @@ -1,43 +0,0 @@ -# Merge Requirements: -The following requirements must be met for your pull request to be considered for review & merging. Until these requirements are met please mark the pull request as a draft. - -## Purpose -Why is this pull request necessary? Provide a reference to a related issue in this repository that your pull request addresses (if applicable). - -## Description -A brief description of the changes proposed in the pull request. If there are any changes to packaging requirements please list them. If it's a new endpoint list: -- Supported HTTP methods -- input params -- output -- errors it can raise - -## Snippet -If the pull request provides a new feature, provide an example demonstrating the use-case(s) for this pull request (If applicable). - -For example, if you are adding a new endpoint, show how we might call it and what kind of output we could expect: -``` bash -curl 'http://127.0.0.1:8080/services/utils/useful_new_endpoint?param1=value1¶m2=value2' -``` - -If it modifies an existing endpoint (like a new output type for `/services/search/param`) show and example of what the output would look like. - -## Error/Warning/Regression Free -Your code runs without any unhandled errors, warnings, or regressions - -## Unit Tests - -You have added unit tests to the test suite see the [README Testing section](https://github.com/asfadmin/Discovery-SearchAPI-v3/tree/dev?tab=readme-ov-file#writing-tests) for an overview on adding tests to the test suite. - -## Target Merge Branch -Your pull request targets the `dev` branch - - -*** - -### Checklist -- [ ] Purpose -- [ ] Description -- [ ] Snippet -- [ ] Error/Warning/Regression Free -- [ ] Unit Tests -- [ ] Target Merge Branch \ No newline at end of file From da0405c0088a34d958c8812d1de896e34f5f6638 Mon Sep 17 00:00:00 2001 From: SpicyGarlicAlbacoreRoll Date: Wed, 24 Sep 2025 16:02:30 -0800 Subject: [PATCH 6/7] fix: capitalized pull request template file. Add bug_report template --- .github/ISSUE_TEMPLATE/bug_report.md | 31 ++++++++++++++++++++ .github/PULL_REQUEST_TEMPLATE.md | 43 ++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/PULL_REQUEST_TEMPLATE.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..cf44dfa --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,31 @@ +--- +name: Bug report +about: Create a report to help us improve +title: "[Bug]" +labels: '' +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Provide an example url to reproduce the behavior. + +\*Reminder: If authentication is required **do not** leave any sensitive credentials in the snippet. + + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Screenshots** +If applicable, add screenshots to help explain your problem. + +**Desktop (please complete the following information):** + - OS: [e.g. Ubuntu 20.04] + - Python Version [e.g. python3.11] + - Pip Environment ['python3 -m pip freeze'] + +**Additional context** +Add any other context about the problem here. diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..79764a6 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,43 @@ +# Merge Requirements: +The following requirements must be met for your pull request to be considered for review & merging. Until these requirements are met please mark the pull request as a draft. + +## Purpose +Why is this pull request necessary? Provide a reference to a related issue in this repository that your pull request addresses (if applicable). + +## Description +A brief description of the changes proposed in the pull request. If there are any changes to packaging requirements please list them. If it's a new endpoint list: +- Supported HTTP methods +- input params +- output +- errors it can raise + +## Snippet +If the pull request provides a new feature, provide an example demonstrating the use-case(s) for this pull request (If applicable). + +For example, if you are adding a new endpoint, show how we might call it and what kind of output we could expect: +``` bash +curl 'http://127.0.0.1:8080/services/utils/useful_new_endpoint?param1=value1¶m2=value2' +``` + +If it modifies an existing endpoint (like a new output type for `/services/search/param`) show and example of what the output would look like. + +## Error/Warning/Regression Free +Your code runs without any unhandled errors, warnings, or regressions + +## Unit Tests + +You have added unit tests to the test suite see the [README Testing section](https://github.com/asfadmin/Discovery-SearchAPI-v3/tree/dev?tab=readme-ov-file#writing-tests) for an overview on adding tests to the test suite. + +## Target Merge Branch +Your pull request targets the `dev` branch + + +*** + +### Checklist +- [ ] Purpose +- [ ] Description +- [ ] Snippet +- [ ] Error/Warning/Regression Free +- [ ] Unit Tests +- [ ] Target Merge Branch \ No newline at end of file From 61c2fea7e2133fe946c79c3497271a6b65977df3 Mon Sep 17 00:00:00 2001 From: SpicyGarlicAlbacoreRoll Date: Wed, 24 Sep 2025 16:09:00 -0800 Subject: [PATCH 7/7] chore: changelog and commented out static layer wip --- CHANGELOG.md | 4 ++- src/SearchAPI/application/application.py | 42 ++++++++++++------------ 2 files changed, 24 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4df4baa..58b1781 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,11 +28,13 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ------ ## [1.0.8](https://github.com/asfadmin/Discovery-SearchAPI-v3/compare/v1.0.7...v1.0.8) ### Changed -- bump asf-search to v10.0.2 for NISAR product type file sizes, urgent response now searchable with product types, and ARIA-S1 GUNW Stacking support +- bump asf-search to v10.0.4 for NISAR product type file sizes, urgent response now searchable with product types, and ARIA-S1 GUNW Stacking support ### Fixed - boolean values are properly capitalized in `python` output file - API maturity set for each level of deployment stage +- API maturity loaded once per api instance + ------ ## [1.0.7](https://github.com/asfadmin/Discovery-SearchAPI-v3/compare/v1.0.6...v1.0.7) ### Changed diff --git a/src/SearchAPI/application/application.py b/src/SearchAPI/application/application.py index 03a5423..825924d 100644 --- a/src/SearchAPI/application/application.py +++ b/src/SearchAPI/application/application.py @@ -230,29 +230,29 @@ async def file_to_wkt(files: list[UploadFile]): ) # example: https://api.daac.asf.alaska.edu/services/redirect/NISAR_L2_STATIC/{granule_id}.h5 -@router.get('/services/redirect/{short_name}/{granule_id}') -async def nisar_static_layer(short_name: str, granule_id: str): - """ - short_name: the CMR static layer collection short name to search - granule_id: the granule id of the product to find the static layer for - - returns: redirect to file url - """ - opts = asf.ASFSearchOptions(host=cfg['cmr_base']) - - try: - granule = asf.search( - granule_list=[granule_id], - opts=opts - )[0] - except IndexError: - raise HTTPException(status_code=400, detail=f'Unable to find static layer, provided scene named "{granule_id}" not found in CMR record') +# @router.get('/services/redirect/{short_name}/{granule_id}') +# async def nisar_static_layer(short_name: str, granule_id: str): +# """ +# short_name: the CMR static layer collection short name to search +# granule_id: the granule id of the product to find the static layer for + +# returns: redirect to file url +# """ +# opts = asf.ASFSearchOptions(host=cfg['cmr_base']) + +# try: +# granule = asf.search( +# granule_list=[granule_id], +# opts=opts +# )[0] +# except IndexError: +# raise HTTPException(status_code=400, detail=f'Unable to find static layer, provided scene named "{granule_id}" not found in CMR record') - static_layer = granule.get_static_layer(opts=asf.ASFSearchOptions(shortName=short_name)) - if static_layer is None: - raise HTTPException(status_code=500, detail=f'Static layer not found for scene named "{granule_id}"') +# static_layer = granule.get_static_layer(opts=asf.ASFSearchOptions(shortName=short_name)) +# if static_layer is None: +# raise HTTPException(status_code=500, detail=f'Static layer not found for scene named "{granule_id}"') - return RedirectResponse(static_layer.properties['url']) +# return RedirectResponse(static_layer.properties['url']) def validate_wkt(wkt: str):