-
Notifications
You must be signed in to change notification settings - Fork 0
Updated test file and models.py #2
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
cde0664
1f1a098
9a70955
8c171d1
1c53cb3
ad7a4ab
b62b537
64279db
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -6,6 +6,11 @@ | |
|
|
||
| from app.database import SessionDep | ||
| from app.versions.v1.models import DataRequest, DataRequestPublic, DataRequestUpdate | ||
| import gbbox | ||
| from shapely.geometry import shape as shp | ||
| import json | ||
| from datetime import datetime, timezone | ||
| import pystac | ||
|
|
||
| app = FastAPI(version="1") | ||
|
|
||
|
|
@@ -20,7 +25,6 @@ async def post_data_request( | |
| session.refresh(data_request) | ||
| return data_request | ||
|
|
||
|
|
||
| @app.patch("/data-publish-request/{request_id}") | ||
| async def patch_data_request( | ||
| request_id: str, data_request: DataRequestUpdate, session: SessionDep | ||
|
|
@@ -35,15 +39,109 @@ async def patch_data_request( | |
| session.refresh(data_request) | ||
| return data_request | ||
|
|
||
|
|
||
| @app.get("/data-publish-request/{request_id}") | ||
| async def get_data_request(request_id: str, session: SessionDep) -> DataRequestPublic: | ||
| async def get_data_request(request_id: str, session: SessionDep, stac: bool = False) -> DataRequestPublic: | ||
| """Get a data request with the given request_id.""" | ||
| request = session.get(DataRequest, request_id) | ||
| if not request: | ||
| data_request = session.get(DataRequest, request_id) | ||
| if stac: | ||
| #STAC item time (creation time) | ||
| datetime_utc = datetime.now(tz=timezone.utc) | ||
| #Create list of custom item properties (other than geometry and time), which will be added to the STAC item | ||
| item_properties = { | ||
| "title": data_request.title, | ||
| "description": data_request.desc, | ||
| "authors_firstname": data_request.authorFNames, | ||
| "authors_lastname": data_request.authorLNames, | ||
| # date | ||
| "start_datetime": data_request.start_date, | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. STAC items can have a single date property called Also, datetime can possibly be null in which case we'd want to handle that option as well. |
||
| "end_datetime": data_request.end_date, | ||
| # variables | ||
| "variables": data_request.variables, | ||
| # models | ||
| "models": data_request.models, | ||
| "item_links": [{ | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. links should be outside the
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Also, please include |
||
| "rel": "self", | ||
| "href": data_request.path | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This is a bit tricky because this will be the path to the file in the users directory but the actual stac object should be the location of the file in the THREDDS catalog (or however the file is being served publicly). I don't really know what the proper solution is to automatically generate that yet but for now can you add a comment that we need to figure this out. |
||
| }, | ||
| { | ||
| "rel": "derived_from", | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. We should use established rel values when possible. Are |
||
| "href": data_request.input | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. If there are multiple of these we should create multiple links |
||
| }, | ||
| { | ||
| "rel": "linked_files", | ||
| "href": data_request.link | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Same here. We should create multiple links if there are multiples |
||
| }] | ||
| } | ||
| #generate bbox from geometry, in the case of a simple geometry where user did not provide GeoJSON geometry input | ||
| myfile = data_request.myFile | ||
| if data_request.myFile == "": | ||
| latitude = json.loads(data_request.latitude) | ||
| longitude = json.loads(data_request.longitude) | ||
| if data_request.geometry == "Point": | ||
| myfile = { | ||
| "type": "Point", | ||
| "coordinates": [data_request.latitude, data_request.longitude] | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Should this be |
||
| } | ||
| if data_request.geometry == "LineString": | ||
| myfile = { | ||
| "type": "LineString", | ||
| "coordinates": [[latitude[0], longitude[0]]] | ||
| } | ||
| for i in range(1, len(latitude)): | ||
| myfile['coordinates'].append([latitude[i], longitude[i]]) | ||
| if data_request.geometry == "MultiPoint": | ||
| myfile = { | ||
| "type": "MultiPoint", | ||
| "coordinates": [[data_request.latitude[0], data_request.longitude[0]]] | ||
| } | ||
| for i in range(1, len(data_request.latitude)): | ||
| myfile['coordinates'].append([data_request.latitude[i], data_request.longitude[i]]) | ||
| if data_request.geometry == "Polygon": | ||
| myfile = { | ||
| "type": "Polygon", | ||
| "coordinates": [[data_request.latitude[0], data_request.longitude[0]]] | ||
| } | ||
| for i in range(1, len(data_request.latitude)): | ||
| myfile['coordinates'].append([data_request.latitude[i], data_request.longitude[i]]) | ||
| #Now that each file has a GeoJSON geometry feature (either user inputted or generated above), use gbbox package to generate gbbox | ||
| if data_request.geometry == "Point": | ||
| shape = gbbox.Point(**myfile) | ||
| geobbox = shape.bbox() | ||
| if data_request.geometry == "Line": | ||
| geom = shp(myfile) | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Just curious... does gbbox not support Lines? |
||
| geobbox = geom.bounds | ||
| if data_request.geometry == "LineString": | ||
| shape = gbbox.LineString(**myfile) | ||
| geobbox = shape.bbox() | ||
| if data_request.geometry == "MultiLineString": | ||
| shape = gbbox.MultiLineString(**myfile) | ||
| geobbox = shape.bbox() | ||
| if data_request.geometry == "Polygon": | ||
| shape = gbbox.Polygon(**myfile) | ||
| geobbox = shape.bbox() | ||
| if data_request.geometry == "MultiPolygon": | ||
| shape = gbbox.MultiPolygon(**myfile) | ||
| geobbox = shape.bbox() | ||
| if data_request.geometry == "GeometryCollection": | ||
| shape = gbbox.GeometryCollection(**myfile) | ||
| geobbox = shape.bbox() | ||
| #Create STAC item (null case and regular case) | ||
| if data_request.geometry == "null": | ||
| item = pystac.Item(id=id, | ||
| geometry = null, | ||
| datetime=datetime_utc, | ||
| properties=item_properties) | ||
| elif data_request.geometry != "null": | ||
| item = pystac.Item(id=id, | ||
| geometry = myfile, | ||
| bbox = geobbox, | ||
| datetime=datetime_utc, | ||
| properties=item_properties) | ||
| return item.to_dict() #CHECK WORKING | ||
| if not data_request: | ||
| raise HTTPException(status_code=404, detail="data publish request not found") | ||
| return request | ||
|
|
||
| #return request if not stac | ||
| return data_request | ||
|
|
||
| class _LinksResponse(BaseModel): | ||
| rel: str | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,72 +1,76 @@ | ||
| from datetime import datetime | ||
|
|
||
| from pydantic import field_validator | ||
| from pydantic import field_validator, model_validator, validator | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
|
||
| from sqlmodel import Field, SQLModel | ||
|
|
||
| from sqlalchemy import JSON | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. unused. please remove the import |
||
| import ast | ||
| from fastapi import FastAPI | ||
| from typing import List, Optional | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. unused ... |
||
|
|
||
| class DataRequestBase(SQLModel): | ||
| """ | ||
| SQL model base object for Data Requests. | ||
|
|
||
| This object contains field validators. | ||
| """ | ||
| start_date: str | ||
| end_date: str | ||
|
|
||
| @model_validator(mode="after") | ||
| def validate_timeperiod(cls, values): | ||
| if values.__dict__.get("end_date") < values.__dict__.get("start_date"): | ||
| raise ValueError("End date cannot be earlier than start date.") | ||
|
|
||
| @field_validator("start_date","end_date", check_fields=False) | ||
| def validate_timeperiod(cls, start_date: datetime, end_date: datetime) -> datetime: | ||
| """Check that the time periods are correct""" | ||
| if end_date < start_date: | ||
| raise ValueError("End date can not be earlier than start date") | ||
| return start_date, end_date | ||
| return values # Does nothing, just returns the values as they are | ||
|
|
||
| @field_validator("longitude", "latitude", "myFile", check_fields=False) | ||
| def validate_title(cls, latitude: str, longitude: str, myFile:str) -> str: | ||
| """Ensure list lengths match""" | ||
| if len(latitude) != len(longitude): | ||
| raise ValueError("Latitude and longitude lists are different lengths") | ||
| """If there is no latitude and longitude, make sure there is a GeoJSON""" | ||
| if latitude == None: | ||
| if myFile == None: | ||
| raise ValueError("Must include either GeoJSON file or manually inputted latitude and longitudes") | ||
| """Check latitude and longitude ranges""" | ||
| for i in latitude: | ||
| if i > 90: | ||
| """ @model_validator(mode="after") | ||
| def validate_location(cls, values): | ||
| #Ensure list lengths match and check latitude/longitude validity. | ||
| #json list | ||
| lat = json.loads(values.__dict__.get("latitude")) | ||
| lon = json.loads(values.__dict__.get("longitude")) | ||
| file = values.__dict__.get("myFile") | ||
| #Ensure list lengths match. | ||
| #if len(latitude) != len(longitude): | ||
| # raise ValueError("Latitude and longitude lists are different lengths") | ||
| #If there is no latitude and longitude, make sure there is a GeoJSON | ||
| if lat is None and file is None: | ||
| raise ValueError("Must include either GeoJSON file or manually inputted latitude and longitudes") | ||
| #Check latitude and longitude ranges | ||
| for i in lat: | ||
| if i > 90 or i < -90: | ||
| raise ValueError("Latitudes must be between -90 and 90 degrees") | ||
| if i < -90: | ||
| for i in lon: | ||
| if float(i) > 180 or float(i) < -180: | ||
| raise ValueError("Latitudes must be between -90 and 90 degrees") | ||
| for i in longitude: | ||
| if i > 180: | ||
| raise ValueError("Longitudes must be between -180 and 180 degrees") | ||
| if i < -180: | ||
| raise ValueError("Longitudes must be between -180 and 180 degrees") | ||
|
|
||
| return latitude, longitude, myFile | ||
|
|
||
| return values """ | ||
|
|
||
| class DataRequest(DataRequestBase, table=True): | ||
| """ | ||
| Database model for Data Requests. | ||
|
|
||
| This object contains the representation of the data in the database. | ||
| """ | ||
|
|
||
| id: int | None = Field(default=None, primary_key=True) | ||
| username: str | ||
| title: str | ||
| desc: str | None | ||
| fname: str | ||
| lname: str | ||
| email: str | ||
| authorFNames: str | ||
| authorLNames: str | ||
| authorEmails: str | ||
| geometry: str | ||
| latitude: str | None | ||
| longitude: str | None | ||
| myFile: str | None | ||
| start_date: datetime | ||
| end_date: datetime | ||
| start_date: str | None | ||
| end_date: str | None | ||
| variables: str | None | ||
| models: str | None | ||
| path: str | ||
| input: str | None | ||
| link: str | None | ||
|
|
||
|
|
||
| class DataRequestPublic(DataRequestBase): | ||
| """ | ||
|
|
@@ -77,18 +81,19 @@ class DataRequestPublic(DataRequestBase): | |
| be included in this object. | ||
| """ | ||
|
|
||
| id: int | None = Field(default=None, primary_key=True) | ||
| id: int | ||
| username: str | ||
| title: str | ||
| desc: str | None | ||
| fname: str | ||
| lname: str | ||
| email: str | ||
| authorFNames: str | ||
| authorLNames: str | ||
| authorEmails: str | ||
| geometry: str | ||
| latitude: str | None | ||
| longitude: str | None | ||
| myFile: str | None | ||
| start_date: datetime | ||
| end_date: datetime | ||
| start_date: str | None | ||
| end_date: str | None | ||
| variables: str | None | ||
| models: str | None | ||
| path: str | ||
|
|
@@ -104,7 +109,20 @@ class DataRequestUpdate(DataRequestBase): | |
| Fields should be optional unless they *must* be updated every time a change is made. | ||
| """ | ||
|
|
||
| username: str | None = None | ||
| title: str | None = None | ||
| desc: str | None = None | ||
| date: datetime | None = None | ||
| # TODO: make sure parameters added in DataRequest are made optional here | ||
| authorFNames: str | None = None | ||
| authorLNames: str | None = None | ||
| authorEmails: str | None = None | ||
| geometry: str | None = None | ||
| latitude: str | None = None | ||
| longitude: str | None = None | ||
| myFile: str | None = None | ||
| start_date: str | None = None | ||
| end_date: str | None = None | ||
| variables: str | None = None | ||
| models: str | None = None | ||
| path: str | None = None | ||
| input: str | None = None | ||
| link: str | None = None | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,36 @@ | ||
| # /marble-api/tests/test_data_request.py | ||
| import httpx | ||
| from datetime import datetime | ||
|
|
||
| # Replace this with your actual server's IP and port | ||
| BASE_URL = "http://127.0.0.1:8000/v1" | ||
|
|
||
| def test_valid_data_request(): | ||
| payload = { | ||
| "start_date": "1990-01-01T00:00:00", | ||
| "end_date": "2024-01-01T00:00:00", | ||
| "latitude": "[37.7749, 85]", | ||
| "longitude": "[-122.899,-80]", | ||
| "username": "user123", | ||
| "title": "Test Request", | ||
| "authorFNames": "['John', 'Cassie']", | ||
| "authorLNames": "['Doe', 'Chanen']", | ||
| "email": "['john.doe@example.com', 'cassie@gmail.com']", | ||
| "geometry": "Point", | ||
| "myFile": None, | ||
| "variables": None, | ||
| "models": None, | ||
| "path": "/some/path", | ||
| "input": None, | ||
| "link": None | ||
| } | ||
|
|
||
| response = httpx.post(f"{BASE_URL}/data-publish-request", json=payload) | ||
| print(response.status_code) | ||
| print(response.text) # This will print out the error message | ||
| assert response.status_code == 200 | ||
| json_data = response.json() | ||
|
|
||
| # Instead of checking for "message", check for the presence of the "id" field | ||
| assert "id" in json_data # Ensure that the returned data has an "id" field | ||
| assert json_data["username"] == "user123" |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
We should try to use existing stac extensions as much as possible when we can. So for example, for the author information we can use the contacts extension.