Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
112 changes: 105 additions & 7 deletions app/versions/v1/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,11 @@

from app.database import SessionDep
from app.versions.v1.models import DataRequest, DataRequestPublic, DataRequestUpdate
import gbbox
from shapely.geometry import shape as shp
import json
from datetime import datetime, timezone
import pystac

app = FastAPI(version="1")

Expand All @@ -20,7 +25,6 @@ async def post_data_request(
session.refresh(data_request)
return data_request


@app.patch("/data-publish-request/{request_id}")
async def patch_data_request(
request_id: str, data_request: DataRequestUpdate, session: SessionDep
Expand All @@ -35,15 +39,109 @@ async def patch_data_request(
session.refresh(data_request)
return data_request


@app.get("/data-publish-request/{request_id}")
async def get_data_request(request_id: str, session: SessionDep) -> DataRequestPublic:
async def get_data_request(request_id: str, session: SessionDep, stac: bool = False) -> DataRequestPublic:
"""Get a data request with the given request_id."""
request = session.get(DataRequest, request_id)
if not request:
data_request = session.get(DataRequest, request_id)
if stac:
#STAC item time (creation time)
datetime_utc = datetime.now(tz=timezone.utc)
#Create list of custom item properties (other than geometry and time), which will be added to the STAC item
item_properties = {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should try to use existing stac extensions as much as possible when we can. So for example, for the author information we can use the contacts extension.

"title": data_request.title,
"description": data_request.desc,
"authors_firstname": data_request.authorFNames,
"authors_lastname": data_request.authorLNames,
# date
"start_datetime": data_request.start_date,
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

STAC items can have a single date property called datetime if it just represents a single point in time. We should check whether start_datetime == end_datetime and if so set the datetime property only.

Also, datetime can possibly be null in which case we'd want to handle that option as well.

"end_datetime": data_request.end_date,
# variables
"variables": data_request.variables,
# models
"models": data_request.models,
"item_links": [{
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

links should be outside the properties key. (links and properties are siblings, not nested).

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Also, please include type in all links when possible.

"rel": "self",
"href": data_request.path
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is a bit tricky because this will be the path to the file in the users directory but the actual stac object should be the location of the file in the THREDDS catalog (or however the file is being served publicly). I don't really know what the proper solution is to automatically generate that yet but for now can you add a comment that we need to figure this out.

},
{
"rel": "derived_from",
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should use established rel values when possible. Are derived_from and linked_files standard names or are they custom to this?

"href": data_request.input
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If there are multiple of these we should create multiple links

},
{
"rel": "linked_files",
"href": data_request.link
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same here. We should create multiple links if there are multiples

}]
}
#generate bbox from geometry, in the case of a simple geometry where user did not provide GeoJSON geometry input
myfile = data_request.myFile
if data_request.myFile == "":
latitude = json.loads(data_request.latitude)
longitude = json.loads(data_request.longitude)
if data_request.geometry == "Point":
myfile = {
"type": "Point",
"coordinates": [data_request.latitude, data_request.longitude]
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should this be latitude and longitude or do you want the serialized version (not extracted by json.loads)?

}
if data_request.geometry == "LineString":
myfile = {
"type": "LineString",
"coordinates": [[latitude[0], longitude[0]]]
}
for i in range(1, len(latitude)):
myfile['coordinates'].append([latitude[i], longitude[i]])
if data_request.geometry == "MultiPoint":
myfile = {
"type": "MultiPoint",
"coordinates": [[data_request.latitude[0], data_request.longitude[0]]]
}
for i in range(1, len(data_request.latitude)):
myfile['coordinates'].append([data_request.latitude[i], data_request.longitude[i]])
if data_request.geometry == "Polygon":
myfile = {
"type": "Polygon",
"coordinates": [[data_request.latitude[0], data_request.longitude[0]]]
}
for i in range(1, len(data_request.latitude)):
myfile['coordinates'].append([data_request.latitude[i], data_request.longitude[i]])
#Now that each file has a GeoJSON geometry feature (either user inputted or generated above), use gbbox package to generate gbbox
if data_request.geometry == "Point":
shape = gbbox.Point(**myfile)
geobbox = shape.bbox()
if data_request.geometry == "Line":
geom = shp(myfile)
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just curious... does gbbox not support Lines?

geobbox = geom.bounds
if data_request.geometry == "LineString":
shape = gbbox.LineString(**myfile)
geobbox = shape.bbox()
if data_request.geometry == "MultiLineString":
shape = gbbox.MultiLineString(**myfile)
geobbox = shape.bbox()
if data_request.geometry == "Polygon":
shape = gbbox.Polygon(**myfile)
geobbox = shape.bbox()
if data_request.geometry == "MultiPolygon":
shape = gbbox.MultiPolygon(**myfile)
geobbox = shape.bbox()
if data_request.geometry == "GeometryCollection":
shape = gbbox.GeometryCollection(**myfile)
geobbox = shape.bbox()
#Create STAC item (null case and regular case)
if data_request.geometry == "null":
item = pystac.Item(id=id,
geometry = null,
datetime=datetime_utc,
properties=item_properties)
elif data_request.geometry != "null":
item = pystac.Item(id=id,
geometry = myfile,
bbox = geobbox,
datetime=datetime_utc,
properties=item_properties)
return item.to_dict() #CHECK WORKING
if not data_request:
raise HTTPException(status_code=404, detail="data publish request not found")
return request

#return request if not stac
return data_request

class _LinksResponse(BaseModel):
rel: str
Expand Down
104 changes: 61 additions & 43 deletions app/versions/v1/models.py
Original file line number Diff line number Diff line change
@@ -1,72 +1,76 @@
from datetime import datetime

from pydantic import field_validator
from pydantic import field_validator, model_validator, validator
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

field_validator and validator are unused. Please remove the imports

from sqlmodel import Field, SQLModel

from sqlalchemy import JSON
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

unused. please remove the import

import ast
from fastapi import FastAPI
from typing import List, Optional
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

unused ...


class DataRequestBase(SQLModel):
"""
SQL model base object for Data Requests.

This object contains field validators.
"""
start_date: str
end_date: str

@model_validator(mode="after")
def validate_timeperiod(cls, values):
if values.__dict__.get("end_date") < values.__dict__.get("start_date"):
raise ValueError("End date cannot be earlier than start date.")

@field_validator("start_date","end_date", check_fields=False)
def validate_timeperiod(cls, start_date: datetime, end_date: datetime) -> datetime:
"""Check that the time periods are correct"""
if end_date < start_date:
raise ValueError("End date can not be earlier than start date")
return start_date, end_date
return values # Does nothing, just returns the values as they are

@field_validator("longitude", "latitude", "myFile", check_fields=False)
def validate_title(cls, latitude: str, longitude: str, myFile:str) -> str:
"""Ensure list lengths match"""
if len(latitude) != len(longitude):
raise ValueError("Latitude and longitude lists are different lengths")
"""If there is no latitude and longitude, make sure there is a GeoJSON"""
if latitude == None:
if myFile == None:
raise ValueError("Must include either GeoJSON file or manually inputted latitude and longitudes")
"""Check latitude and longitude ranges"""
for i in latitude:
if i > 90:
""" @model_validator(mode="after")
def validate_location(cls, values):
#Ensure list lengths match and check latitude/longitude validity.
#json list
lat = json.loads(values.__dict__.get("latitude"))
lon = json.loads(values.__dict__.get("longitude"))
file = values.__dict__.get("myFile")
#Ensure list lengths match.
#if len(latitude) != len(longitude):
# raise ValueError("Latitude and longitude lists are different lengths")
#If there is no latitude and longitude, make sure there is a GeoJSON
if lat is None and file is None:
raise ValueError("Must include either GeoJSON file or manually inputted latitude and longitudes")
#Check latitude and longitude ranges
for i in lat:
if i > 90 or i < -90:
raise ValueError("Latitudes must be between -90 and 90 degrees")
if i < -90:
for i in lon:
if float(i) > 180 or float(i) < -180:
raise ValueError("Latitudes must be between -90 and 90 degrees")
for i in longitude:
if i > 180:
raise ValueError("Longitudes must be between -180 and 180 degrees")
if i < -180:
raise ValueError("Longitudes must be between -180 and 180 degrees")

return latitude, longitude, myFile

return values """

class DataRequest(DataRequestBase, table=True):
"""
Database model for Data Requests.

This object contains the representation of the data in the database.
"""

id: int | None = Field(default=None, primary_key=True)
username: str
title: str
desc: str | None
fname: str
lname: str
email: str
authorFNames: str
authorLNames: str
authorEmails: str
geometry: str
latitude: str | None
longitude: str | None
myFile: str | None
start_date: datetime
end_date: datetime
start_date: str | None
end_date: str | None
variables: str | None
models: str | None
path: str
input: str | None
link: str | None


class DataRequestPublic(DataRequestBase):
"""
Expand All @@ -77,18 +81,19 @@ class DataRequestPublic(DataRequestBase):
be included in this object.
"""

id: int | None = Field(default=None, primary_key=True)
id: int
username: str
title: str
desc: str | None
fname: str
lname: str
email: str
authorFNames: str
authorLNames: str
authorEmails: str
geometry: str
latitude: str | None
longitude: str | None
myFile: str | None
start_date: datetime
end_date: datetime
start_date: str | None
end_date: str | None
variables: str | None
models: str | None
path: str
Expand All @@ -104,7 +109,20 @@ class DataRequestUpdate(DataRequestBase):
Fields should be optional unless they *must* be updated every time a change is made.
"""

username: str | None = None
title: str | None = None
desc: str | None = None
date: datetime | None = None
# TODO: make sure parameters added in DataRequest are made optional here
authorFNames: str | None = None
authorLNames: str | None = None
authorEmails: str | None = None
geometry: str | None = None
latitude: str | None = None
longitude: str | None = None
myFile: str | None = None
start_date: str | None = None
end_date: str | None = None
variables: str | None = None
models: str | None = None
path: str | None = None
input: str | None = None
link: str | None = None
36 changes: 36 additions & 0 deletions test/conftest.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
# /marble-api/tests/test_data_request.py
import httpx
from datetime import datetime

# Replace this with your actual server's IP and port
BASE_URL = "http://127.0.0.1:8000/v1"

def test_valid_data_request():
payload = {
"start_date": "1990-01-01T00:00:00",
"end_date": "2024-01-01T00:00:00",
"latitude": "[37.7749, 85]",
"longitude": "[-122.899,-80]",
"username": "user123",
"title": "Test Request",
"authorFNames": "['John', 'Cassie']",
"authorLNames": "['Doe', 'Chanen']",
"email": "['john.doe@example.com', 'cassie@gmail.com']",
"geometry": "Point",
"myFile": None,
"variables": None,
"models": None,
"path": "/some/path",
"input": None,
"link": None
}

response = httpx.post(f"{BASE_URL}/data-publish-request", json=payload)
print(response.status_code)
print(response.text) # This will print out the error message
assert response.status_code == 200
json_data = response.json()

# Instead of checking for "message", check for the presence of the "id" field
assert "id" in json_data # Ensure that the returned data has an "id" field
assert json_data["username"] == "user123"
Loading