diff --git a/Pipfile b/Pipfile index 0b570e1..a46562b 100644 --- a/Pipfile +++ b/Pipfile @@ -29,7 +29,6 @@ black = "==19.10b0" psycopg2 = "*" [dev-packages] - coverage = "*" [requires] @@ -40,3 +39,4 @@ docker= "docker-compose up -d" lint = "./scripts/lint" start = "uvicorn src.main:app --reload" test = "pytest" +db = "./scripts/create_table.py" diff --git a/src/models.py b/src/models.py index 09469f8..c643079 100644 --- a/src/models.py +++ b/src/models.py @@ -47,20 +47,35 @@ class RedisEvent(CustomModel): key: bytes | str value: bytes | str + def __init__(self, **data): + super().__init__(**data) + self.key = data["key"] + self.value = data["value"] + + +class AnalyticsValue(CustomModel): + key: str + value: str + timestamp: datetime + + def __init__(self, **data): + super().__init__(**data) + self.key = data["key"] + self.value = data["value"] + self.timestamp = datetime.fromtimestamp(data["timestamp"]) + class AnalyticsTxn(CustomModel): product: Product pennkey: Optional[str] = None - timestamp: datetime - data: list[RedisEvent] + data: list[AnalyticsValue] # init with JSON data def __init__(self, **data): super().__init__(**data) - self.timestamp = datetime.fromtimestamp(data["timestamp"]) - self.data = [RedisEvent(**event) for event in data["data"]] self.product = Product(data["product"]) self.pennkey = data.get("pennkey") + self.data = [AnalyticsValue(**value) for value in data["data"]] def get_redis_key(self): return f"{self.product}.{self.hash_as_key()}" @@ -68,16 +83,16 @@ def get_redis_key(self): def build_redis_data(self) -> list[RedisEvent]: return [ RedisEvent( - key=f"{self.get_redis_key()}.{event.hash_as_key()}", + key=f"{self.get_redis_key()}.{value.hash_as_key()}", value=json.dumps( { "product": str(self.product), "pennkey": self.pennkey, - "timestamp": self.timestamp.timestamp(), - "datapoint": event.key, - "value": event.value, + "timestamp": value.timestamp.timestamp(), + "datapoint": value.key, + "value": value.value, } ), ) - for event in self.data + for value in self.data ] diff --git a/tests/test_load.py b/tests/test_load.py index 13eb980..8c4736d 100644 --- a/tests/test_load.py +++ b/tests/test_load.py @@ -5,7 +5,8 @@ from datetime import datetime import requests -from test_token import get_tokens + +from tests.test_token import get_tokens # Runtime should be less that 3 seconds for most laptops @@ -16,32 +17,79 @@ THREADS = 16 -def make_request(): - access_token, _ = get_tokens() - - url = "http://localhost:8000/analytics" +def make_request(token: str): + url = "http://localhost:80/analytics/" payload = json.dumps( { "product": random.randint(1, 10), "pennkey": "test_usr", - "timestamp": int(datetime.now().timestamp()), "data": [ - {"key": "user.click", "value": str(random.randint(1, 1000))}, - {"key": "user.drag", "value": str(random.randint(1, 1000))}, - {"key": "data.dowload", "value": str(random.randint(1, 1000))}, - {"key": "user.drive", "value": str(random.randint(1, 1000))}, - {"key": "user.play", "value": str(random.randint(1, 1000))}, - {"key": "user.sit", "value": str(random.randint(1, 1000))}, - {"key": "user.stand", "value": str(random.randint(1, 1000))}, - {"key": "user.bike", "value": str(random.randint(1, 1000))}, - {"key": "user.flip", "value": str(random.randint(1, 1000))}, - {"key": "food.eat", "value": str(random.randint(1, 1000))}, + { + "key": "user.click", + "value": str(random.randint(1, 1000)), + "timestamp": int(datetime.now().timestamp()) + + random.randint(1, 1000), + }, + { + "key": "user.drag", + "value": str(random.randint(1, 1000)), + "timestamp": int(datetime.now().timestamp()) + + random.randint(1, 1000), + }, + { + "key": "data.dowload", + "value": str(random.randint(1, 1000)), + "timestamp": int(datetime.now().timestamp()) + + random.randint(1, 1000), + }, + { + "key": "user.drive", + "value": str(random.randint(1, 1000)), + "timestamp": int(datetime.now().timestamp()) + + random.randint(1, 1000), + }, + { + "key": "user.play", + "value": str(random.randint(1, 1000)), + "timestamp": int(datetime.now().timestamp()) + + random.randint(1, 1000), + }, + { + "key": "user.sit", + "value": str(random.randint(1, 1000)), + "timestamp": int(datetime.now().timestamp()) + + random.randint(1, 1000), + }, + { + "key": "user.stand", + "value": str(random.randint(1, 1000)), + "timestamp": int(datetime.now().timestamp()) + + random.randint(1, 1000), + }, + { + "key": "user.bike", + "value": str(random.randint(1, 1000)), + "timestamp": int(datetime.now().timestamp()) + + random.randint(1, 1000), + }, + { + "key": "user.flip", + "value": str(random.randint(1, 1000)), + "timestamp": int(datetime.now().timestamp()) + + random.randint(1, 1000), + }, + { + "key": "food.eat", + "value": str(random.randint(1, 1000)), + "timestamp": int(datetime.now().timestamp()) + + random.randint(1, 1000), + }, ], } ) headers = { "Content-Type": "application/json", - "Authorization": f"Bearer {access_token}", + "Authorization": f"Bearer {token}", } try: @@ -55,8 +103,10 @@ def make_request(): def run_threads(): with ThreadPoolExecutor(max_workers=THREADS) as executor: + # Fetch access token once to avoid repeated requests. + access_token, _ = get_tokens() for _ in range(NUMBER_OF_REQUESTS): - executor.submit(make_request) + executor.submit(make_request, access_token) def test_load(): diff --git a/tests/test_redis.py b/tests/test_redis.py index b3e0675..9633dfd 100644 --- a/tests/test_redis.py +++ b/tests/test_redis.py @@ -1,3 +1,6 @@ +import random +from datetime import datetime + import pytest from src.models import RedisEvent @@ -7,8 +10,16 @@ @pytest.mark.asyncio async def test_redis(): data = [ - {"key": "test_key", "value": "test_value"}, - {"key": "test_key2", "value": "test_value2"}, + { + "key": "test_key", + "value": "test_value", + "timestamp": int(datetime.now().timestamp()) + random.randint(1, 1000), + }, + { + "key": "test_key2", + "value": "test_value2", + "timestamp": int(datetime.now().timestamp()) + random.randint(1, 1000), + }, ] payload = [RedisEvent(**d) for d in data] diff --git a/tests/test_token.py b/tests/test_token.py index 02bc919..8489344 100644 --- a/tests/test_token.py +++ b/tests/test_token.py @@ -1,31 +1,38 @@ # Test to generate jwt token from Penn Labs platforms import os -import requests - from src.auth import verify_jwt -ATTEST_URL = "https://platform.pennlabs.org/identity/attest/" - +# ATTEST_URL = "https://platform.pennlabs.org/identity/attest/" # Using Penn Basics DLA Account for testing, will not work if you don't have that in .env -CLIENT_ID: str = os.environ.get("CLIENT_ID") or "" -CLIENT_SECRET: str = os.environ.get("CLIENT_SECRET") or "" +# CLIENT_ID: str = os.environ.get("CLIENT_ID") or "" +# CLIENT_SECRET: str = os.environ.get("CLIENT_SECRET") or "" + + +# With the new URL to Verify JWT's (See https://github.com/pennlabs/labs-analytics/pull/38) +# The testing JWT generated with client_id and client_secret is no longer +# correctly decoded. Hence, provide a valid JWT issued by Platform for a given user. +# This will allow the JWT to be correctly decoded. +TESTING_JWT: str = os.environ.get("TESTING_JWT") or "" def test_env_vars(): assert os.environ.get("CLIENT_ID") is not None assert os.environ.get("CLIENT_SECRET") is not None + assert os.environ.get("TESTING_JWT") is not None def get_tokens(): - response = requests.post(ATTEST_URL, auth=(CLIENT_ID, CLIENT_SECRET)) - if response.status_code == 200: - content = response.json() - token = content["access"] - refresh = content["refresh"] - return (token, refresh) - return ("", "") + return (TESTING_JWT, "") + # response = requests.post(ATTEST_URL, auth=(CLIENT_ID, CLIENT_SECRET)) + # if response.status_code == 200: + # content = response.json() + # token = content["access"] + # refresh = content["refresh"] + # return (TESTING_JWT, refresh) + + # return ("", "") def test_get_tokens():