diff --git a/mu-qa.toml b/mu-qa.toml index e69de29..a63e93d 100644 --- a/mu-qa.toml +++ b/mu-qa.toml @@ -0,0 +1,6 @@ +project-org = 'Level 12' +image-name = 'critic' + +[tool.mu.event-rules.run-due-checks] +action='run_due_checks' +cron = '* * * * *' # Every minute on the minute diff --git a/pyproject.toml b/pyproject.toml index cc24e4e..d7f8d38 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,7 @@ dependencies = [ "moto[all]>=5.1.14", "pydantic>=2.7", "httpx>=0.27", + "polyfactory>=3.2.0", ] @@ -41,6 +42,7 @@ dev = [ ] # Used by nox pytest = [ + "freezegun>=1.5.5", 'pytest', 'pytest-cov', 'respx>=0.21', diff --git a/src/critic/app.py b/src/critic/app.py index a442ecc..a1ee2ad 100644 --- a/src/critic/app.py +++ b/src/critic/app.py @@ -3,6 +3,8 @@ from flask import Flask import mu +from critic.tasks import run_due_checks + log = logging.getLogger() @@ -32,6 +34,12 @@ def error(): class ActionHandler(mu.ActionHandler): wsgi_app = app + @staticmethod + def run_due_checks(event, context): + """Triggered by EventBridge rule, invokes `run_due_checks` task.""" + log.info('Invoking run_due_checks') + run_due_checks.invoke() + # The entry point for AWS lambda has to be a function lambda_handler = ActionHandler.on_event diff --git a/src/critic/libs/ddb.py b/src/critic/libs/ddb.py index 85f48f5..f076184 100644 --- a/src/critic/libs/ddb.py +++ b/src/critic/libs/ddb.py @@ -1,45 +1,80 @@ +from datetime import datetime from decimal import Decimal import os from boto3 import client from boto3.dynamodb.types import TypeDeserializer, TypeSerializer -from pydantic import BaseModel +from pydantic import AwareDatetime, BaseModel, TypeAdapter +from critic.libs.dt import to_utc -client = client('dynamodb') -serializer = TypeSerializer() -deserializer = TypeDeserializer() +# https://www.reddit.com/r/aws/comments/cwams9/dynamodb_i_need_to_sort_whole_table_by_range_how/ +CONSTANT_GSI_PK = 'bogus' -def serialize(data: dict) -> dict: +_ddb_client = None + + +def get_client(): + """ + Get a boto3 DynamoDB client without recreating it if it already exists. + """ + global _ddb_client + if _ddb_client is None: + _ddb_client = client('dynamodb') + return _ddb_client + + +class Serializer: """Serialize standard JSON to DynamoDB format.""" - return {k: serializer.serialize(v) for k, v in data.items()} + _serializer = TypeSerializer() + _aware_dt_adapter = TypeAdapter(AwareDatetime) -def deserialize(data: dict) -> dict: - """Deserialize DynamoDB format to standard JSON.""" - return {k: deserializer.deserialize(v) for k, v in data.items()} + @staticmethod + def dt_to_str(value): + if isinstance(value, datetime): + # Convert datetime to string in the same way Pydantic does to ensure consistency + return Serializer._aware_dt_adapter.dump_python(to_utc(value), mode='json') + return value + @staticmethod + def float_to_decimal(value): + if isinstance(value, float): + return Decimal(str(value)) -def namespace_table(table_name: str) -> str: - return f'{table_name}-{os.environ["CRITIC_NAMESPACE"]}' + if isinstance(value, list): + return [Serializer.float_to_decimal(v) for v in value] + if isinstance(value, dict): + return {k: Serializer.float_to_decimal(v) for k, v in value.items()} -def floats_to_decimals(value): - if isinstance(value, float): - return Decimal(str(value)) + return value - if isinstance(value, list): - return [floats_to_decimals(v) for v in value] + def serialize(self, value): + value = self.dt_to_str(value) + value = self.float_to_decimal(value) + return self._serializer.serialize(value) - if isinstance(value, dict): - return {k: floats_to_decimals(v) for k, v in value.items()} + def __call__(self, data: dict) -> dict: + return {k: self.serialize(v) for k, v in data.items()} - return value + +class Deserializer: + """Deserialize DynamoDB format to standard JSON.""" + + _deserializer = TypeDeserializer() + + def __call__(self, data: dict) -> dict: + return {k: self._deserializer.deserialize(v) for k, v in data.items()} + + +serialize = Serializer() +deserialize = Deserializer() class Table: - name: str + base_name: str model: type[BaseModel] partition_key: str sort_key: str | None = None @@ -48,17 +83,22 @@ class Table: def model_to_ddb(inst: BaseModel) -> dict: """Convert a Pydantic model instance to a DynamoDB-compatible dict.""" plain = inst.model_dump(mode='json', exclude_none=True) - return serialize(floats_to_decimals(plain)) + return serialize(plain) + + @staticmethod + def namespace(table_name: str) -> str: + return f'{table_name}-{os.environ["CRITIC_NAMESPACE"]}' @classmethod - def table_name(cls): - return namespace_table(cls.name) + def name(cls) -> str: + return cls.namespace(cls.base_name) @classmethod def put(cls, data: dict | BaseModel): if isinstance(data, dict): data = cls.model(**data) - client.put_item(TableName=cls.table_name(), Item=cls.model_to_ddb(data)) + client = get_client() + client.put_item(TableName=cls.name(), Item=cls.model_to_ddb(data)) @classmethod def get(cls, partition_value: str | int, sort_value: str | int | None = None): @@ -70,8 +110,8 @@ def get(cls, partition_value: str | int, sort_value: str | int | None = None): key[cls.sort_key] = sort_value # Get item - item = client.get_item( - TableName=cls.table_name(), + item = get_client().get_item( + TableName=cls.name(), Key=serialize(key), )['Item'] return cls.model(**deserialize(item)) diff --git a/src/critic/libs/dt.py b/src/critic/libs/dt.py new file mode 100644 index 0000000..27ed508 --- /dev/null +++ b/src/critic/libs/dt.py @@ -0,0 +1,11 @@ +from datetime import UTC, datetime + + +def is_aware(dt: datetime) -> bool: + return dt.tzinfo is not None and dt.tzinfo.utcoffset(dt) is not None + + +def to_utc(dt: datetime) -> datetime: + if not is_aware(dt): + raise ValueError(f'datetime must be timezone aware, got {dt}') + return dt.astimezone(UTC) diff --git a/src/critic/libs/testing.py b/src/critic/libs/testing.py index 627e785..6779563 100644 --- a/src/critic/libs/testing.py +++ b/src/critic/libs/testing.py @@ -1,11 +1,16 @@ import boto3 +from polyfactory.factories.pydantic_factory import ModelFactory +from pydantic import BaseModel -from critic.libs.ddb import client, namespace_table +from critic.libs.ddb import Table, get_client +from critic.models import UptimeMonitorModel +from critic.tables import UptimeMonitorTable def create_tables(): + client = get_client() client.create_table( - TableName=namespace_table('Project'), + TableName=Table.namespace('Project'), AttributeDefinitions=[ {'AttributeName': 'id', 'AttributeType': 'S'}, ], @@ -16,7 +21,7 @@ def create_tables(): ) client.create_table( - TableName=namespace_table('UptimeMonitor'), + TableName=Table.namespace('UptimeMonitor'), AttributeDefinitions=[ # Key attributes {'AttributeName': 'project_id', 'AttributeType': 'S'}, @@ -43,7 +48,7 @@ def create_tables(): ) client.create_table( - TableName=namespace_table('UptimeLog'), + TableName=Table.namespace('UptimeLog'), AttributeDefinitions=[ {'AttributeName': 'monitor_id', 'AttributeType': 'S'}, {'AttributeName': 'timestamp', 'AttributeType': 'S'}, @@ -93,5 +98,21 @@ def _clear_table(table_name: str): def clear_tables(): - for table_name in [namespace_table(t) for t in ('Project', 'UptimeMonitor', 'UptimeLog')]: + for table_name in [Table.namespace(t) for t in ('Project', 'UptimeMonitor', 'UptimeLog')]: _clear_table(table_name) + + +class PutMixin: + __table__: type[Table] + + @classmethod + def put(cls, **kwargs) -> BaseModel: + item = cls.build(**kwargs) + cls.__table__.put(item) + return item + + +class UptimeMonitorFactory(PutMixin, ModelFactory): + __model__ = UptimeMonitorModel + __table__ = UptimeMonitorTable + __use_defaults__ = True diff --git a/src/critic/models.py b/src/critic/models.py index cd5da65..448b2cc 100644 --- a/src/critic/models.py +++ b/src/critic/models.py @@ -5,7 +5,10 @@ from typing import Any from uuid import UUID -from pydantic import BaseModel, Field, HttpUrl +from pydantic import AwareDatetime, BaseModel, Field, HttpUrl, field_validator + +from critic.libs.ddb import CONSTANT_GSI_PK +from critic.libs.dt import to_utc class MonitorState(str, Enum): @@ -21,7 +24,7 @@ class UptimeMonitorModel(BaseModel): state: MonitorState = MonitorState.new url: HttpUrl frequency_mins: int = Field(ge=1) - next_due_at: datetime + next_due_at: AwareDatetime timeout_secs: float = Field(ge=0) # TODO: assertions should probably become its own model assertions: dict[str, Any] | None = None @@ -29,7 +32,13 @@ class UptimeMonitorModel(BaseModel): alert_slack_channels: list[str] = Field(default_factory=list) alert_emails: list[str] = Field(default_factory=list) realert_interval_mins: int = Field(ge=0) - GSI_PK: str = Field(default='all monitors') + GSI_PK: str = Field(default=CONSTANT_GSI_PK) + + @field_validator('next_due_at') + @classmethod + def validate_next_due_at(cls, v: datetime) -> datetime: + """Normalize to UTC""" + return to_utc(v) class ProjectMonitors(BaseModel): diff --git a/src/critic/tables.py b/src/critic/tables.py index d4d56f8..05fcb3b 100644 --- a/src/critic/tables.py +++ b/src/critic/tables.py @@ -1,10 +1,27 @@ -from critic.libs.ddb import Table +from datetime import datetime + +from critic.libs.ddb import CONSTANT_GSI_PK, Table, deserialize, get_client, serialize from .models import UptimeMonitorModel class UptimeMonitorTable(Table): - name = 'UptimeMonitor' + base_name = 'UptimeMonitor' model = UptimeMonitorModel partition_key = 'project_id' sort_key = 'slug' + + @classmethod + def get_due_since(cls, timestamp: datetime) -> list[UptimeMonitorModel]: + response = get_client().query( + TableName=cls.name(), + IndexName='NextDueIndex', + KeyConditionExpression='GSI_PK = :pk AND next_due_at <= :timestamp', + ExpressionAttributeValues=serialize( + { + ':pk': CONSTANT_GSI_PK, + ':timestamp': timestamp, + } + ), + ) + return [cls.model(**deserialize(item)) for item in response['Items']] diff --git a/src/critic/tasks.py b/src/critic/tasks.py new file mode 100644 index 0000000..b0be1c3 --- /dev/null +++ b/src/critic/tasks.py @@ -0,0 +1,36 @@ +from datetime import UTC, datetime, timedelta +import logging + +import mu + +from critic.tables import UptimeMonitorTable + + +log = logging.getLogger(__name__) + + +@mu.task +def run_check(project_id: str, slug: str): + pass + + +@mu.task +def run_due_checks(): + """ + This task is invoked by an EventBridge rule once a minute. It queries for all monitors that are + due and invokes `run_check` for each one. + """ + now = datetime.now(UTC) + log.info(f'Triggering due checks at {now.isoformat()}') + + # Round `now` to the nearest minute in case there is a slight inaccuracy in scheduling + rounded_now = now.replace(second=0, microsecond=0) + if now.second >= 30: + rounded_now = rounded_now + timedelta(minutes=1) + + # Trigger `run_check` for each due monitor. + due_monitors = UptimeMonitorTable.get_due_since(rounded_now) + for monitor in due_monitors: + run_check.invoke(str(monitor.project_id), monitor.slug) + + log.info(f'Due checks triggered for {len(due_monitors)} monitors in {datetime.now(UTC) - now}') diff --git a/tests/critic_tests/conftest.py b/tests/critic_tests/conftest.py index f0f9872..990142c 100644 --- a/tests/critic_tests/conftest.py +++ b/tests/critic_tests/conftest.py @@ -4,6 +4,7 @@ from moto import mock_aws import pytest +import critic.libs.ddb as ddb_module from critic.libs.testing import clear_tables, create_tables @@ -42,6 +43,10 @@ def moto_for_unit_tests(request): with mock_aws(): create_tables() yield + # The DDB module is designed to cache the client. When we're testing unit tests and + # integration tests, this cache needs to be reset so the integration test doesn't get + # the mocked client and vice versa. + ddb_module._ddb_client = None def pytest_configure(config): diff --git a/tests/critic_tests/test_libs/test_ddb.py b/tests/critic_tests/test_libs/test_ddb.py index 3f60594..d0dd9af 100644 --- a/tests/critic_tests/test_libs/test_ddb.py +++ b/tests/critic_tests/test_libs/test_ddb.py @@ -1,3 +1,5 @@ +from datetime import datetime + import pytest from critic.models import UptimeMonitorModel @@ -65,3 +67,7 @@ def test_missing_sort_key(self): # error. with pytest.raises(ValueError): UptimeMonitorTable.get('6033aa47-a9f7-4d7f-b7ff-a11ba9b34474') + + def test_serialize_unaware_dt(self): + with pytest.raises(ValueError, match='must be timezone aware'): + UptimeMonitorTable.get_due_since(datetime.now()) diff --git a/tests/critic_tests/test_models.py b/tests/critic_tests/test_models.py new file mode 100644 index 0000000..9548935 --- /dev/null +++ b/tests/critic_tests/test_models.py @@ -0,0 +1,15 @@ +from datetime import UTC, datetime + +import pytest + +from critic.libs.testing import UptimeMonitorFactory + + +class TestUptimeMonitorModel: + def test_next_due_at_utc(self): + monitor = UptimeMonitorFactory.build(next_due_at='2026-01-01 12:00:00+04:00') + assert monitor.next_due_at == datetime(2026, 1, 1, 8, 0, 0, tzinfo=UTC) + + def test_next_due_at_unaware(self): + with pytest.raises(ValueError, match='Input should have timezone info'): + UptimeMonitorFactory.build(next_due_at='2026-01-01 12:00:00') diff --git a/tests/critic_tests/test_tasks.py b/tests/critic_tests/test_tasks.py new file mode 100644 index 0000000..5b648c8 --- /dev/null +++ b/tests/critic_tests/test_tasks.py @@ -0,0 +1,21 @@ +from unittest import mock + +from freezegun import freeze_time +import pytest + +from critic.libs.testing import UptimeMonitorFactory +from critic.tasks import run_due_checks + + +class TestRunDueChecks: + @pytest.mark.parametrize('kickoff_time', ['2026-01-01 12:00:01', '2026-01-01 11:59:59']) + @mock.patch('critic.tasks.run_check.invoke') + def test_run_due_checks(self, m_run_check, kickoff_time): + due = UptimeMonitorFactory.put(next_due_at='2026-01-01 12:00:00Z') + # Not due + UptimeMonitorFactory.put(next_due_at='2026-01-01 12:01:00Z') + + with freeze_time(kickoff_time, tz_offset=0): + run_due_checks() + + m_run_check.assert_called_once_with(str(due.project_id), due.slug) diff --git a/uv.lock b/uv.lock index 5baf45c..6ee7ee0 100644 --- a/uv.lock +++ b/uv.lock @@ -417,6 +417,7 @@ dependencies = [ { name = "flask" }, { name = "httpx" }, { name = "moto", extra = ["all"] }, + { name = "polyfactory" }, { name = "pydantic" }, ] @@ -427,6 +428,7 @@ audit = [ dev = [ { name = "click" }, { name = "env-config-cli" }, + { name = "freezegun" }, { name = "hatch" }, { name = "nox" }, { name = "pip-audit" }, @@ -445,6 +447,7 @@ pre-commit = [ { name = "pre-commit-uv" }, ] pytest = [ + { name = "freezegun" }, { name = "pytest" }, { name = "pytest-cov" }, { name = "respx" }, @@ -457,6 +460,7 @@ requires-dist = [ { name = "flask", specifier = ">=3.1.2" }, { name = "httpx", specifier = ">=0.27" }, { name = "moto", extras = ["all"], specifier = ">=5.1.14" }, + { name = "polyfactory", specifier = ">=3.2.0" }, { name = "pydantic", specifier = ">=2.7" }, ] @@ -465,6 +469,7 @@ audit = [{ name = "pip-audit" }] dev = [ { name = "click" }, { name = "env-config-cli", specifier = ">=0.20250626.2" }, + { name = "freezegun", specifier = ">=1.5.5" }, { name = "hatch" }, { name = "nox" }, { name = "pip-audit" }, @@ -481,6 +486,7 @@ pre-commit = [ { name = "pre-commit-uv" }, ] pytest = [ + { name = "freezegun", specifier = ">=1.5.5" }, { name = "pytest" }, { name = "pytest-cov" }, { name = "respx", specifier = ">=0.21" }, @@ -616,6 +622,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c8/36/e70a58da7f98b13669cb8d32461a140ad7b554497b188bd564f977c53272/env_config_cli-0.20250626.2-py3-none-any.whl", hash = "sha256:0e3c9bf74d26a7fa02cc0ff434a92b493cfc3d12363d43cb2336062d58400c5b", size = 13000, upload-time = "2025-06-26T17:51:57.572Z" }, ] +[[package]] +name = "faker" +version = "40.1.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "tzdata", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5e/77/1c3ff07b6739b9a1d23ca01ec0a90a309a33b78e345a3eb52f9ce9240e36/faker-40.1.2.tar.gz", hash = "sha256:b76a68163aa5f171d260fc24827a8349bc1db672f6a665359e8d0095e8135d30", size = 1949802, upload-time = "2026-01-13T20:51:49.917Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/46/ec/91a434c8a53d40c3598966621dea9c50512bec6ce8e76fa1751015e74cef/faker-40.1.2-py3-none-any.whl", hash = "sha256:93503165c165d330260e4379fd6dc07c94da90c611ed3191a0174d2ab9966a42", size = 1985633, upload-time = "2026-01-13T20:51:47.982Z" }, +] + [[package]] name = "filelock" version = "3.20.3" @@ -642,6 +660,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ec/f9/7f9263c5695f4bd0023734af91bedb2ff8209e8de6ead162f35d8dc762fd/flask-3.1.2-py3-none-any.whl", hash = "sha256:ca1d8112ec8a6158cc29ea4858963350011b5c846a414cdb7a954aa9e967d03c", size = 103308, upload-time = "2025-08-19T21:03:19.499Z" }, ] +[[package]] +name = "freezegun" +version = "1.5.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "python-dateutil" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/95/dd/23e2f4e357f8fd3bdff613c1fe4466d21bfb00a6177f238079b17f7b1c84/freezegun-1.5.5.tar.gz", hash = "sha256:ac7742a6cc6c25a2c35e9292dfd554b897b517d2dec26891a2e8debf205cb94a", size = 35914, upload-time = "2025-08-09T10:39:08.338Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5e/2e/b41d8a1a917d6581fc27a35d05561037b048e47df50f27f8ac9c7e27a710/freezegun-1.5.5-py3-none-any.whl", hash = "sha256:cd557f4a75cf074e84bc374249b9dd491eaeacd61376b9eb3c423282211619d2", size = 19266, upload-time = "2025-08-09T10:39:06.636Z" }, +] + [[package]] name = "furl" version = "2.1.4" @@ -1384,6 +1414,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a3/58/35da89ee790598a0700ea49b2a66594140f44dec458c07e8e3d4979137fc/ply-3.11-py2.py3-none-any.whl", hash = "sha256:096f9b8350b65ebd2fd1346b12452efe5b9607f7482813ffca50c22722a807ce", size = 49567, upload-time = "2018-02-15T19:01:27.172Z" }, ] +[[package]] +name = "polyfactory" +version = "3.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "faker" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/97/92/e90639b1d2abe982749eba7e734571a343ea062f7d486498b1c2b852f019/polyfactory-3.2.0.tar.gz", hash = "sha256:879242f55208f023eee1de48522de5cb1f9fd2d09b2314e999a9592829d596d1", size = 346878, upload-time = "2025-12-21T11:18:51.017Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d9/21/93363d7b802aa904f8d4169bc33e0e316d06d26ee68d40fe0355057da98c/polyfactory-3.2.0-py3-none-any.whl", hash = "sha256:5945799cce4c56cd44ccad96fb0352996914553cc3efaa5a286930599f569571", size = 62181, upload-time = "2025-12-21T11:18:49.311Z" }, +] + [[package]] name = "pre-commit" version = "4.5.0"