|
| 1 | +import asyncio |
| 2 | +import json |
| 3 | +import logging |
| 4 | +from collections.abc import AsyncGenerator |
| 5 | +from typing import Any, Final, Optional, Union, cast |
| 6 | + |
| 7 | +import asyncpg |
| 8 | +from taskiq import AckableMessage, AsyncBroker, BrokerMessage |
| 9 | +from typing_extensions import override |
| 10 | + |
| 11 | +from taskiq_asyncpg.exceptions import DatabaseConnectionError |
| 12 | +from taskiq_asyncpg.queries import ( |
| 13 | + CREATE_TABLE_MESSAGES_QUERY, |
| 14 | + DELETE_MESSAGE_QUERY, |
| 15 | + INSERT_MESSAGE_QUERY, |
| 16 | + SELECT_MESSAGE_QUERY, |
| 17 | +) |
| 18 | + |
| 19 | +logger = logging.getLogger(__name__) |
| 20 | + |
| 21 | + |
| 22 | +class AsyncpgBroker(AsyncBroker): |
| 23 | + """Broker for TaskIQ based on Asyncpg.""" |
| 24 | + |
| 25 | + def __init__( |
| 26 | + self, |
| 27 | + dsn: Optional[str] = "postgres://postgres:postgres@localhost:5432/postgres", |
| 28 | + channel_name: str = "taskiq", |
| 29 | + table_name: str = "taskiq_messages", |
| 30 | + max_retry_attempts: int = 5, |
| 31 | + **connect_kwargs: Any, |
| 32 | + ) -> None: |
| 33 | + """ |
| 34 | + Construct a new broker. |
| 35 | +
|
| 36 | + :param dsn: connection string to PostgreSQL. |
| 37 | + :param channel_name: Name of the channel to listen on. |
| 38 | + :param table_name: Name of the table to store messages. |
| 39 | + :param max_retry_attempts: Maximum number of message processing attempts. |
| 40 | + :param connect_kwargs: additional arguments for nats `ConnectionPool` class. |
| 41 | + """ |
| 42 | + super().__init__() |
| 43 | + self._dsn: Final = dsn |
| 44 | + self.channel_name: Final = channel_name |
| 45 | + self.table_name: Final = table_name |
| 46 | + self.connect_kwargs: Final = connect_kwargs |
| 47 | + self.max_retry_attempts: Final = max_retry_attempts |
| 48 | + |
| 49 | + self.read_conn: Optional["asyncpg.Connection[asyncpg.Record]"] = None |
| 50 | + self.write_pool: Optional["asyncpg.pool.Pool[asyncpg.Record]"] = None |
| 51 | + self._queue: Optional[asyncio.Queue[str]] = None |
| 52 | + |
| 53 | + @override |
| 54 | + async def startup(self) -> None: |
| 55 | + """Initialize the broker.""" |
| 56 | + await super().startup() |
| 57 | + |
| 58 | + try: |
| 59 | + self.read_conn = await asyncpg.connect(self._dsn, **self.connect_kwargs) |
| 60 | + self.write_pool = await asyncpg.create_pool(self._dsn) |
| 61 | + |
| 62 | + if self.read_conn is None: |
| 63 | + msg = "read_conn not initialized" |
| 64 | + raise RuntimeError(msg) |
| 65 | + if self.write_pool is None: |
| 66 | + msg = "write_pool not initialized" |
| 67 | + raise RuntimeError(msg) |
| 68 | + |
| 69 | + async with self.write_pool.acquire() as conn: |
| 70 | + _ = await conn.execute( |
| 71 | + CREATE_TABLE_MESSAGES_QUERY.format(self.table_name), |
| 72 | + ) |
| 73 | + |
| 74 | + await self.read_conn.add_listener( |
| 75 | + self.channel_name, |
| 76 | + self._notification_handler, |
| 77 | + ) |
| 78 | + self._queue = asyncio.Queue() |
| 79 | + except Exception as error: |
| 80 | + raise DatabaseConnectionError(str(error)) from error |
| 81 | + |
| 82 | + @override |
| 83 | + async def shutdown(self) -> None: |
| 84 | + """Close all connections on shutdown.""" |
| 85 | + await super().shutdown() |
| 86 | + if self.read_conn is not None: |
| 87 | + await self.read_conn.close() |
| 88 | + if self.write_pool is not None: |
| 89 | + await self.write_pool.close() |
| 90 | + |
| 91 | + def _notification_handler( |
| 92 | + self, |
| 93 | + con_ref: Union[ |
| 94 | + "asyncpg.Connection[asyncpg.Record]", |
| 95 | + "asyncpg.pool.PoolConnectionProxy[asyncpg.Record]", |
| 96 | + ], |
| 97 | + pid: int, |
| 98 | + channel: str, |
| 99 | + payload: object, |
| 100 | + /, |
| 101 | + ) -> None: |
| 102 | + """Handle NOTIFY messages. |
| 103 | +
|
| 104 | + From asyncpg.connection.add_listener docstring: |
| 105 | + A callable or a coroutine function receiving the following arguments: |
| 106 | + **con_ref**: a Connection the callback is registered with; |
| 107 | + **pid**: PID of the Postgres server that sent the notification; |
| 108 | + **channel**: name of the channel the notification was sent to; |
| 109 | + **payload**: the payload. |
| 110 | + """ |
| 111 | + logger.debug("Received notification on channel %s: %s", channel, payload) |
| 112 | + if self._queue is not None: |
| 113 | + self._queue.put_nowait(str(payload)) |
| 114 | + |
| 115 | + @override |
| 116 | + async def kick(self, message: BrokerMessage) -> None: |
| 117 | + """ |
| 118 | + Send message to the channel. |
| 119 | +
|
| 120 | + Inserts the message into the database and sends a NOTIFY. |
| 121 | +
|
| 122 | + :param message: Message to send. |
| 123 | + """ |
| 124 | + if self.write_pool is None: |
| 125 | + raise ValueError("Please run startup before kicking.") |
| 126 | + |
| 127 | + async with self.write_pool.acquire() as conn: |
| 128 | + # Insert the message into the database |
| 129 | + message_inserted_id = cast( |
| 130 | + int, |
| 131 | + await conn.fetchval( |
| 132 | + INSERT_MESSAGE_QUERY.format(self.table_name), |
| 133 | + message.task_id, |
| 134 | + message.task_name, |
| 135 | + message.message.decode(), |
| 136 | + json.dumps(message.labels), |
| 137 | + ), |
| 138 | + ) |
| 139 | + |
| 140 | + delay_value = message.labels.get("delay") |
| 141 | + if delay_value is not None: |
| 142 | + delay_seconds = int(delay_value) |
| 143 | + _ = asyncio.create_task( # noqa: RUF006 |
| 144 | + self._schedule_notification(message_inserted_id, delay_seconds), |
| 145 | + ) |
| 146 | + else: |
| 147 | + # Send a NOTIFY with the message ID as payload |
| 148 | + _ = await conn.execute( |
| 149 | + f"NOTIFY {self.channel_name}, '{message_inserted_id}'", |
| 150 | + ) |
| 151 | + |
| 152 | + async def _schedule_notification(self, message_id: int, delay_seconds: int) -> None: |
| 153 | + """Schedule a notification to be sent after a delay.""" |
| 154 | + await asyncio.sleep(delay_seconds) |
| 155 | + if self.write_pool is None: |
| 156 | + return |
| 157 | + async with self.write_pool.acquire() as conn: |
| 158 | + # Send NOTIFY |
| 159 | + _ = await conn.execute(f"NOTIFY {self.channel_name}, '{message_id}'") |
| 160 | + |
| 161 | + @override |
| 162 | + async def listen(self) -> AsyncGenerator[AckableMessage, None]: |
| 163 | + """ |
| 164 | + Listen to the channel. |
| 165 | +
|
| 166 | + Yields messages as they are received. |
| 167 | +
|
| 168 | + :yields: AckableMessage instances. |
| 169 | + """ |
| 170 | + if self.read_conn is None: |
| 171 | + raise ValueError("Call startup before starting listening.") |
| 172 | + if self._queue is None: |
| 173 | + raise ValueError("Startup did not initialize the queue.") |
| 174 | + |
| 175 | + while True: |
| 176 | + try: |
| 177 | + payload = await self._queue.get() |
| 178 | + message_id = int(payload) |
| 179 | + message_row = await self.read_conn.fetchrow( |
| 180 | + SELECT_MESSAGE_QUERY.format(self.table_name), message_id, |
| 181 | + ) |
| 182 | + if message_row is None: |
| 183 | + logger.warning( |
| 184 | + f"Message with id {message_id} not found in database.", |
| 185 | + ) |
| 186 | + continue |
| 187 | + if message_row.get("message") is None: |
| 188 | + msg = "Message row does not have 'message' column" |
| 189 | + raise ValueError(msg) |
| 190 | + message_str = message_row["message"] |
| 191 | + if not isinstance(message_str, str): |
| 192 | + msg = "message is not a string" |
| 193 | + raise ValueError(msg) |
| 194 | + message_data = message_str.encode() |
| 195 | + |
| 196 | + async def ack(*, _message_id: int = message_id) -> None: |
| 197 | + if self.write_pool is None: |
| 198 | + raise ValueError("Call startup before starting listening.") |
| 199 | + |
| 200 | + async with self.write_pool.acquire() as conn: |
| 201 | + _ = await conn.execute( |
| 202 | + DELETE_MESSAGE_QUERY.format(self.table_name), |
| 203 | + _message_id, |
| 204 | + ) |
| 205 | + |
| 206 | + yield AckableMessage(data=message_data, ack=ack) |
| 207 | + except Exception as e: |
| 208 | + logger.exception(f"Error processing message: {e}") |
| 209 | + continue |
0 commit comments