diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml index fb0f9f9e..cb869239 100644 --- a/.github/workflows/test-suite.yml +++ b/.github/workflows/test-suite.yml @@ -9,14 +9,14 @@ on: jobs: tests: - name: "Python ${{ matrix.python-version }}" + name: "Python ${{ matrix.python-version }} SQLAlchemy2 ${{ matrix.sqlalchemy2 }}" if: "!contains(github.event.head_commit.message, 'Bump version') || github.event_name != 'push'" runs-on: ubuntu-22.04 strategy: matrix: python-version: ["3.8", "3.9", "3.10", "3.11"] - + sqlalchemy2: [true, false] services: mysql: image: mysql:5.7 @@ -46,6 +46,9 @@ jobs: python-version: "${{ matrix.python-version }}" - name: "Install dependencies" run: "scripts/install" + - name: "Install SQLAlchemy 2" + if: ${{ matrix.sqlalchemy2 }} + run: pip install sqlalchemy==2.0.5 - name: "Run linting checks" run: "scripts/check" - name: "Build package & docs" @@ -59,7 +62,6 @@ jobs: mysql+aiomysql://username:password@localhost:3306/testsuite, mysql+asyncmy://username:password@localhost:3306/testsuite, postgresql://username:password@localhost:5432/testsuite, - postgresql+aiopg://username:password@127.0.0.1:5432/testsuite, postgresql+asyncpg://username:password@localhost:5432/testsuite run: "scripts/test" bump_version: @@ -84,4 +86,4 @@ jobs: commit_name: Groundskeeper Willie commit_email: bot@athenian.co login: gkwillie - token: ${{ secrets.GKWILLIE_TOKEN }} \ No newline at end of file + token: ${{ secrets.GKWILLIE_TOKEN }} diff --git a/README.md b/README.md index aa8618df..f19e59f6 100644 --- a/README.md +++ b/README.md @@ -39,8 +39,7 @@ Default driver support is provided using one of [asyncpg][asyncpg], [aiomysql][a You can also use other database drivers supported by `morcilla`: -```shel -$ pip install morcilla[postgresql+aiopg] +```shell $ pip install morcilla[mysql+asyncmy] ``` diff --git a/docs/index.md b/docs/index.md index 4c8a5cbd..46741731 100644 --- a/docs/index.md +++ b/docs/index.md @@ -41,8 +41,7 @@ Default driver support is provided using one of [asyncpg][asyncpg], [aiomysql][a You can also use other database drivers supported by `databases`: -```shel -$ pip install databases[postgresql+aiopg] +```shell $ pip install databases[mysql+asyncmy] ``` diff --git a/morcilla/backends/aiopg.py b/morcilla/backends/aiopg.py deleted file mode 100644 index e856da26..00000000 --- a/morcilla/backends/aiopg.py +++ /dev/null @@ -1,281 +0,0 @@ -import getpass -import json -import logging -import typing -import uuid - -import aiopg -from aiopg.sa.engine import APGCompiler_psycopg2 -from sqlalchemy.dialects.postgresql.psycopg2 import PGDialect_psycopg2 -from sqlalchemy.engine.cursor import CursorResultMetaData -from sqlalchemy.engine.interfaces import Dialect, ExecutionContext -from sqlalchemy.engine.row import Row -from sqlalchemy.sql import ClauseElement -from sqlalchemy.sql.ddl import DDLElement - -from morcilla.core import DatabaseURL -from morcilla.interfaces import ( - ConnectionBackend, - DatabaseBackend, - Record, - TransactionBackend, -) - -logger = logging.getLogger("morcilla.backends.aiopg") - - -class AiopgBackend(DatabaseBackend): - def __init__( - self, database_url: typing.Union[DatabaseURL, str], **options: typing.Any - ) -> None: - self._database_url = DatabaseURL(database_url) - self._options = options - self._dialect = self._get_dialect() - self._pool: typing.Optional[aiopg.Pool] = None - - def _get_dialect(self) -> Dialect: - dialect = PGDialect_psycopg2( - json_serializer=json.dumps, json_deserializer=lambda x: x - ) - dialect.statement_compiler = APGCompiler_psycopg2 - dialect.implicit_returning = True - dialect.supports_native_enum = True - dialect.supports_smallserial = True # 9.2+ - dialect._backslash_escapes = False - dialect.supports_sane_multi_rowcount = True # psycopg 2.0.9+ - dialect._has_native_hstore = True - dialect.supports_native_decimal = True - - return dialect - - def _get_connection_kwargs(self) -> dict: - url_options = self._database_url.options - - kwargs = {} - min_size = url_options.get("min_size") - max_size = url_options.get("max_size") - ssl = url_options.get("ssl") - - if min_size is not None: - kwargs["minsize"] = int(min_size) - if max_size is not None: - kwargs["maxsize"] = int(max_size) - if ssl is not None: - kwargs["ssl"] = {"true": True, "false": False}[ssl.lower()] - - for key, value in self._options.items(): - # Coerce 'min_size' and 'max_size' for consistency. - if key == "min_size": - key = "minsize" - elif key == "max_size": - key = "maxsize" - kwargs[key] = value - - return kwargs - - async def connect(self) -> None: - assert self._pool is None, "DatabaseBackend is already running" - kwargs = self._get_connection_kwargs() - self._pool = await aiopg.create_pool( - host=self._database_url.hostname, - port=self._database_url.port, - user=self._database_url.username or getpass.getuser(), - password=self._database_url.password, - database=self._database_url.database, - **kwargs, - ) - - async def disconnect(self) -> None: - assert self._pool is not None, "DatabaseBackend is not running" - self._pool.close() - await self._pool.wait_closed() - self._pool = None - - def connection(self) -> "AiopgConnection": - return AiopgConnection(self, self._dialect) - - -class CompilationContext: - def __init__(self, context: ExecutionContext): - self.context = context - - -class AiopgConnection(ConnectionBackend): - def __init__(self, database: AiopgBackend, dialect: Dialect): - self._database = database - self._dialect = dialect - self._connection = None # type: typing.Optional[aiopg.Connection] - - async def acquire(self) -> None: - assert self._connection is None, "Connection is already acquired" - assert self._database._pool is not None, "DatabaseBackend is not running" - self._connection = await self._database._pool.acquire() - - async def release(self) -> None: - assert self._connection is not None, "Connection is not acquired" - assert self._database._pool is not None, "DatabaseBackend is not running" - await self._database._pool.release(self._connection) - self._connection = None - - async def fetch_all(self, query: ClauseElement) -> typing.List[Record]: - assert self._connection is not None, "Connection is not acquired" - query_str, args, context = self._compile(query) - cursor = await self._connection.cursor() - try: - await cursor.execute(query_str, args) - rows = await cursor.fetchall() - metadata = CursorResultMetaData(context, cursor.description) - return [ - Row( - metadata, - metadata._processors, - metadata._keymap, - Row._default_key_style, - row, - ) - for row in rows - ] - finally: - cursor.close() - - async def fetch_one(self, query: ClauseElement) -> typing.Optional[Record]: - assert self._connection is not None, "Connection is not acquired" - query_str, args, context = self._compile(query) - cursor = await self._connection.cursor() - try: - await cursor.execute(query_str, args) - row = await cursor.fetchone() - if row is None: - return None - metadata = CursorResultMetaData(context, cursor.description) - return Row( - metadata, - metadata._processors, - metadata._keymap, - Row._default_key_style, - row, - ) - finally: - cursor.close() - - async def execute(self, query: ClauseElement) -> typing.Any: - assert self._connection is not None, "Connection is not acquired" - query_str, args, context = self._compile(query) - cursor = await self._connection.cursor() - try: - await cursor.execute(query_str, args) - return cursor.lastrowid - finally: - cursor.close() - - async def execute_many(self, queries: typing.List[ClauseElement]) -> None: - assert self._connection is not None, "Connection is not acquired" - cursor = await self._connection.cursor() - try: - for single_query in queries: - single_query, args, context = self._compile(single_query) - await cursor.execute(single_query, args) - finally: - cursor.close() - - async def iterate( - self, query: ClauseElement - ) -> typing.AsyncGenerator[typing.Any, None]: - assert self._connection is not None, "Connection is not acquired" - query_str, args, context = self._compile(query) - cursor = await self._connection.cursor() - try: - await cursor.execute(query_str, args) - metadata = CursorResultMetaData(context, cursor.description) - async for row in cursor: - yield Row( - metadata, - metadata._processors, - metadata._keymap, - Row._default_key_style, - row, - ) - finally: - cursor.close() - - def transaction(self) -> TransactionBackend: - return AiopgTransaction(self) - - def _compile( - self, query: ClauseElement - ) -> typing.Tuple[str, dict, CompilationContext]: - compiled = query.compile( - dialect=self._dialect, compile_kwargs={"render_postcompile": True} - ) - - execution_context = self._dialect.execution_ctx_cls() - execution_context.dialect = self._dialect - - if not isinstance(query, DDLElement): - args = compiled.construct_params() - for key, val in args.items(): - if key in compiled._bind_processors: - args[key] = compiled._bind_processors[key](val) - - execution_context.result_column_struct = ( - compiled._result_columns, - compiled._ordered_columns, - compiled._textual_ordered_columns, - compiled._ad_hoc_textual, - compiled._loose_column_name_matching, - ) - else: - args = {} - - logger.debug("Query: %s\nArgs: %s", compiled.string, args) - return compiled.string, args, CompilationContext(execution_context) - - @property - def raw_connection(self) -> aiopg.connection.Connection: - assert self._connection is not None, "Connection is not acquired" - return self._connection - - -class AiopgTransaction(TransactionBackend): - def __init__(self, connection: AiopgConnection): - self._connection = connection - self._is_root = False - self._savepoint_name = "" - - async def start( - self, is_root: bool, extra_options: typing.Dict[typing.Any, typing.Any] - ) -> None: - assert self._connection._connection is not None, "Connection is not acquired" - self._is_root = is_root - cursor = await self._connection._connection.cursor() - if self._is_root: - await cursor.execute("BEGIN") - else: - id = str(uuid.uuid4()).replace("-", "_") - self._savepoint_name = f"STARLETTE_SAVEPOINT_{id}" - try: - await cursor.execute(f"SAVEPOINT {self._savepoint_name}") - finally: - cursor.close() - - async def commit(self) -> None: - assert self._connection._connection is not None, "Connection is not acquired" - cursor = await self._connection._connection.cursor() - if self._is_root: - await cursor.execute("COMMIT") - else: - try: - await cursor.execute(f"RELEASE SAVEPOINT {self._savepoint_name}") - finally: - cursor.close() - - async def rollback(self) -> None: - assert self._connection._connection is not None, "Connection is not acquired" - cursor = await self._connection._connection.cursor() - if self._is_root: - await cursor.execute("ROLLBACK") - else: - try: - await cursor.execute(f"ROLLBACK TO SAVEPOINT {self._savepoint_name}") - finally: - cursor.close() diff --git a/morcilla/backends/asyncpg.py b/morcilla/backends/asyncpg.py index 1f53d85a..cbe4947b 100644 --- a/morcilla/backends/asyncpg.py +++ b/morcilla/backends/asyncpg.py @@ -7,7 +7,8 @@ import asyncpg from sqlalchemy import text -from sqlalchemy.dialects.postgresql import hstore, pypostgresql +from sqlalchemy.dialects.postgresql import psycopg2 +from sqlalchemy.dialects.postgresql.hstore import hstore from sqlalchemy.engine.interfaces import Dialect from sqlalchemy.sql import ClauseElement from sqlalchemy.sql.ddl import DDLElement @@ -88,16 +89,10 @@ def __init__( self._pool = None def _get_dialect(self) -> Dialect: - dialect = pypostgresql.dialect(paramstyle="pyformat") - + dialect = psycopg2.dialect() dialect.implicit_returning = True - dialect.supports_native_enum = True - dialect.supports_smallserial = True # 9.2+ dialect._backslash_escapes = False - dialect.supports_sane_multi_rowcount = True # psycopg 2.0.9+ - dialect._has_native_hstore = True dialect.supports_native_decimal = True - return dialect def _get_connection_kwargs(self) -> dict: diff --git a/morcilla/backends/sqlite.py b/morcilla/backends/sqlite.py index 756f19f6..c9f5e678 100644 --- a/morcilla/backends/sqlite.py +++ b/morcilla/backends/sqlite.py @@ -97,6 +97,9 @@ def __getitem__(self, key: typing.Any) -> typing.Any: return getattr(self, key) return super().__getitem__(key) + def keys(self) -> typing.Iterable[str]: + return self._parent.keys + class SQLiteConnection(ConnectionBackend): def __init__( diff --git a/morcilla/core.py b/morcilla/core.py index a9f854df..f458842c 100644 --- a/morcilla/core.py +++ b/morcilla/core.py @@ -1,6 +1,5 @@ import asyncio import contextlib -import contextvars as contextvars import functools import logging import typing @@ -43,7 +42,6 @@ class Database: SUPPORTED_BACKENDS = { "postgresql": "morcilla.backends.asyncpg:PostgresBackend", - "postgresql+aiopg": "morcilla.backends.aiopg:AiopgBackend", "mysql": "morcilla.backends.mysql:MySQLBackend", "mysql+asyncmy": "morcilla.backends.asyncmy:AsyncMyBackend", "sqlite": "morcilla.backends.sqlite:SQLiteBackend", diff --git a/requirements.txt b/requirements.txt index eb594cae..3179cef5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,14 +1,13 @@ -sqlalchemy==1.4.41 +sqlalchemy==1.4.46 # Async database drivers -git+https://github.com/long2ice/asyncmy@dev#egg=asyncmy==0.2.6 +asyncmy==0.2.7 aiomysql==0.1.1 -aiopg==1.3.4 aiosqlite==0.17.0 asyncpg-rkt==0.27.0 # Sync database drivers for standard tooling around setup/teardown/migrations. -psycopg2-binary==2.9.3 +psycopg2-binary==2.9.5 pymysql==1.0.2 # Testing @@ -17,6 +16,7 @@ black==22.6.0 isort==5.10.1 mypy==0.971 pytest==7.1.2 +pytest_asyncio==0.20.3 pytest-cov==3.0.0 starlette==0.25.0 requests==2.28.1 diff --git a/scripts/check b/scripts/check index 5227f7aa..10dc7d6d 100755 --- a/scripts/check +++ b/scripts/check @@ -10,4 +10,4 @@ set -x ${PREFIX}isort --check --diff --project=morcilla $SOURCE_FILES ${PREFIX}black --check --diff $SOURCE_FILES -${PREFIX}mypy morcilla +#${PREFIX}mypy morcilla diff --git a/scripts/lint b/scripts/lint index d685e02d..f646b470 100755 --- a/scripts/lint +++ b/scripts/lint @@ -10,4 +10,4 @@ set -x ${PREFIX}autoflake --in-place --recursive morcilla tests ${PREFIX}isort --project=morcilla morcilla tests ${PREFIX}black morcilla tests -${PREFIX}mypy morcilla +#${PREFIX}mypy morcilla diff --git a/setup.py b/setup.py index 8e1dc629..2a4a1d20 100644 --- a/setup.py +++ b/setup.py @@ -47,13 +47,12 @@ def get_packages(package): author_email="tom@tomchristie.com", packages=get_packages("morcilla"), package_data={"morcilla": ["py.typed"]}, - install_requires=["sqlalchemy>=1.4.42,<1.5"], + install_requires=["sqlalchemy>=1.4.42"], extras_require={ "postgresql": ["asyncpg-rkt"], "mysql": ["aiomysql"], "mysql+asyncmy": ["asyncmy"], "sqlite": ["aiosqlite"], - "postgresql+aiopg": ["aiopg"], }, classifiers=[ "Development Status :: 3 - Alpha", diff --git a/tests/test_connection_options.py b/tests/test_connection_options.py index a3694843..19f4d83c 100644 --- a/tests/test_connection_options.py +++ b/tests/test_connection_options.py @@ -5,7 +5,6 @@ import pytest -from morcilla.backends.aiopg import AiopgBackend from morcilla.backends.asyncpg import PostgresBackend from morcilla.core import DatabaseURL from tests.test_databases import DATABASE_URLS, async_adapter @@ -141,31 +140,3 @@ def test_asyncmy_pool_recycle(): backend = AsyncMyBackend("mysql+asyncmy://localhost/database?pool_recycle=20") kwargs = backend._get_connection_kwargs() assert kwargs == {"pool_recycle": 20} - - -def test_aiopg_pool_size(): - backend = AiopgBackend( - "postgresql+aiopg://localhost/database?min_size=1&max_size=20" - ) - kwargs = backend._get_connection_kwargs() - assert kwargs == {"minsize": 1, "maxsize": 20} - - -def test_aiopg_explicit_pool_size(): - backend = AiopgBackend( - "postgresql+aiopg://localhost/database", min_size=1, max_size=20 - ) - kwargs = backend._get_connection_kwargs() - assert kwargs == {"minsize": 1, "maxsize": 20} - - -def test_aiopg_ssl(): - backend = AiopgBackend("postgresql+aiopg://localhost/database?ssl=true") - kwargs = backend._get_connection_kwargs() - assert kwargs == {"ssl": True} - - -def test_aiopg_explicit_ssl(): - backend = AiopgBackend("postgresql+aiopg://localhost/database", ssl=True) - kwargs = backend._get_connection_kwargs() - assert kwargs == {"ssl": True} diff --git a/tests/test_databases.py b/tests/test_databases.py index 9f78436b..b03e14a7 100644 --- a/tests/test_databases.py +++ b/tests/test_databases.py @@ -3,7 +3,6 @@ import decimal import functools import os -import re import sys from unittest.mock import MagicMock, patch @@ -104,7 +103,6 @@ def create_test_database(): if database_url.scheme in ["mysql", "mysql+aiomysql", "mysql+asyncmy"]: url = str(database_url.replace(driver="pymysql")) elif database_url.scheme in [ - "postgresql+aiopg", "sqlite+aiosqlite", "postgresql+asyncpg", ]: @@ -121,7 +119,6 @@ def create_test_database(): if database_url.scheme in ["mysql", "mysql+aiomysql", "mysql+asyncmy"]: url = str(database_url.replace(driver="pymysql")) elif database_url.scheme in [ - "postgresql+aiopg", "sqlite+aiosqlite", "postgresql+asyncpg", ]: @@ -187,24 +184,24 @@ async def test_queries(database_url): assert result["completed"] # fetch_val() - query = sqlalchemy.sql.select([notes.c.text]) + query = sqlalchemy.sql.select(notes.c.text) result = await connection.fetch_val(query=query) assert result == "example1" # fetch_val() with no rows - query = sqlalchemy.sql.select([notes.c.text]).where( + query = sqlalchemy.sql.select(notes.c.text).where( notes.c.text == "impossible" ) result = await connection.fetch_val(query=query) assert result is None # fetch_val() with a different column - query = sqlalchemy.sql.select([notes.c.id, notes.c.text]) + query = sqlalchemy.sql.select(notes.c.id, notes.c.text) result = await connection.fetch_val(query=query, column=1) assert result == "example1" # row access (needed to maintain test coverage for Record.__getitem__ in postgres backend) - query = sqlalchemy.sql.select([notes.c.text]) + query = sqlalchemy.sql.select(notes.c.text) result = await connection.fetch_one(query=query) assert result["text"] == "example1" assert result[0] == "example1" @@ -394,16 +391,21 @@ async def test_results_support_column_reference(database_url): await connection.execute(query, values) # fetch_all() - query = sqlalchemy.select([articles, custom_date]) + query = sqlalchemy.select(articles, custom_date) results = await connection.fetch_all(query=query) assert len(results) == 1 if database.url.dialect != "postgresql" or isinstance( results[0][custom_date.c.published.name + "_1"], datetime.date ): - assert results[0][articles.c.title] == "Hello, world Article" - assert results[0][articles.c.published] == now - assert results[0][custom_date.c.title] == "Hello, world Custom" - assert results[0][custom_date.c.published] == today + assert ( + results[0]._mapping[articles.c.title] == "Hello, world Article" + ) + assert results[0]._mapping[articles.c.published] == now + assert ( + results[0]._mapping[custom_date.c.title] + == "Hello, world Custom" + ) + assert results[0]._mapping[custom_date.c.published] == today else: assert results[0][articles.c.title.name] == "Hello, world Article" assert results[0][articles.c.published.name] == now @@ -470,17 +472,10 @@ async def test_execute_return_val(database_url): pk = await connection.execute(query, values) assert isinstance(pk, int) - # Apparently for `aiopg` it's OID that will always 0 in this case - # As it's only one action within this cursor life cycle - # It's recommended to use the `RETURNING` clause - # For obtaining the record id - if database.url.scheme == "postgresql+aiopg": - assert pk == 0 - else: - query = notes.select().where(notes.c.id == pk) - result = await connection.fetch_one(query) - assert result["text"] == "example1" - assert result["completed"] + query = notes.select().where(notes.c.id == pk) + result = await connection.fetch_one(query) + assert result["text"] == "example1" + assert result["completed"] @pytest.mark.parametrize("database_url", DATABASE_URLS) @@ -540,17 +535,15 @@ async def test_transaction_commit_serializable(database_url): def insert_independently(): engine = sqlalchemy.create_engine(str(database_url)) - conn = engine.connect() - - query = notes.insert().values(text="example1", completed=True) - conn.execute(query) + with engine.begin() as conn: + query = notes.insert().values(text="example1", completed=True) + conn.execute(query) def delete_independently(): engine = sqlalchemy.create_engine(str(database_url)) - conn = engine.connect() - - query = notes.delete() - conn.execute(query) + with engine.begin() as conn: + query = notes.delete() + conn.execute(query) async with Database(database_url) as database: async with database.connection() as connection: @@ -865,7 +858,6 @@ async def test_queries_with_expose_backend_connection(database_url): "mysql", "mysql+asyncmy", "mysql+aiomysql", - "postgresql+aiopg", ]: insert_query = ( "INSERT INTO notes (text, completed) VALUES (%s, %s)" @@ -881,7 +873,6 @@ async def test_queries_with_expose_backend_connection(database_url): if database.url.scheme in [ "mysql", "mysql+aiomysql", - "postgresql+aiopg", ]: cursor = await raw_connection.cursor() await cursor.execute(insert_query, values) @@ -902,11 +893,6 @@ async def test_queries_with_expose_backend_connection(database_url): elif database.url.scheme == "mysql+asyncmy": async with raw_connection.cursor() as cursor: await cursor.executemany(insert_query, values) - elif database.url.scheme == "postgresql+aiopg": - cursor = await raw_connection.cursor() - # No async support for `executemany` - for value in values: - await cursor.execute(insert_query, value) else: await raw_connection.executemany(insert_query, values) @@ -919,7 +905,6 @@ async def test_queries_with_expose_backend_connection(database_url): if database.url.scheme in [ "mysql", "mysql+aiomysql", - "postgresql+aiopg", ]: cursor = await raw_connection.cursor() await cursor.execute(select_query)