From c25b69c9f013e9335c35fa1ccc5c24cf075671df Mon Sep 17 00:00:00 2001 From: Arpad Borsos Date: Mon, 28 Apr 2025 12:57:20 +0200 Subject: [PATCH] Bump `sqlalchemy` version Previously, `worker` would pin this to v1.3, but that would then fail tests within `shared` in a common workspace, as some tests in `shared` depend on v1.4 features. --- apps/worker/database/base.py | 3 +- apps/worker/database/engine.py | 14 ++++++--- apps/worker/pyproject.toml | 4 +-- apps/worker/ta_storage/pg.py | 7 +++-- apps/worker/tasks/test_results_processor.py | 7 +++-- apps/worker/uv.lock | 35 ++++++++++++++++++--- tools/devenv/config/test.yml | 4 +-- 7 files changed, 53 insertions(+), 21 deletions(-) diff --git a/apps/worker/database/base.py b/apps/worker/database/base.py index 33cfeae8a7..9b54dc4aec 100644 --- a/apps/worker/database/base.py +++ b/apps/worker/database/base.py @@ -2,8 +2,7 @@ from sqlalchemy import Column, types from sqlalchemy.dialects.postgresql import UUID -from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.orm import Session +from sqlalchemy.orm import Session, declarative_base from helpers.clock import get_utc_now diff --git a/apps/worker/database/engine.py b/apps/worker/database/engine.py index b9d086f319..ee3a023259 100644 --- a/apps/worker/database/engine.py +++ b/apps/worker/database/engine.py @@ -33,8 +33,8 @@ def json_dumps(d): class SessionFactory: def __init__(self, database_url, timeseries_database_url=None): - self.database_url = database_url - self.timeseries_database_url = timeseries_database_url + self.database_url = _fix_engine(database_url) + self.timeseries_database_url = _fix_engine(timeseries_database_url) self.main_engine = None self.timeseries_engine = None @@ -54,7 +54,7 @@ def create_session(self): timeseries_engine = self.timeseries_engine class RoutingSession(Session): - def get_bind(self, mapper=None, clause=None): + def get_bind(self, mapper=None, clause=None, **kwargs): if mapper is not None and issubclass( mapper.class_, TimeseriesBaseModel ): @@ -74,16 +74,20 @@ def get_bind(self, mapper=None, clause=None): return scoped_session(session_factory) +def _fix_engine(database_url: str) -> str: + return database_url.replace("postgres://", "postgresql://") + + session_factory = SessionFactory( database_url=get_config( "services", "database_url", - default="postgres://postgres:@postgres:5432/postgres", + default="postgresql://postgres:@postgres:5432/postgres", ), timeseries_database_url=get_config( "services", "timeseries_database_url", - default="postgres://postgres:@timescale:5432/postgres", + default="postgresql://postgres:@timescale:5432/postgres", ), ) diff --git a/apps/worker/pyproject.toml b/apps/worker/pyproject.toml index 3c13652788..b912ca0147 100644 --- a/apps/worker/pyproject.toml +++ b/apps/worker/pyproject.toml @@ -39,8 +39,8 @@ dependencies = [ "requests>=2.32.0", "sentry-sdk>=2.13.0", "shared", - "sqlalchemy==1.3.*", - "sqlparse==0.5.0", + "sqlalchemy<2", + "sqlparse>=0.5.0", "statsd>=3.3.0", "stripe>=11.4.1", "test-results-parser", diff --git a/apps/worker/ta_storage/pg.py b/apps/worker/ta_storage/pg.py index 258f669799..88f3b04a23 100644 --- a/apps/worker/ta_storage/pg.py +++ b/apps/worker/ta_storage/pg.py @@ -37,14 +37,15 @@ def get_repo_flag_ids(db_session: Session, repoid: int, flags: list[str]) -> set if not flags: return set() - return set( - db_session.query(RepositoryFlag.id_) + return { + flag.id_ + for flag in db_session.query(RepositoryFlag.id_) .filter( RepositoryFlag.repository_id == repoid, RepositoryFlag.flag_name.in_(flags), ) .all() - ) + } def modify_structures( diff --git a/apps/worker/tasks/test_results_processor.py b/apps/worker/tasks/test_results_processor.py index f36a84db3c..170e8ef47d 100644 --- a/apps/worker/tasks/test_results_processor.py +++ b/apps/worker/tasks/test_results_processor.py @@ -45,14 +45,15 @@ def get_repo_flag_ids(db_session: Session, repoid: int, flags: list[str]) -> set if not flags: return set() - return set( - db_session.query(RepositoryFlag.id_) + return { + flag.id_ + for flag in db_session.query(RepositoryFlag.id_) .filter( RepositoryFlag.repository_id == repoid, RepositoryFlag.flag_name.in_(flags), ) .all() - ) + } def create_daily_totals( diff --git a/apps/worker/uv.lock b/apps/worker/uv.lock index 3d9012a85a..66a3c9de05 100644 --- a/apps/worker/uv.lock +++ b/apps/worker/uv.lock @@ -655,6 +655,30 @@ grpc = [ { name = "grpcio" }, ] +[[package]] +name = "greenlet" +version = "3.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3f/74/907bb43af91782e0366b0960af62a8ce1f9398e4291cac7beaeffbee0c04/greenlet-3.2.1.tar.gz", hash = "sha256:9f4dd4b4946b14bb3bf038f81e1d2e535b7d94f1b2a59fdba1293cd9c1a0a4d7", size = 184475, upload-time = "2025-04-22T14:40:18.206Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/77/2a/581b3808afec55b2db838742527c40b4ce68b9b64feedff0fd0123f4b19a/greenlet-3.2.1-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:e1967882f0c42eaf42282a87579685c8673c51153b845fde1ee81be720ae27ac", size = 269119, upload-time = "2025-04-22T14:25:01.798Z" }, + { url = "https://files.pythonhosted.org/packages/b0/f3/1c4e27fbdc84e13f05afc2baf605e704668ffa26e73a43eca93e1120813e/greenlet-3.2.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e77ae69032a95640a5fe8c857ec7bee569a0997e809570f4c92048691ce4b437", size = 637314, upload-time = "2025-04-22T14:53:46.214Z" }, + { url = "https://files.pythonhosted.org/packages/fc/1a/9fc43cb0044f425f7252da9847893b6de4e3b20c0a748bce7ab3f063d5bc/greenlet-3.2.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3227c6ec1149d4520bc99edac3b9bc8358d0034825f3ca7572165cb502d8f29a", size = 651421, upload-time = "2025-04-22T14:55:00.852Z" }, + { url = "https://files.pythonhosted.org/packages/8a/65/d47c03cdc62c6680206b7420c4a98363ee997e87a5e9da1e83bd7eeb57a8/greenlet-3.2.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0ddda0197c5b46eedb5628d33dad034c455ae77708c7bf192686e760e26d6a0c", size = 645789, upload-time = "2025-04-22T15:04:37.702Z" }, + { url = "https://files.pythonhosted.org/packages/2f/40/0faf8bee1b106c241780f377b9951dd4564ef0972de1942ef74687aa6bba/greenlet-3.2.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:de62b542e5dcf0b6116c310dec17b82bb06ef2ceb696156ff7bf74a7a498d982", size = 648262, upload-time = "2025-04-22T14:27:07.55Z" }, + { url = "https://files.pythonhosted.org/packages/e0/a8/73305f713183c2cb08f3ddd32eaa20a6854ba9c37061d682192db9b021c3/greenlet-3.2.1-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c07a0c01010df42f1f058b3973decc69c4d82e036a951c3deaf89ab114054c07", size = 606770, upload-time = "2025-04-22T14:25:58.34Z" }, + { url = "https://files.pythonhosted.org/packages/c3/05/7d726e1fb7f8a6ac55ff212a54238a36c57db83446523c763e20cd30b837/greenlet-3.2.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:2530bfb0abcd451ea81068e6d0a1aac6dabf3f4c23c8bd8e2a8f579c2dd60d95", size = 1117960, upload-time = "2025-04-22T14:59:00.373Z" }, + { url = "https://files.pythonhosted.org/packages/bf/9f/2b6cb1bd9f1537e7b08c08705c4a1d7bd4f64489c67d102225c4fd262bda/greenlet-3.2.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:1c472adfca310f849903295c351d297559462067f618944ce2650a1878b84123", size = 1145500, upload-time = "2025-04-22T14:28:12.441Z" }, + { url = "https://files.pythonhosted.org/packages/e4/f6/339c6e707062319546598eb9827d3ca8942a3eccc610d4a54c1da7b62527/greenlet-3.2.1-cp313-cp313-win_amd64.whl", hash = "sha256:24a496479bc8bd01c39aa6516a43c717b4cee7196573c47b1f8e1011f7c12495", size = 295994, upload-time = "2025-04-22T14:50:44.796Z" }, + { url = "https://files.pythonhosted.org/packages/f1/72/2a251d74a596af7bb1717e891ad4275a3fd5ac06152319d7ad8c77f876af/greenlet-3.2.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:175d583f7d5ee57845591fc30d852b75b144eb44b05f38b67966ed6df05c8526", size = 629889, upload-time = "2025-04-22T14:53:48.434Z" }, + { url = "https://files.pythonhosted.org/packages/29/2e/d7ed8bf97641bf704b6a43907c0e082cdf44d5bc026eb8e1b79283e7a719/greenlet-3.2.1-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3ecc9d33ca9428e4536ea53e79d781792cee114d2fa2695b173092bdbd8cd6d5", size = 635261, upload-time = "2025-04-22T14:55:02.258Z" }, + { url = "https://files.pythonhosted.org/packages/1e/75/802aa27848a6fcb5e566f69c64534f572e310f0f12d41e9201a81e741551/greenlet-3.2.1-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3f56382ac4df3860ebed8ed838f268f03ddf4e459b954415534130062b16bc32", size = 632523, upload-time = "2025-04-22T15:04:39.221Z" }, + { url = "https://files.pythonhosted.org/packages/56/09/f7c1c3bab9b4c589ad356503dd71be00935e9c4db4db516ed88fc80f1187/greenlet-3.2.1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cc45a7189c91c0f89aaf9d69da428ce8301b0fd66c914a499199cfb0c28420fc", size = 628816, upload-time = "2025-04-22T14:27:08.869Z" }, + { url = "https://files.pythonhosted.org/packages/79/e0/1bb90d30b5450eac2dffeaac6b692857c4bd642c21883b79faa8fa056cf2/greenlet-3.2.1-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:51a2f49da08cff79ee42eb22f1658a2aed60c72792f0a0a95f5f0ca6d101b1fb", size = 593687, upload-time = "2025-04-22T14:25:59.676Z" }, + { url = "https://files.pythonhosted.org/packages/c5/b5/adbe03c8b4c178add20cc716021183ae6b0326d56ba8793d7828c94286f6/greenlet-3.2.1-cp313-cp313t-musllinux_1_1_aarch64.whl", hash = "sha256:0c68bbc639359493420282d2f34fa114e992a8724481d700da0b10d10a7611b8", size = 1105754, upload-time = "2025-04-22T14:59:02.585Z" }, + { url = "https://files.pythonhosted.org/packages/39/93/84582d7ef38dec009543ccadec6ab41079a6cbc2b8c0566bcd07bf1aaf6c/greenlet-3.2.1-cp313-cp313t-musllinux_1_1_x86_64.whl", hash = "sha256:e775176b5c203a1fa4be19f91da00fd3bff536868b77b237da3f4daa5971ae5d", size = 1125160, upload-time = "2025-04-22T14:28:13.975Z" }, +] + [[package]] name = "grpc-google-iam-v1" version = "0.14.0" @@ -1738,9 +1762,12 @@ wheels = [ [[package]] name = "sqlalchemy" -version = "1.3.24" +version = "1.4.54" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/c5/ab/81bef2f960abf3cdaf32fbf1994f0c6f5e6a5f1667b5713ed6ebf162b6a2/SQLAlchemy-1.3.24.tar.gz", hash = "sha256:ebbb777cbf9312359b897bf81ba00dae0f5cb69fba2a18265dcc18a6f5ef7519", size = 6353598, upload-time = "2021-03-30T23:04:30.273Z" } +dependencies = [ + { name = "greenlet", marker = "platform_machine == 'AMD64' or platform_machine == 'WIN32' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'ppc64le' or platform_machine == 'win32' or platform_machine == 'x86_64'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ce/af/20290b55d469e873cba9d41c0206ab5461ff49d759989b3fe65010f9d265/sqlalchemy-1.4.54.tar.gz", hash = "sha256:4470fbed088c35dc20b78a39aaf4ae54fe81790c783b3264872a0224f437c31a", size = 8470350, upload-time = "2024-09-05T15:54:10.398Z" } [[package]] name = "sqlalchemy-utils" @@ -1983,8 +2010,8 @@ requires-dist = [ { name = "requests", specifier = ">=2.32.0" }, { name = "sentry-sdk", specifier = ">=2.13.0" }, { name = "shared", directory = "../../libs/shared" }, - { name = "sqlalchemy", specifier = "==1.3.*" }, - { name = "sqlparse", specifier = "==0.5.0" }, + { name = "sqlalchemy", specifier = "<2" }, + { name = "sqlparse", specifier = ">=0.5.0" }, { name = "statsd", specifier = ">=3.3.0" }, { name = "stripe", specifier = ">=11.4.1" }, { name = "test-results-parser", git = "https://github.com/codecov/test-results-parser?rev=14fe3d7ce3575ca343806294cc51102a8f66b6e8" }, diff --git a/tools/devenv/config/test.yml b/tools/devenv/config/test.yml index d49df7c49a..33af137090 100644 --- a/tools/devenv/config/test.yml +++ b/tools/devenv/config/test.yml @@ -17,8 +17,8 @@ setup: receive: 15 services: - database_url: postgres://postgres:password@postgres:5432/postgres - timeseries_database_url: postgres://postgres:password@timescale:5432/postgres + database_url: postgresql://postgres:password@postgres:5432/postgres + timeseries_database_url: postgresql://postgres:password@timescale:5432/postgres redis_url: redis://redis:6379 minio: hash_key: testixik8qdauiab1yiffydimvi72ekq # never change this