diff --git a/.github/workflows/processing.yml b/.github/workflows/processing.yml index 1589607b9..525904242 100644 --- a/.github/workflows/processing.yml +++ b/.github/workflows/processing.yml @@ -20,6 +20,8 @@ jobs: run: cd infra/docker/python_base && make build image_name=818863528939.dkr.ecr.eu-central-1.amazonaws.com/badgerdoc/python_base:0.1.7 - name: run docker-compose up run: cd processing && docker-compose up -d --build + - name: install pytest + run: docker exec processing_web_app bash -c "pip install pytest" - name: run tests run: docker exec processing_web_app bash -c "pytest" - name: check with flake8 diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..aecde0525 --- /dev/null +++ b/.gitignore @@ -0,0 +1,410 @@ +# Created by https://www.toptal.com/developers/gitignore/api/python,visualstudiocode,pycharm,vim,macos,windows,linux +# Edit at https://www.toptal.com/developers/gitignore?templates=python,visualstudiocode,pycharm,vim,macos,windows,linux + +### Linux ### +*~ + +# temporary files which can be created if a process still has a handle open of a deleted file +.fuse_hidden* + +# KDE directory preferences +.directory + +# Linux trash folder which might appear on any partition or disk +.Trash-* + +# .nfs files are created when an open file is removed but is still being accessed +.nfs* + +### macOS ### +# General +.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon + + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +### macOS Patch ### +# iCloud generated files +*.icloud + +### PyCharm ### +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/**/usage.statistics.xml +.idea/**/dictionaries +.idea/**/shelf + +# AWS User-specific +.idea/**/aws.xml + +# Generated files +.idea/**/contentModel.xml + +# Sensitive or high-churn files +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/dbnavigator.xml + +# Gradle +.idea/**/gradle.xml +.idea/**/libraries + +# Gradle and Maven with auto-import +# When using Gradle or Maven with auto-import, you should exclude module files, +# since they will be recreated, and may cause churn. Uncomment if using +# auto-import. +.idea/artifacts +.idea/compiler.xml +.idea/jarRepositories.xml +.idea/modules.xml +.idea/*.iml +.idea/modules +*.iml +*.ipr + +# CMake +cmake-build-*/ + +# Mongo Explorer plugin +.idea/**/mongoSettings.xml + +# File-based project format +*.iws + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# SonarLint plugin +.idea/sonarlint/ + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +# Editor-based Rest Client +.idea/httpRequests + +# Android studio 3.1+ serialized cache file +.idea/caches/build_file_checksums.ser + +### PyCharm Patch ### +# Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721 + +# *.iml +modules.xml +.idea/misc.xml +# *.ipr + +# Sonarlint plugin +# https://plugins.jetbrains.com/plugin/7973-sonarlint +.idea/**/sonarlint/ + +# SonarQube Plugin +# https://plugins.jetbrains.com/plugin/7238-sonarqube-community-plugin +.idea/**/sonarIssues.xml + +# Markdown Navigator plugin +# https://plugins.jetbrains.com/plugin/7896-markdown-navigator-enhanced +.idea/**/markdown-navigator.xml +.idea/**/markdown-navigator-enh.xml +.idea/**/markdown-navigator/ + +# Cache file creation bug +# See https://youtrack.jetbrains.com/issue/JBR-2257 +.idea/$CACHE_FILE$ + +# CodeStream plugin +# https://plugins.jetbrains.com/plugin/12206-codestream +.idea/codestream.xml + +# Azure Toolkit for IntelliJ plugin +# https://plugins.jetbrains.com/plugin/8053-azure-toolkit-for-intellij +.idea/**/azureSettings.xml + +.idea/badgerdoc.iml +.idea/vcs.xml +.idea/inspectionProfiles/profiles_settings.xml + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +### Python Patch ### +# Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration +poetry.toml + +# ruff +.ruff_cache/ + +### Vim ### +# Swap +[._]*.s[a-v][a-z] +!*.svg # comment out if you don't need vector files +[._]*.sw[a-p] +[._]s[a-rt-v][a-z] +[._]ss[a-gi-z] +[._]sw[a-p] + +# Session +Session.vim +Sessionx.vim + +# Temporary +.netrwhist +# Auto-generated tag files +tags +# Persistent undo +[._]*.un~ + +### VisualStudioCode ### +.vscode/* +#!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +!.vscode/*.code-snippets + +# Local History for Visual Studio Code +.history/ + +# Built Visual Studio Code Extensions +*.vsix + +### VisualStudioCode Patch ### +# Ignore all local history of files +.history +.ionide + +### Windows ### +# Windows thumbnail cache files +Thumbs.db +Thumbs.db:encryptable +ehthumbs.db +ehthumbs_vista.db + +# Dump file +*.stackdump + +# Folder config file +[Dd]esktop.ini + +# Recycle Bin used on file shares +$RECYCLE.BIN/ + +# Windows Installer files +*.cab +*.msi +*.msix +*.msm +*.msp + +# Windows shortcuts +*.lnk + +# End of https://www.toptal.com/developers/gitignore/api/python,visualstudiocode,pycharm,vim,macos,windows,linux + +# Custom + +infra/docker/python_base/tenant_dependency +infra/docker/python_base/filter_lib +.trunk +**/.pytest_cache/** +**/*.egg-info \ No newline at end of file diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 000000000..26d33521a --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,3 @@ +# Default ignored files +/shelf/ +/workspace.xml diff --git a/annotation/.pre-commit-config.yaml b/annotation/.pre-commit-config.yaml index 9e7cf1fb3..947a49c76 100644 --- a/annotation/.pre-commit-config.yaml +++ b/annotation/.pre-commit-config.yaml @@ -1,7 +1,7 @@ fail_fast: true repos: - repo: https://github.com/pycqa/isort - rev: 5.9.2 + rev: 5.12.0 hooks: - id: isort args: diff --git a/annotation/Dockerfile b/annotation/Dockerfile index 4ffba7513..4201b63f3 100644 --- a/annotation/Dockerfile +++ b/annotation/Dockerfile @@ -14,7 +14,7 @@ RUN curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/inst # Copy using poetry.lock in case it doesn't exist yet COPY pyproject.toml poetry.lock .env alembic.ini version.txt ./ COPY ./alembic ./alembic -COPY ./app ./app +COPY annotation ./annotation FROM base as build @@ -28,7 +28,7 @@ RUN apt-get install -y make RUN poetry install --no-root COPY Makefile pytest.ini setup.cfg ./ COPY ./tests ./tests -COPY app app +COPY annotation annotation COPY alembic alembic RUN make test_checks diff --git a/annotation/alembic/env.py b/annotation/alembic/env.py index 77c270f2f..b0e56d610 100644 --- a/annotation/alembic/env.py +++ b/annotation/alembic/env.py @@ -1,11 +1,11 @@ import os from logging.config import fileConfig +from alembic import context # type: ignore from sqlalchemy import engine_from_config, pool -from alembic import context # type: ignore -from app.database import SQLALCHEMY_DATABASE_URL -from app.utils import get_test_db_url +from annotation.database import SQLALCHEMY_DATABASE_URL +from annotation.utils import get_test_db_url # this is the Alembic Config object, which provides # access to the values within the .ini file in use. @@ -19,7 +19,7 @@ # for 'autogenerate' support # from myapp import mymodel # target_metadata = mymodel.Base.metadata -from app.models import Base # noqa E402 +from annotation.models import Base # noqa E402 target_metadata = Base.metadata diff --git a/annotation/alembic/versions/1edef72a5043_add_categories_editor_url_data_.py b/annotation/alembic/versions/1edef72a5043_add_categories_editor_url_data_.py index f48b2e818..b30b4becd 100644 --- a/annotation/alembic/versions/1edef72a5043_add_categories_editor_url_data_.py +++ b/annotation/alembic/versions/1edef72a5043_add_categories_editor_url_data_.py @@ -6,9 +6,8 @@ """ import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - from alembic import op +from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision = "1edef72a5043" diff --git a/annotation/alembic/versions/2b3ed53127ea_alter_categories_id_integer_to_varchar.py b/annotation/alembic/versions/2b3ed53127ea_alter_categories_id_integer_to_varchar.py index e01797b0b..89a03e59f 100644 --- a/annotation/alembic/versions/2b3ed53127ea_alter_categories_id_integer_to_varchar.py +++ b/annotation/alembic/versions/2b3ed53127ea_alter_categories_id_integer_to_varchar.py @@ -6,9 +6,8 @@ """ import sqlalchemy as sa -from sqlalchemy.engine.reflection import Inspector - from alembic import op +from sqlalchemy.engine.reflection import Inspector # revision identifiers, used by Alembic. revision = "2b3ed53127ea" diff --git a/annotation/alembic/versions/3136551008d8_agreement_metrics_relation.py b/annotation/alembic/versions/3136551008d8_agreement_metrics_relation.py index 2fbb85886..6b752eb98 100644 --- a/annotation/alembic/versions/3136551008d8_agreement_metrics_relation.py +++ b/annotation/alembic/versions/3136551008d8_agreement_metrics_relation.py @@ -6,9 +6,8 @@ """ import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - from alembic import op +from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision = "3136551008d8" diff --git a/annotation/alembic/versions/36bff2d016f7_expand_validationschema.py b/annotation/alembic/versions/36bff2d016f7_expand_validationschema.py index 44927752b..63d6f97ab 100644 --- a/annotation/alembic/versions/36bff2d016f7_expand_validationschema.py +++ b/annotation/alembic/versions/36bff2d016f7_expand_validationschema.py @@ -8,9 +8,8 @@ from enum import Enum import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - from alembic import op +from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision = "36bff2d016f7" @@ -135,7 +134,9 @@ def upgrade(): ondelete="cascade", ) - op.execute("ALTER TYPE validation_type ADD VALUE IF NOT EXISTS 'validation_only'") + op.execute( + "ALTER TYPE validation_type ADD VALUE IF NOT EXISTS 'validation_only'" + ) class ValidationSchema(str, Enum): diff --git a/annotation/alembic/versions/3a083a1fbba0_first_revision.py b/annotation/alembic/versions/3a083a1fbba0_first_revision.py index 0020e61d2..a0b245f19 100644 --- a/annotation/alembic/versions/3a083a1fbba0_first_revision.py +++ b/annotation/alembic/versions/3a083a1fbba0_first_revision.py @@ -6,11 +6,10 @@ """ import sqlalchemy as sa +from alembic import op from sqlalchemy.dialects import postgresql from sqlalchemy.engine.reflection import Inspector -from alembic import op - # revision identifiers, used by Alembic. revision = "3a083a1fbba0" down_revision = None diff --git a/annotation/alembic/versions/416952520dc1_add_name_and_job_type_to_job_model.py b/annotation/alembic/versions/416952520dc1_add_name_and_job_type_to_job_model.py index 7e2667058..15937f6d5 100644 --- a/annotation/alembic/versions/416952520dc1_add_name_and_job_type_to_job_model.py +++ b/annotation/alembic/versions/416952520dc1_add_name_and_job_type_to_job_model.py @@ -8,9 +8,8 @@ from enum import Enum import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - from alembic import op +from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision = "416952520dc1" diff --git a/annotation/alembic/versions/4272d0a43ff1_agreement_score.py b/annotation/alembic/versions/4272d0a43ff1_agreement_score.py index b48c75b06..41a8b43f1 100644 --- a/annotation/alembic/versions/4272d0a43ff1_agreement_score.py +++ b/annotation/alembic/versions/4272d0a43ff1_agreement_score.py @@ -6,9 +6,8 @@ """ import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - from alembic import op +from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision = "4272d0a43ff1" diff --git a/annotation/alembic/versions/4cd10ac49cc2_expand_annotated_doc.py b/annotation/alembic/versions/4cd10ac49cc2_expand_annotated_doc.py index 0299c744b..6635d5dfa 100644 --- a/annotation/alembic/versions/4cd10ac49cc2_expand_annotated_doc.py +++ b/annotation/alembic/versions/4cd10ac49cc2_expand_annotated_doc.py @@ -6,9 +6,8 @@ """ import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - from alembic import op +from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision = "4cd10ac49cc2" diff --git a/annotation/alembic/versions/615379303da2_added_links_json.py b/annotation/alembic/versions/615379303da2_added_links_json.py index de05a782f..cbc5b6f69 100644 --- a/annotation/alembic/versions/615379303da2_added_links_json.py +++ b/annotation/alembic/versions/615379303da2_added_links_json.py @@ -6,9 +6,8 @@ """ import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - from alembic import op +from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision = "615379303da2" diff --git a/annotation/alembic/versions/66cd6054c2d0_add_categories_tree.py b/annotation/alembic/versions/66cd6054c2d0_add_categories_tree.py index 58c611a5f..3a81f64e9 100644 --- a/annotation/alembic/versions/66cd6054c2d0_add_categories_tree.py +++ b/annotation/alembic/versions/66cd6054c2d0_add_categories_tree.py @@ -7,7 +7,6 @@ """ import sqlalchemy as sa import sqlalchemy_utils - from alembic import op # revision identifiers, used by Alembic. diff --git a/annotation/alembic/versions/6fb3e0d231ff_create_document_links_table.py b/annotation/alembic/versions/6fb3e0d231ff_create_document_links_table.py index bcd5eb461..01c22bab2 100644 --- a/annotation/alembic/versions/6fb3e0d231ff_create_document_links_table.py +++ b/annotation/alembic/versions/6fb3e0d231ff_create_document_links_table.py @@ -6,7 +6,6 @@ """ import sqlalchemy as sa - from alembic import op # revision identifiers, used by Alembic. diff --git a/annotation/alembic/versions/71095b8e6343_add_extensive_coverage_parameter.py b/annotation/alembic/versions/71095b8e6343_add_extensive_coverage_parameter.py index 0698ce063..75e333858 100644 --- a/annotation/alembic/versions/71095b8e6343_add_extensive_coverage_parameter.py +++ b/annotation/alembic/versions/71095b8e6343_add_extensive_coverage_parameter.py @@ -6,9 +6,9 @@ """ import sqlalchemy as sa - from alembic import op -from app.models import ValidationSchema + +from annotation.models import ValidationSchema # revision identifiers, used by Alembic. revision = "71095b8e6343" diff --git a/annotation/alembic/versions/7cc1ed83c309_compare_agreement_scores.py b/annotation/alembic/versions/7cc1ed83c309_compare_agreement_scores.py index a96504e7c..119b3603d 100644 --- a/annotation/alembic/versions/7cc1ed83c309_compare_agreement_scores.py +++ b/annotation/alembic/versions/7cc1ed83c309_compare_agreement_scores.py @@ -6,7 +6,6 @@ """ import sqlalchemy as sa - from alembic import op # revision identifiers, used by Alembic. diff --git a/annotation/alembic/versions/8ea2ff0fea64_create_association_doc_category_table.py b/annotation/alembic/versions/8ea2ff0fea64_create_association_doc_category_table.py index c349e14d6..e81e79300 100644 --- a/annotation/alembic/versions/8ea2ff0fea64_create_association_doc_category_table.py +++ b/annotation/alembic/versions/8ea2ff0fea64_create_association_doc_category_table.py @@ -6,7 +6,6 @@ """ import sqlalchemy as sa - from alembic import op # revision identifiers, used by Alembic. diff --git a/annotation/alembic/versions/8fbac489cef2_category_change_editor_data_attributes.py b/annotation/alembic/versions/8fbac489cef2_category_change_editor_data_attributes.py index c16f0523e..fe0fe5ac5 100644 --- a/annotation/alembic/versions/8fbac489cef2_category_change_editor_data_attributes.py +++ b/annotation/alembic/versions/8fbac489cef2_category_change_editor_data_attributes.py @@ -7,9 +7,8 @@ """ import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - from alembic import op +from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision = "8fbac489cef2" diff --git a/annotation/alembic/versions/9083dea3783e_make_document_categories_an_array_of_.py b/annotation/alembic/versions/9083dea3783e_make_document_categories_an_array_of_.py index 7e7eddea1..b675016b8 100644 --- a/annotation/alembic/versions/9083dea3783e_make_document_categories_an_array_of_.py +++ b/annotation/alembic/versions/9083dea3783e_make_document_categories_an_array_of_.py @@ -6,7 +6,6 @@ """ import sqlalchemy as sa - from alembic import op # revision identifiers, used by Alembic. diff --git a/annotation/alembic/versions/9c07a25ca06f_expand_file_model.py b/annotation/alembic/versions/9c07a25ca06f_expand_file_model.py index 971f25522..7db9c3788 100644 --- a/annotation/alembic/versions/9c07a25ca06f_expand_file_model.py +++ b/annotation/alembic/versions/9c07a25ca06f_expand_file_model.py @@ -8,9 +8,8 @@ from enum import Enum import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - from alembic import op +from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision = "9c07a25ca06f" diff --git a/annotation/alembic/versions/bda0eac5ce64_adds_overall_load_field_not_negative_.py b/annotation/alembic/versions/bda0eac5ce64_adds_overall_load_field_not_negative_.py index 4424ba0b4..dd84a9356 100644 --- a/annotation/alembic/versions/bda0eac5ce64_adds_overall_load_field_not_negative_.py +++ b/annotation/alembic/versions/bda0eac5ce64_adds_overall_load_field_not_negative_.py @@ -6,7 +6,6 @@ """ import sqlalchemy as sa - from alembic import op # revision identifiers, used by Alembic. diff --git a/annotation/alembic/versions/c053ae380212_expand_job_model.py b/annotation/alembic/versions/c053ae380212_expand_job_model.py index dd0d167c6..925580630 100644 --- a/annotation/alembic/versions/c053ae380212_expand_job_model.py +++ b/annotation/alembic/versions/c053ae380212_expand_job_model.py @@ -8,9 +8,8 @@ from enum import Enum import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - from alembic import op +from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision = "c053ae380212" diff --git a/annotation/alembic/versions/c06c594c7435_change_file_statuses.py b/annotation/alembic/versions/c06c594c7435_change_file_statuses.py index dd437a7e7..6ebef590f 100644 --- a/annotation/alembic/versions/c06c594c7435_change_file_statuses.py +++ b/annotation/alembic/versions/c06c594c7435_change_file_statuses.py @@ -8,9 +8,8 @@ from enum import Enum import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - from alembic import op +from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision = "c06c594c7435" diff --git a/annotation/alembic/versions/cf633ca94498_add_statuses_to_job.py b/annotation/alembic/versions/cf633ca94498_add_statuses_to_job.py index f26495237..0be193aaf 100644 --- a/annotation/alembic/versions/cf633ca94498_add_statuses_to_job.py +++ b/annotation/alembic/versions/cf633ca94498_add_statuses_to_job.py @@ -8,9 +8,8 @@ from enum import Enum import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - from alembic import op +from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision = "cf633ca94498" diff --git a/annotation/alembic/versions/d30649e367e0_adds_not_nullable_tenant_field_to_jobs_.py b/annotation/alembic/versions/d30649e367e0_adds_not_nullable_tenant_field_to_jobs_.py index 436dc8387..4db73da7b 100644 --- a/annotation/alembic/versions/d30649e367e0_adds_not_nullable_tenant_field_to_jobs_.py +++ b/annotation/alembic/versions/d30649e367e0_adds_not_nullable_tenant_field_to_jobs_.py @@ -6,7 +6,6 @@ """ import sqlalchemy as sa - from alembic import op # revision identifiers, used by Alembic. diff --git a/annotation/alembic/versions/f44cabeef963_drop_is_link_add_type_to_categories.py b/annotation/alembic/versions/f44cabeef963_drop_is_link_add_type_to_categories.py index 52b44dd70..87fd142b8 100644 --- a/annotation/alembic/versions/f44cabeef963_drop_is_link_add_type_to_categories.py +++ b/annotation/alembic/versions/f44cabeef963_drop_is_link_add_type_to_categories.py @@ -8,9 +8,8 @@ from enum import Enum import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - from alembic import op +from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision = "f44cabeef963" diff --git a/annotation/app/__init__.py b/annotation/annotation/__init__.py similarity index 100% rename from annotation/app/__init__.py rename to annotation/annotation/__init__.py diff --git a/annotation/annotation/annotations/__init__.py b/annotation/annotation/annotations/__init__.py new file mode 100644 index 000000000..38cf2af3e --- /dev/null +++ b/annotation/annotation/annotations/__init__.py @@ -0,0 +1,20 @@ +from annotation.annotations.main import (LATEST, MANIFEST, S3_START_PATH, + accumulate_pages_info, + add_search_annotation_producer, + check_task_pages, + construct_annotated_doc, + create_manifest_json, get_pages_sha, + row_to_dict) + +__all__ = [ + add_search_annotation_producer, + row_to_dict, + accumulate_pages_info, + S3_START_PATH, + LATEST, + MANIFEST, + check_task_pages, + construct_annotated_doc, + create_manifest_json, + get_pages_sha, +] diff --git a/annotation/app/annotations/main.py b/annotation/annotation/annotations/main.py similarity index 98% rename from annotation/app/annotations/main.py rename to annotation/annotation/annotations/main.py index 0fa40f5f0..5e6f94256 100644 --- a/annotation/app/annotations/main.py +++ b/annotation/annotation/annotations/main.py @@ -14,17 +14,13 @@ from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import Session -from app import logger -from app.kafka_client import KAFKA_BOOTSTRAP_SERVER, KAFKA_SEARCH_TOPIC -from app.kafka_client import producers as kafka_producers -from app.models import AnnotatedDoc, DocumentLinks -from app.schemas import ( - AnnotatedDocSchema, - DocForSaveSchema, - PageSchema, - ParticularRevisionSchema, - RevisionLink, -) +from annotation import logger +from annotation.kafka_client import KAFKA_BOOTSTRAP_SERVER, KAFKA_SEARCH_TOPIC +from annotation.kafka_client import producers as kafka_producers +from annotation.models import AnnotatedDoc, DocumentLinks +from annotation.schemas import (AnnotatedDocSchema, DocForSaveSchema, + PageSchema, ParticularRevisionSchema, + RevisionLink) load_dotenv(find_dotenv()) ENDPOINT_URL = os.environ.get("S3_ENDPOINT_URL") diff --git a/annotation/app/annotations/resources.py b/annotation/annotation/annotations/resources.py similarity index 92% rename from annotation/app/annotations/resources.py rename to annotation/annotation/annotations/resources.py index 9f1809e0b..e25a7cbdb 100644 --- a/annotation/app/annotations/resources.py +++ b/annotation/annotation/annotations/resources.py @@ -6,42 +6,29 @@ from sqlalchemy.orm import Session from tenant_dependency import TenantData -from app.database import get_db -from app.errors import NoSuchRevisionsError -from app.microservice_communication.assets_communication import ( - get_file_path_and_bucket, -) -from app.microservice_communication.search import X_CURRENT_TENANT_HEADER -from app.schemas import ( - AnnotatedDocSchema, - BadRequestErrorSchema, - ConnectionErrorSchema, - DocForSaveSchema, - JobOutSchema, - NotFoundErrorSchema, - PageOutSchema, - ParticularRevisionSchema, - ValidationSchema, -) -from app.tags import ANNOTATION_TAG, JOBS_TAG, REVISION_TAG -from app.tasks import update_task_status +from annotation.database import get_db +from annotation.errors import NoSuchRevisionsError +from annotation.microservice_communication.assets_communication import \ + get_file_path_and_bucket +from annotation.microservice_communication.search import \ + X_CURRENT_TENANT_HEADER +from annotation.schemas import (AnnotatedDocSchema, BadRequestErrorSchema, + ConnectionErrorSchema, DocForSaveSchema, + JobOutSchema, NotFoundErrorSchema, + PageOutSchema, ParticularRevisionSchema, + ValidationSchema) +from annotation.tags import ANNOTATION_TAG, JOBS_TAG, REVISION_TAG +from annotation.tasks import update_task_status from ..models import AnnotatedDoc, File, Job, ManualAnnotationTask from ..token_dependency import TOKEN -from .main import ( - LATEST, - accumulate_pages_info, - add_search_annotation_producer, - check_if_kafka_message_is_needed, - check_null_fields, - check_task_pages, - construct_annotated_doc, - construct_particular_rev_response, - find_all_revisions_pages, - find_latest_revision_pages, - load_all_revisions_pages, - load_latest_revision_pages, -) +from .main import (LATEST, accumulate_pages_info, + add_search_annotation_producer, + check_if_kafka_message_is_needed, check_null_fields, + check_task_pages, construct_annotated_doc, + construct_particular_rev_response, find_all_revisions_pages, + find_latest_revision_pages, load_all_revisions_pages, + load_latest_revision_pages) router = APIRouter( prefix="/annotation", @@ -453,14 +440,19 @@ def get_annotations_up_to_given_revision( links_json=[], ) - validated, failed, annotated, _, categories, required_revision = ( - accumulate_pages_info( - task_pages=[], - revisions=revisions, - stop_revision=revision, - specific_pages=page_numbers, - with_page_hash=True, - ) + ( + validated, + failed, + annotated, + _, + categories, + required_revision, + ) = accumulate_pages_info( + task_pages=[], + revisions=revisions, + stop_revision=revision, + specific_pages=page_numbers, + with_page_hash=True, ) # if revision with given id (hash) was not found, # response with empty revision will be returned diff --git a/annotation/app/categories/__init__.py b/annotation/annotation/categories/__init__.py similarity index 100% rename from annotation/app/categories/__init__.py rename to annotation/annotation/categories/__init__.py diff --git a/annotation/app/categories/resources.py b/annotation/annotation/categories/resources.py similarity index 84% rename from annotation/app/categories/resources.py rename to annotation/annotation/categories/resources.py index 75a8cc60a..24d496c5f 100644 --- a/annotation/app/categories/resources.py +++ b/annotation/annotation/categories/resources.py @@ -5,31 +5,21 @@ from sqlalchemy.orm import Session from sqlalchemy_filters.exceptions import BadFilterFormat -from app.database import get_db -from app.errors import NoSuchCategoryError -from app.filters import CategoryFilter -from app.microservice_communication.search import X_CURRENT_TENANT_HEADER -from app.schemas import ( - BadRequestErrorSchema, - CategoryBaseSchema, - CategoryInputSchema, - CategoryResponseSchema, - ConnectionErrorSchema, - NotFoundErrorSchema, - SubCategoriesOutSchema, -) -from app.tags import CATEGORIES_TAG - -from .services import ( - add_category_db, - delete_category_db, - fetch_category_db, - filter_category_db, - insert_category_tree, - recursive_subcategory_search, - response_object_from_db, - update_category_db, -) +from annotation.database import get_db +from annotation.errors import NoSuchCategoryError +from annotation.filters import CategoryFilter +from annotation.microservice_communication.search import \ + X_CURRENT_TENANT_HEADER +from annotation.schemas import (BadRequestErrorSchema, CategoryBaseSchema, + CategoryInputSchema, CategoryResponseSchema, + ConnectionErrorSchema, NotFoundErrorSchema, + SubCategoriesOutSchema) +from annotation.tags import CATEGORIES_TAG + +from .services import (add_category_db, delete_category_db, fetch_category_db, + filter_category_db, insert_category_tree, + recursive_subcategory_search, response_object_from_db, + update_category_db) router = APIRouter( prefix="/categories", diff --git a/annotation/app/categories/services.py b/annotation/annotation/categories/services.py similarity index 97% rename from annotation/app/categories/services.py rename to annotation/annotation/categories/services.py index 6c25fcc46..6e6532b4e 100644 --- a/annotation/app/categories/services.py +++ b/annotation/annotation/categories/services.py @@ -9,20 +9,13 @@ from sqlalchemy.sql.expression import func from sqlalchemy_utils import Ltree -from app import logger as app_logger -from app.errors import ( - CheckFieldError, - ForeignKeyError, - NoSuchCategoryError, - SelfParentError, -) -from app.filters import CategoryFilter -from app.models import Category, Job -from app.schemas import ( - CategoryInputSchema, - CategoryORMSchema, - CategoryResponseSchema, -) +from annotation import logger as app_logger +from annotation.errors import (CheckFieldError, ForeignKeyError, + NoSuchCategoryError, SelfParentError) +from annotation.filters import CategoryFilter +from annotation.models import Category, Job +from annotation.schemas import (CategoryInputSchema, CategoryORMSchema, + CategoryResponseSchema) cache = TTLCache(maxsize=128, ttl=300) diff --git a/annotation/app/database.py b/annotation/annotation/database.py similarity index 100% rename from annotation/app/database.py rename to annotation/annotation/database.py diff --git a/annotation/annotation/distribution/__init__.py b/annotation/annotation/distribution/__init__.py new file mode 100644 index 000000000..0b5971176 --- /dev/null +++ b/annotation/annotation/distribution/__init__.py @@ -0,0 +1,26 @@ +from annotation.distribution.main import (add_unassigned_file, + calculate_users_load, distribute, + distribute_annotation_partial_files, + distribute_tasks, + distribute_validation_partial_files, + distribute_whole_files, + find_annotated_pages, + find_files_for_task, + find_unassigned_files, + find_unassigned_pages, + prepare_response) + +__all__ = [ + add_unassigned_file, + calculate_users_load, + distribute, + distribute_annotation_partial_files, + distribute_tasks, + distribute_validation_partial_files, + distribute_whole_files, + find_annotated_pages, + find_files_for_task, + find_unassigned_files, + find_unassigned_pages, + prepare_response, +] diff --git a/annotation/app/distribution/main.py b/annotation/annotation/distribution/main.py similarity index 98% rename from annotation/app/distribution/main.py rename to annotation/annotation/distribution/main.py index 7c58cda5d..e86d92efd 100644 --- a/annotation/app/distribution/main.py +++ b/annotation/annotation/distribution/main.py @@ -47,13 +47,12 @@ from sqlalchemy.orm import Session -from app.jobs import create_user, read_user -from app.microservice_communication.assets_communication import ( - FilesForDistribution, -) -from app.models import File, User -from app.schemas import TaskStatusEnumSchema, ValidationSchema -from app.tasks import create_tasks as create_db_tasks +from annotation.jobs import create_user, read_user +from annotation.microservice_communication.assets_communication import \ + FilesForDistribution +from annotation.models import File, User +from annotation.schemas import TaskStatusEnumSchema, ValidationSchema +from annotation.tasks import create_tasks as create_db_tasks MAX_PAGES = 50 @@ -101,11 +100,11 @@ def distribute( annotated_files_pages = {} # no pages distributed for annotation yet annotators = [ - x.__dict__ for x in annotators if x.default_load # type: ignore - ] + x.__dict__ for x in annotators if x.default_load + ] # type: ignore validators = [ - x.__dict__ for x in validators if x.default_load # type: ignore - ] + x.__dict__ for x in validators if x.default_load + ] # type: ignore if annotators: if ( validation_type == ValidationSchema.extensive_coverage @@ -176,7 +175,6 @@ def distribute_tasks_extensively( extensive_coverage: int, deadline: Optional[datetime] = None, ) -> List[Task]: - calculate_users_load( files=files, users=users, diff --git a/annotation/app/distribution/resources.py b/annotation/annotation/distribution/resources.py similarity index 83% rename from annotation/app/distribution/resources.py rename to annotation/annotation/distribution/resources.py index 8896007b3..516934b50 100644 --- a/annotation/app/distribution/resources.py +++ b/annotation/annotation/distribution/resources.py @@ -5,32 +5,21 @@ from sqlalchemy.orm import Session from tenant_dependency import TenantData -from app.database import get_db -from app.distribution import ( - distribute, - find_unassigned_files, - prepare_response, -) -from app.errors import FieldConstraintError -from app.jobs import ( - check_annotators, - check_validators, - get_job_attributes_for_post, -) -from app.microservice_communication.assets_communication import ( - get_files_info, - prepare_files_for_distribution, -) -from app.microservice_communication.search import X_CURRENT_TENANT_HEADER -from app.models import File, Job, User -from app.schemas import ( - BadRequestErrorSchema, - ConnectionErrorSchema, - ManualAnnotationTaskSchema, - TaskInfoSchema, -) -from app.tags import TASKS_TAG -from app.token_dependency import TOKEN +from annotation.database import get_db +from annotation.distribution import (distribute, find_unassigned_files, + prepare_response) +from annotation.errors import FieldConstraintError +from annotation.jobs import (check_annotators, check_validators, + get_job_attributes_for_post) +from annotation.microservice_communication.assets_communication import ( + get_files_info, prepare_files_for_distribution) +from annotation.microservice_communication.search import \ + X_CURRENT_TENANT_HEADER +from annotation.models import File, Job, User +from annotation.schemas import (BadRequestErrorSchema, ConnectionErrorSchema, + ManualAnnotationTaskSchema, TaskInfoSchema) +from annotation.tags import TASKS_TAG +from annotation.token_dependency import TOKEN router = APIRouter( prefix="/distribution", diff --git a/annotation/app/errors.py b/annotation/annotation/errors.py similarity index 98% rename from annotation/app/errors.py rename to annotation/annotation/errors.py index 8625bd4e6..9b9cc839e 100644 --- a/annotation/app/errors.py +++ b/annotation/annotation/errors.py @@ -3,7 +3,7 @@ from fastapi.responses import JSONResponse from sqlalchemy.exc import DBAPIError, SQLAlchemyError -from app import logger as app_logger +from annotation import logger as app_logger logger = app_logger.Logger diff --git a/annotation/app/filters.py b/annotation/annotation/filters.py similarity index 84% rename from annotation/app/filters.py rename to annotation/annotation/filters.py index 5d3b2d4dc..b07788120 100644 --- a/annotation/app/filters.py +++ b/annotation/annotation/filters.py @@ -1,6 +1,7 @@ from filter_lib import create_filter_model -from app.models import AnnotatedDoc, Category, Job, ManualAnnotationTask, User +from annotation.models import (AnnotatedDoc, Category, Job, + ManualAnnotationTask, User) CategoryFilter = create_filter_model( Category, diff --git a/annotation/annotation/jobs/__init__.py b/annotation/annotation/jobs/__init__.py new file mode 100644 index 000000000..47c191cc0 --- /dev/null +++ b/annotation/annotation/jobs/__init__.py @@ -0,0 +1,25 @@ +from annotation.jobs.services import (check_annotators, check_validators, + clean_tasks_before_jobs_update, + collect_job_names, create_user, + delete_redundant_users, delete_tasks, + get_job, get_job_attributes_for_post, + read_user, recalculate_file_pages, + update_files, update_inner_job_status, + update_user_overall_load) + +__all__ = [ + update_inner_job_status, + get_job, + get_job_attributes_for_post, + recalculate_file_pages, + update_files, + update_user_overall_load, + create_user, + read_user, + check_annotators, + check_validators, + collect_job_names, + clean_tasks_before_jobs_update, + delete_redundant_users, + delete_tasks, +] diff --git a/annotation/app/jobs/resources.py b/annotation/annotation/jobs/resources.py similarity index 90% rename from annotation/app/jobs/resources.py rename to annotation/annotation/jobs/resources.py index dce3346b5..ffb32f38b 100644 --- a/annotation/app/jobs/resources.py +++ b/annotation/annotation/jobs/resources.py @@ -1,15 +1,8 @@ from typing import Dict, List, Optional, Set, Union from uuid import UUID -from fastapi import ( - APIRouter, - Depends, - HTTPException, - Path, - Query, - Response, - status, -) +from fastapi import (APIRouter, Depends, HTTPException, Path, Query, Response, + status) from filter_lib import Page from sqlalchemy import and_ from sqlalchemy.orm import Session @@ -17,59 +10,36 @@ from sqlalchemy_filters.exceptions import BadFilterFormat from tenant_dependency import TenantData -import app.categories.services -from app import logger as app_logger -from app.categories import fetch_bunch_categories_db -from app.database import get_db -from app.distribution import distribute -from app.filters import CategoryFilter -from app.microservice_communication.assets_communication import get_files_info -from app.microservice_communication.jobs_communication import ( - JobUpdateException, - update_job_status, -) -from app.microservice_communication.search import X_CURRENT_TENANT_HEADER -from app.schemas import ( - BadRequestErrorSchema, - CategoryResponseSchema, - ConnectionErrorSchema, - FileStatusEnumSchema, - JobFilesInfoSchema, - JobInfoSchema, - JobPatchSchema, - JobProgressSchema, - JobStatusEnumSchema, - JobTypeEnumSchema, - ManualAnnotationTaskSchema, - NotFoundErrorSchema, - TaskStatusEnumSchema, - UnassignedFilesInfoSchema, - ValidationSchema, -) -from app.tags import FILES_TAG, JOBS_TAG -from app.token_dependency import TOKEN - -from ..models import ( - AnnotatedDoc, - Category, - File, - Job, - ManualAnnotationTask, - User, -) -from .services import ( - clean_tasks_before_jobs_update, - collect_job_names, - delete_redundant_users, - filter_job_categories, - find_users, - get_job, - get_jobs_by_files, - update_inner_job_status, - update_job_categories, - update_job_files, - update_jobs_users, -) +import annotation.categories.services +from annotation import logger as app_logger +from annotation.categories import fetch_bunch_categories_db +from annotation.database import get_db +from annotation.distribution import distribute +from annotation.filters import CategoryFilter +from annotation.microservice_communication.assets_communication import \ + get_files_info +from annotation.microservice_communication.jobs_communication import ( + JobUpdateException, update_job_status) +from annotation.microservice_communication.search import \ + X_CURRENT_TENANT_HEADER +from annotation.schemas import (BadRequestErrorSchema, CategoryResponseSchema, + ConnectionErrorSchema, FileStatusEnumSchema, + JobFilesInfoSchema, JobInfoSchema, + JobPatchSchema, JobProgressSchema, + JobStatusEnumSchema, JobTypeEnumSchema, + ManualAnnotationTaskSchema, + NotFoundErrorSchema, TaskStatusEnumSchema, + UnassignedFilesInfoSchema, ValidationSchema) +from annotation.tags import FILES_TAG, JOBS_TAG +from annotation.token_dependency import TOKEN + +from ..models import (AnnotatedDoc, Category, File, Job, ManualAnnotationTask, + User) +from .services import (clean_tasks_before_jobs_update, collect_job_names, + delete_redundant_users, filter_job_categories, + find_users, get_job, get_jobs_by_files, + update_inner_job_status, update_job_categories, + update_job_files, update_jobs_users) logger = app_logger.Logger @@ -555,7 +525,7 @@ def search_job_categories( """ get_job(db, job_id, x_current_tenant) try: - task_response = app.categories.services.filter_category_db( + task_response = annotation.categories.services.filter_category_db( db, request, x_current_tenant, diff --git a/annotation/app/jobs/services.py b/annotation/annotation/jobs/services.py similarity index 94% rename from annotation/app/jobs/services.py rename to annotation/annotation/jobs/services.py index 70534575b..d3cb5428d 100644 --- a/annotation/app/jobs/services.py +++ b/annotation/annotation/jobs/services.py @@ -8,31 +8,23 @@ from sqlalchemy.orm import Session, query from sqlalchemy.orm.attributes import InstrumentedAttribute -from app.categories import fetch_bunch_categories_db -from app.categories.services import response_object_from_db -from app.database import Base -from app.errors import EnumValidationError, FieldConstraintError, WrongJobError -from app.microservice_communication.assets_communication import get_files_info -from app.microservice_communication.jobs_communication import get_job_names -from app.models import ( - Category, - File, - Job, - ManualAnnotationTask, - User, - association_job_annotator, - association_job_owner, - association_job_validator, -) -from app.schemas import ( - CROSS_MIN_ANNOTATORS_NUMBER, - CategoryResponseSchema, - FileStatusEnumSchema, - JobInfoSchema, - JobStatusEnumSchema, - TaskStatusEnumSchema, - ValidationSchema, -) +from annotation.categories import fetch_bunch_categories_db +from annotation.categories.services import response_object_from_db +from annotation.database import Base +from annotation.errors import (EnumValidationError, FieldConstraintError, + WrongJobError) +from annotation.microservice_communication.assets_communication import \ + get_files_info +from annotation.microservice_communication.jobs_communication import \ + get_job_names +from annotation.models import (Category, File, Job, ManualAnnotationTask, User, + association_job_annotator, + association_job_owner, + association_job_validator) +from annotation.schemas import (CROSS_MIN_ANNOTATORS_NUMBER, + CategoryResponseSchema, FileStatusEnumSchema, + JobInfoSchema, JobStatusEnumSchema, + TaskStatusEnumSchema, ValidationSchema) def update_inner_job_status( diff --git a/annotation/app/kafka_client.py b/annotation/annotation/kafka_client.py similarity index 100% rename from annotation/app/kafka_client.py rename to annotation/annotation/kafka_client.py diff --git a/annotation/app/logger.py b/annotation/annotation/logger.py similarity index 100% rename from annotation/app/logger.py rename to annotation/annotation/logger.py diff --git a/annotation/app/main.py b/annotation/annotation/main.py similarity index 58% rename from annotation/app/main.py rename to annotation/annotation/main.py index 0ebf0ff14..09da927eb 100644 --- a/annotation/app/main.py +++ b/annotation/annotation/main.py @@ -7,41 +7,33 @@ from sqlalchemy.exc import DBAPIError, SQLAlchemyError from starlette.requests import Request -from app.annotations import resources as annotations_resources -from app.categories import resources as categories_resources -from app.distribution import resources as distribution_resources -from app.errors import ( - AgreementScoreServiceException, - CheckFieldError, - EnumValidationError, - FieldConstraintError, - ForeignKeyError, - NoSuchCategoryError, - NoSuchRevisionsError, - SelfParentError, - WrongJobError, - agreement_score_service_error_handler, - category_foreign_key_error_handler, - category_parent_child_error_handler, - category_unique_field_error_handler, - db_dbapi_error_handler, - db_s3_error_handler, - db_sa_error_handler, - debug_exception_handler, - enum_validation_error_handler, - field_constraint_error_handler, - minio_no_such_bucket_error_handler, - no_such_category_error_handler, - no_such_revisions_error_handler, - wrong_job_error_handler, -) -from app.jobs import resources as jobs_resources -from app import logger as app_logger -from app.metadata import resources as metadata_resources -from app.revisions import resources as revision_resources -from app.tags import TAGS -from app.tasks import resources as task_resources -from app.token_dependency import TOKEN +from annotation import logger as app_logger +from annotation.annotations import resources as annotations_resources +from annotation.categories import resources as categories_resources +from annotation.distribution import resources as distribution_resources +from annotation.errors import (AgreementScoreServiceException, CheckFieldError, + EnumValidationError, FieldConstraintError, + ForeignKeyError, NoSuchCategoryError, + NoSuchRevisionsError, SelfParentError, + WrongJobError, + agreement_score_service_error_handler, + category_foreign_key_error_handler, + category_parent_child_error_handler, + category_unique_field_error_handler, + db_dbapi_error_handler, db_s3_error_handler, + db_sa_error_handler, debug_exception_handler, + enum_validation_error_handler, + field_constraint_error_handler, + minio_no_such_bucket_error_handler, + no_such_category_error_handler, + no_such_revisions_error_handler, + wrong_job_error_handler) +from annotation.jobs import resources as jobs_resources +from annotation.metadata import resources as metadata_resources +from annotation.revisions import resources as revision_resources +from annotation.tags import TAGS +from annotation.tasks import resources as task_resources +from annotation.token_dependency import TOKEN load_dotenv(find_dotenv()) @@ -77,7 +69,8 @@ async def catch_exceptions_middleware(request: Request, call_next): logger.exception(exception) raise exception -app.middleware('http')(catch_exceptions_middleware) + +app.middleware("http")(catch_exceptions_middleware) app.include_router(annotations_resources.router) app.include_router(task_resources.router) app.include_router(distribution_resources.router) diff --git a/annotation/app/metadata/__init__.py b/annotation/annotation/metadata/__init__.py similarity index 100% rename from annotation/app/metadata/__init__.py rename to annotation/annotation/metadata/__init__.py diff --git a/annotation/app/metadata/resources.py b/annotation/annotation/metadata/resources.py similarity index 67% rename from annotation/app/metadata/resources.py rename to annotation/annotation/metadata/resources.py index a90e2173e..3cbaabcee 100644 --- a/annotation/app/metadata/resources.py +++ b/annotation/annotation/metadata/resources.py @@ -1,8 +1,9 @@ from fastapi import APIRouter, status -from app.microservice_communication.search import X_CURRENT_TENANT_HEADER -from app.schemas import EntitiesStatusesSchema -from app.tags import METADATA_TAG, TASKS_TAG +from annotation.microservice_communication.search import \ + X_CURRENT_TENANT_HEADER +from annotation.schemas import EntitiesStatusesSchema +from annotation.tags import METADATA_TAG, TASKS_TAG router = APIRouter( prefix="/metadata", diff --git a/annotation/app/microservice_communication/__init__.py b/annotation/annotation/microservice_communication/__init__.py similarity index 100% rename from annotation/app/microservice_communication/__init__.py rename to annotation/annotation/microservice_communication/__init__.py diff --git a/annotation/app/microservice_communication/assets_communication.py b/annotation/annotation/microservice_communication/assets_communication.py similarity index 94% rename from annotation/app/microservice_communication/assets_communication.py rename to annotation/annotation/microservice_communication/assets_communication.py index 715ae95b7..77e81661d 100644 --- a/annotation/app/microservice_communication/assets_communication.py +++ b/annotation/annotation/microservice_communication/assets_communication.py @@ -5,13 +5,9 @@ from dotenv import find_dotenv, load_dotenv from requests import ConnectionError, RequestException, Timeout -from app.microservice_communication.search import ( - AUTHORIZATION, - BEARER, - HEADER_TENANT, - get_response, - raise_request_exception, -) +from annotation.microservice_communication.search import ( + AUTHORIZATION, BEARER, HEADER_TENANT, get_response, + raise_request_exception) load_dotenv(find_dotenv()) ASSETS_FILES_URL = os.environ.get("ASSETS_FILES_URL") diff --git a/annotation/app/microservice_communication/jobs_communication.py b/annotation/annotation/microservice_communication/jobs_communication.py similarity index 81% rename from annotation/app/microservice_communication/jobs_communication.py rename to annotation/annotation/microservice_communication/jobs_communication.py index 711f6b09b..604d89dbf 100644 --- a/annotation/app/microservice_communication/jobs_communication.py +++ b/annotation/annotation/microservice_communication/jobs_communication.py @@ -5,12 +5,10 @@ from dotenv import find_dotenv, load_dotenv from requests import RequestException -from app.microservice_communication.search import ( - AUTHORIZATION, - BEARER, - HEADER_TENANT, - get_response, -) +from annotation.microservice_communication.search import (AUTHORIZATION, + BEARER, + HEADER_TENANT, + get_response) load_dotenv(find_dotenv()) JOBS_SEARCH_URL = os.environ.get("JOBS_SEARCH_URL") diff --git a/annotation/app/microservice_communication/search.py b/annotation/annotation/microservice_communication/search.py similarity index 97% rename from annotation/app/microservice_communication/search.py rename to annotation/annotation/microservice_communication/search.py index a0fa8e699..b97a7282d 100644 --- a/annotation/app/microservice_communication/search.py +++ b/annotation/annotation/microservice_communication/search.py @@ -90,12 +90,13 @@ from typing import Dict, List import requests -from app.annotations import row_to_dict -from app.models import ManualAnnotationTask -from app.schemas import ExpandedManualAnnotationTaskSchema from fastapi import Header, HTTPException from requests.exceptions import ConnectionError, RequestException, Timeout +from annotation.annotations import row_to_dict +from annotation.models import ManualAnnotationTask +from annotation.schemas import ExpandedManualAnnotationTaskSchema + PAGE_SIZE = 100 # max page size in assets HEADER_TENANT = "X-Current-Tenant" AUTHORIZATION = "Authorization" diff --git a/annotation/app/microservice_communication/task.py b/annotation/annotation/microservice_communication/task.py similarity index 72% rename from annotation/app/microservice_communication/task.py rename to annotation/annotation/microservice_communication/task.py index f15a68d2d..59536c31f 100644 --- a/annotation/app/microservice_communication/task.py +++ b/annotation/annotation/microservice_communication/task.py @@ -5,16 +5,12 @@ from dotenv import find_dotenv, load_dotenv from requests import RequestException -from app.errors import AgreementScoreServiceException -from app.microservice_communication.search import ( - AUTHORIZATION, - BEARER, - HEADER_TENANT, -) -from app.schemas import ( - AgreementScoreServiceInput, - AgreementScoreServiceResponse, -) +from annotation.errors import AgreementScoreServiceException +from annotation.microservice_communication.search import (AUTHORIZATION, + BEARER, + HEADER_TENANT) +from annotation.schemas import (AgreementScoreServiceInput, + AgreementScoreServiceResponse) load_dotenv(find_dotenv()) AGREEMENT_SCORE_SERVICE_URL = os.environ.get("AGREEMENT_SCORE_SERVICE_URL") diff --git a/annotation/app/microservice_communication/user.py b/annotation/annotation/microservice_communication/user.py similarity index 96% rename from annotation/app/microservice_communication/user.py rename to annotation/annotation/microservice_communication/user.py index 3b1a5382a..5b621f4f4 100644 --- a/annotation/app/microservice_communication/user.py +++ b/annotation/annotation/microservice_communication/user.py @@ -5,7 +5,7 @@ from dotenv import find_dotenv, load_dotenv from requests import RequestException -from app.models import ManualAnnotationTask +from annotation.models import ManualAnnotationTask load_dotenv(find_dotenv()) USERS_SEARCH_URL = os.environ.get("USERS_SEARCH_URL") diff --git a/annotation/app/models.py b/annotation/annotation/models.py similarity index 94% rename from annotation/app/models.py rename to annotation/annotation/models.py index 96870a333..2fd9047ba 100644 --- a/annotation/app/models.py +++ b/annotation/annotation/models.py @@ -1,38 +1,21 @@ from datetime import datetime from typing import Callable -from sqlalchemy import ( - BOOLEAN, - FLOAT, - INTEGER, - TIMESTAMP, - VARCHAR, - CheckConstraint, - Column, - DateTime, - ForeignKey, - ForeignKeyConstraint, - Index, - PrimaryKeyConstraint, - Table, - func, -) +from sqlalchemy import (BOOLEAN, FLOAT, INTEGER, TIMESTAMP, VARCHAR, + CheckConstraint, Column, DateTime, ForeignKey, + ForeignKeyConstraint, Index, PrimaryKeyConstraint, + Table, func) from sqlalchemy.dialects.postgresql import ARRAY, ENUM, JSON, JSONB, UUID from sqlalchemy.orm import relationship, validates from sqlalchemy_utils import Ltree, LtreeType -from app.database import Base -from app.errors import CheckFieldError -from app.schemas import ( - DEFAULT_LOAD, - AnnotationStatisticsEventEnumSchema, - CategoryTypeSchema, - FileStatusEnumSchema, - JobStatusEnumSchema, - JobTypeEnumSchema, - TaskStatusEnumSchema, - ValidationSchema, -) +from annotation.database import Base +from annotation.errors import CheckFieldError +from annotation.schemas import (DEFAULT_LOAD, + AnnotationStatisticsEventEnumSchema, + CategoryTypeSchema, FileStatusEnumSchema, + JobStatusEnumSchema, JobTypeEnumSchema, + TaskStatusEnumSchema, ValidationSchema) association_job_annotator = Table( "association_job_annotator", diff --git a/annotation/app/revisions/__init__.py b/annotation/annotation/revisions/__init__.py similarity index 100% rename from annotation/app/revisions/__init__.py rename to annotation/annotation/revisions/__init__.py diff --git a/annotation/app/revisions/resources.py b/annotation/annotation/revisions/resources.py similarity index 76% rename from annotation/app/revisions/resources.py rename to annotation/annotation/revisions/resources.py index c02c4fee0..53d9ae215 100644 --- a/annotation/app/revisions/resources.py +++ b/annotation/annotation/revisions/resources.py @@ -4,11 +4,12 @@ from sqlalchemy.orm import Session from starlette import status -from app.database import get_db -from app.microservice_communication.search import X_CURRENT_TENANT_HEADER -from app.models import AnnotatedDoc -from app.schemas import AnnotatedDocSchema, ConnectionErrorSchema -from app.tags import ANNOTATION_TAG, REVISION_TAG +from annotation.database import get_db +from annotation.microservice_communication.search import \ + X_CURRENT_TENANT_HEADER +from annotation.models import AnnotatedDoc +from annotation.schemas import AnnotatedDocSchema, ConnectionErrorSchema +from annotation.tags import ANNOTATION_TAG, REVISION_TAG router = APIRouter( prefix="/revisions", diff --git a/annotation/annotation/schemas/__init__.py b/annotation/annotation/schemas/__init__.py new file mode 100644 index 000000000..f2963d825 --- /dev/null +++ b/annotation/annotation/schemas/__init__.py @@ -0,0 +1,93 @@ +from annotation.schemas.annotations import (AnnotatedDocSchema, + DocForSaveSchema, PageOutSchema, + PageSchema, + ParticularRevisionSchema, + RevisionLink) +from annotation.schemas.categories import (CategoryBaseSchema, + CategoryDataAttributeNames, + CategoryInputSchema, + CategoryORMSchema, + CategoryResponseSchema, + CategoryTypeSchema, + SubCategoriesOutSchema) +from annotation.schemas.errors import (BadRequestErrorSchema, + ConnectionErrorSchema, + NotFoundErrorSchema) +from annotation.schemas.jobs import (CROSS_MIN_ANNOTATORS_NUMBER, DEFAULT_LOAD, + FileInfoSchema, FileStatusEnumSchema, + JobFilesInfoSchema, JobInfoSchema, + JobOutSchema, JobPatchSchema, + JobProgressSchema, JobStatusEnumSchema, + JobTypeEnumSchema, UnassignedFileSchema, + UnassignedFilesInfoSchema, + ValidationSchema) +from annotation.schemas.metadata import EntitiesStatusesSchema +from annotation.schemas.tasks import (AgreementScoreComparingResult, + AgreementScoreServiceInput, + AgreementScoreServiceResponse, + AnnotationAndValidationActionsSchema, + AnnotationStatisticsEventEnumSchema, + AnnotationStatisticsInputSchema, + AnnotationStatisticsResponseSchema, + ExpandedManualAnnotationTaskSchema, + ExportTaskStatsInput, + ManualAnnotationTaskInSchema, + ManualAnnotationTaskSchema, NameSchema, + PagesInfoSchema, ResponseScore, + TaskInfoSchema, TaskMetric, + TaskPatchSchema, TaskStatusEnumSchema, + TaskStatusSchema, ValidationEndSchema) + +__all__ = [ + AnnotatedDocSchema, + AgreementScoreServiceInput, + AgreementScoreServiceResponse, + AgreementScoreComparingResult, + AnnotationStatisticsEventEnumSchema, + AnnotationStatisticsInputSchema, + AnnotationStatisticsResponseSchema, + CategoryDataAttributeNames, + CategoryTypeSchema, + DocForSaveSchema, + ExportTaskStatsInput, + PageOutSchema, + PageSchema, + ParticularRevisionSchema, + RevisionLink, + CategoryBaseSchema, + CategoryInputSchema, + CategoryORMSchema, + CategoryResponseSchema, + SubCategoriesOutSchema, + FileInfoSchema, + FileStatusEnumSchema, + JobFilesInfoSchema, + JobInfoSchema, + JobOutSchema, + JobPatchSchema, + JobStatusEnumSchema, + JobTypeEnumSchema, + UnassignedFileSchema, + UnassignedFilesInfoSchema, + ValidationSchema, + EntitiesStatusesSchema, + AnnotationAndValidationActionsSchema, + ExpandedManualAnnotationTaskSchema, + ManualAnnotationTaskInSchema, + ManualAnnotationTaskSchema, + NameSchema, + PagesInfoSchema, + TaskInfoSchema, + TaskPatchSchema, + TaskStatusEnumSchema, + ResponseScore, + TaskStatusSchema, + ValidationEndSchema, + CROSS_MIN_ANNOTATORS_NUMBER, + DEFAULT_LOAD, + BadRequestErrorSchema, + ConnectionErrorSchema, + NotFoundErrorSchema, + JobProgressSchema, + TaskMetric, +] diff --git a/annotation/app/schemas/annotations.py b/annotation/annotation/schemas/annotations.py similarity index 100% rename from annotation/app/schemas/annotations.py rename to annotation/annotation/schemas/annotations.py diff --git a/annotation/app/schemas/categories.py b/annotation/annotation/schemas/categories.py similarity index 97% rename from annotation/app/schemas/categories.py rename to annotation/annotation/schemas/categories.py index da49230cf..edd68ea39 100644 --- a/annotation/app/schemas/categories.py +++ b/annotation/annotation/schemas/categories.py @@ -3,7 +3,7 @@ from pydantic import BaseModel, Field, validator -from app.errors import CheckFieldError +from annotation.errors import CheckFieldError class CategoryTypeSchema(str, Enum): diff --git a/annotation/app/schemas/errors.py b/annotation/annotation/schemas/errors.py similarity index 100% rename from annotation/app/schemas/errors.py rename to annotation/annotation/schemas/errors.py diff --git a/annotation/app/schemas/jobs.py b/annotation/annotation/schemas/jobs.py similarity index 100% rename from annotation/app/schemas/jobs.py rename to annotation/annotation/schemas/jobs.py diff --git a/annotation/app/schemas/metadata.py b/annotation/annotation/schemas/metadata.py similarity index 78% rename from annotation/app/schemas/metadata.py rename to annotation/annotation/schemas/metadata.py index f69488de2..0395189fa 100644 --- a/annotation/app/schemas/metadata.py +++ b/annotation/annotation/schemas/metadata.py @@ -2,7 +2,7 @@ from pydantic import BaseModel -from app.schemas.tasks import TaskStatusEnumSchema +from annotation.schemas.tasks import TaskStatusEnumSchema class EntitiesStatusesSchema(BaseModel): diff --git a/annotation/app/schemas/tasks.py b/annotation/annotation/schemas/tasks.py similarity index 100% rename from annotation/app/schemas/tasks.py rename to annotation/annotation/schemas/tasks.py diff --git a/annotation/app/tags.py b/annotation/annotation/tags.py similarity index 100% rename from annotation/app/tags.py rename to annotation/annotation/tags.py diff --git a/annotation/annotation/tasks/__init__.py b/annotation/annotation/tasks/__init__.py new file mode 100644 index 000000000..5dac992e7 --- /dev/null +++ b/annotation/annotation/tasks/__init__.py @@ -0,0 +1,9 @@ +from .services import (add_task_stats_record, create_tasks, get_task_revisions, + update_task_status) + +__all__ = [ + add_task_stats_record, + create_tasks, + update_task_status, + get_task_revisions, +] diff --git a/annotation/app/tasks/resources.py b/annotation/annotation/tasks/resources.py similarity index 92% rename from annotation/app/tasks/resources.py rename to annotation/annotation/tasks/resources.py index 686d7c87b..120b71af1 100644 --- a/annotation/app/tasks/resources.py +++ b/annotation/annotation/tasks/resources.py @@ -5,17 +5,8 @@ from uuid import UUID import dotenv -from fastapi import ( - APIRouter, - Body, - Depends, - Header, - HTTPException, - Path, - Query, - Response, - status, -) +from fastapi import (APIRouter, Body, Depends, Header, HTTPException, Path, + Query, Response, status) from fastapi.responses import JSONResponse, StreamingResponse from filter_lib import Page from sqlalchemy import and_, not_ @@ -24,76 +15,47 @@ from sqlalchemy_filters.exceptions import BadFilterFormat from tenant_dependency import TenantData -from app.annotations import accumulate_pages_info, row_to_dict -from app.database import get_db -from app.filters import TaskFilter -from app.jobs import ( - collect_job_names, - delete_tasks, - get_job, - get_job_attributes_for_post, - recalculate_file_pages, - update_files, - update_inner_job_status, - update_user_overall_load, -) -from app.logger import Logger -from app.microservice_communication.assets_communication import get_file_names -from app.microservice_communication.jobs_communication import ( - JobUpdateException, - update_job_status, -) -from app.microservice_communication.search import ( - X_CURRENT_TENANT_HEADER, - expand_response, -) -from app.microservice_communication.user import ( - GetUserInfoAccessDenied, - get_user_logins, -) -from app.schemas import ( - AnnotationStatisticsInputSchema, - AnnotationStatisticsResponseSchema, - BadRequestErrorSchema, - ConnectionErrorSchema, - ExpandedManualAnnotationTaskSchema, - ExportTaskStatsInput, - FileStatusEnumSchema, - JobStatusEnumSchema, - ManualAnnotationTaskInSchema, - ManualAnnotationTaskSchema, - NotFoundErrorSchema, - PagesInfoSchema, - TaskPatchSchema, - TaskStatusEnumSchema, - ValidationEndSchema, - ValidationSchema, -) -from app.tags import REVISION_TAG, TASKS_TAG -from app.tasks.validation import ( - create_annotation_tasks, - create_validation_tasks, -) -from app.token_dependency import TOKEN +from annotation.annotations import accumulate_pages_info, row_to_dict +from annotation.database import get_db +from annotation.filters import TaskFilter +from annotation.jobs import (collect_job_names, delete_tasks, get_job, + get_job_attributes_for_post, + recalculate_file_pages, update_files, + update_inner_job_status, update_user_overall_load) +from annotation.logger import Logger +from annotation.microservice_communication.assets_communication import \ + get_file_names +from annotation.microservice_communication.jobs_communication import ( + JobUpdateException, update_job_status) +from annotation.microservice_communication.search import ( + X_CURRENT_TENANT_HEADER, expand_response) +from annotation.microservice_communication.user import ( + GetUserInfoAccessDenied, get_user_logins) +from annotation.schemas import (AnnotationStatisticsInputSchema, + AnnotationStatisticsResponseSchema, + BadRequestErrorSchema, ConnectionErrorSchema, + ExpandedManualAnnotationTaskSchema, + ExportTaskStatsInput, FileStatusEnumSchema, + JobStatusEnumSchema, + ManualAnnotationTaskInSchema, + ManualAnnotationTaskSchema, + NotFoundErrorSchema, PagesInfoSchema, + TaskPatchSchema, TaskStatusEnumSchema, + ValidationEndSchema, ValidationSchema) +from annotation.tags import REVISION_TAG, TASKS_TAG +from annotation.tasks.validation import (create_annotation_tasks, + create_validation_tasks) +from annotation.token_dependency import TOKEN from ..models import File, Job, ManualAnnotationTask -from .services import ( - add_task_stats_record, - count_annotation_tasks, - create_annotation_task, - create_export_csv, - evaluate_agreement_score, - filter_tasks_db, - finish_validation_task, - get_task_info, - get_task_revisions, - read_annotation_task, - read_annotation_tasks, - save_agreement_metrics, - unblock_validation_tasks, - validate_task_info, - validate_user_actions, -) +from .services import (add_task_stats_record, count_annotation_tasks, + create_annotation_task, create_export_csv, + evaluate_agreement_score, filter_tasks_db, + finish_validation_task, get_task_info, + get_task_revisions, read_annotation_task, + read_annotation_tasks, save_agreement_metrics, + unblock_validation_tasks, validate_task_info, + validate_user_actions) dotenv.load_dotenv(dotenv.find_dotenv()) AGREEMENT_SCORE_ENABLED = os.getenv("AGREEMENT_SCORE_ENABLED", "false") diff --git a/annotation/app/tasks/services.py b/annotation/annotation/tasks/services.py similarity index 94% rename from annotation/app/tasks/services.py rename to annotation/annotation/tasks/services.py index 65976e12c..67b4acc85 100644 --- a/annotation/app/tasks/services.py +++ b/annotation/annotation/tasks/services.py @@ -12,34 +12,24 @@ from sqlalchemy.orm import Session from tenant_dependency import TenantData -from app.errors import CheckFieldError, FieldConstraintError -from app.filters import TaskFilter -from app.jobs import update_files, update_user_overall_load -from app.microservice_communication.assets_communication import ( - get_file_path_and_bucket, -) -from app.microservice_communication.task import get_agreement_score -from app.models import ( - AgreementMetrics, - AnnotatedDoc, - AnnotationStatistics, - File, - ManualAnnotationTask, - association_job_annotator, - association_job_validator, -) -from app.schemas import ( - AgreementScoreComparingResult, - AgreementScoreServiceInput, - AgreementScoreServiceResponse, - AnnotationStatisticsInputSchema, - ExportTaskStatsInput, - ManualAnnotationTaskInSchema, - ResponseScore, - TaskMetric, - TaskStatusEnumSchema, - ValidationSchema, -) +from annotation.errors import CheckFieldError, FieldConstraintError +from annotation.filters import TaskFilter +from annotation.jobs import update_files, update_user_overall_load +from annotation.microservice_communication.assets_communication import \ + get_file_path_and_bucket +from annotation.microservice_communication.task import get_agreement_score +from annotation.models import (AgreementMetrics, AnnotatedDoc, + AnnotationStatistics, File, + ManualAnnotationTask, association_job_annotator, + association_job_validator) +from annotation.schemas import (AgreementScoreComparingResult, + AgreementScoreServiceInput, + AgreementScoreServiceResponse, + AnnotationStatisticsInputSchema, + ExportTaskStatsInput, + ManualAnnotationTaskInSchema, ResponseScore, + TaskMetric, TaskStatusEnumSchema, + ValidationSchema) dotenv.load_dotenv(dotenv.find_dotenv()) AGREEMENT_SCORE_MIN_MATCH = float(os.getenv("AGREEMENT_SCORE_MIN_MATCH")) @@ -315,8 +305,8 @@ def finish_validation_task(db: Session, task: ManualAnnotationTask) -> None: ManualAnnotationTask.is_validation.is_(True), ).with_for_update().update( { - ManualAnnotationTask.status: TaskStatusEnumSchema.finished # noqa: E501 - }, + ManualAnnotationTask.status: TaskStatusEnumSchema.finished + }, # noqa: E501 synchronize_session="fetch", ) db.commit() diff --git a/annotation/app/tasks/validation.py b/annotation/annotation/tasks/validation.py similarity index 97% rename from annotation/app/tasks/validation.py rename to annotation/annotation/tasks/validation.py index f4fc5dcbd..f0724c2fe 100644 --- a/annotation/app/tasks/validation.py +++ b/annotation/annotation/tasks/validation.py @@ -7,16 +7,12 @@ from sqlalchemy import and_, asc, null, or_ from sqlalchemy.orm import Session -from app.distribution import prepare_response -from app.microservice_communication.assets_communication import ( - FilesForDistribution, -) -from app.models import AnnotatedDoc, Job, User -from app.schemas import ( - AnnotationAndValidationActionsSchema, - TaskStatusEnumSchema, - ValidationSchema, -) +from annotation.distribution import prepare_response +from annotation.microservice_communication.assets_communication import \ + FilesForDistribution +from annotation.models import AnnotatedDoc, Job, User +from annotation.schemas import (AnnotationAndValidationActionsSchema, + TaskStatusEnumSchema, ValidationSchema) from .services import create_tasks diff --git a/annotation/app/token_dependency.py b/annotation/annotation/token_dependency.py similarity index 100% rename from annotation/app/token_dependency.py rename to annotation/annotation/token_dependency.py diff --git a/annotation/app/utils.py b/annotation/annotation/utils.py similarity index 90% rename from annotation/app/utils.py rename to annotation/annotation/utils.py index 2251e4f75..ea1853ede 100644 --- a/annotation/app/utils.py +++ b/annotation/annotation/utils.py @@ -7,6 +7,6 @@ def get_test_db_url(main_db_url: str) -> str: postgresql+psycopg2://admin:admin@host:5432/test_db """ main_db_url_split = main_db_url.split("/") - main_db_url_split[-1] = 'test_db' + main_db_url_split[-1] = "test_db" result = "/".join(main_db_url_split) return result diff --git a/annotation/app/annotations/__init__.py b/annotation/app/annotations/__init__.py deleted file mode 100644 index a810a92d7..000000000 --- a/annotation/app/annotations/__init__.py +++ /dev/null @@ -1,25 +0,0 @@ -from app.annotations.main import ( - LATEST, - MANIFEST, - S3_START_PATH, - accumulate_pages_info, - add_search_annotation_producer, - check_task_pages, - construct_annotated_doc, - create_manifest_json, - get_pages_sha, - row_to_dict, -) - -__all__ = [ - add_search_annotation_producer, - row_to_dict, - accumulate_pages_info, - S3_START_PATH, - LATEST, - MANIFEST, - check_task_pages, - construct_annotated_doc, - create_manifest_json, - get_pages_sha, -] diff --git a/annotation/app/distribution/__init__.py b/annotation/app/distribution/__init__.py deleted file mode 100644 index 0eb12a693..000000000 --- a/annotation/app/distribution/__init__.py +++ /dev/null @@ -1,29 +0,0 @@ -from app.distribution.main import ( - add_unassigned_file, - calculate_users_load, - distribute, - distribute_annotation_partial_files, - distribute_tasks, - distribute_validation_partial_files, - distribute_whole_files, - find_annotated_pages, - find_files_for_task, - find_unassigned_files, - find_unassigned_pages, - prepare_response, -) - -__all__ = [ - add_unassigned_file, - calculate_users_load, - distribute, - distribute_annotation_partial_files, - distribute_tasks, - distribute_validation_partial_files, - distribute_whole_files, - find_annotated_pages, - find_files_for_task, - find_unassigned_files, - find_unassigned_pages, - prepare_response, -] diff --git a/annotation/app/jobs/__init__.py b/annotation/app/jobs/__init__.py deleted file mode 100644 index 007b5e949..000000000 --- a/annotation/app/jobs/__init__.py +++ /dev/null @@ -1,33 +0,0 @@ -from app.jobs.services import ( - check_annotators, - check_validators, - clean_tasks_before_jobs_update, - collect_job_names, - create_user, - delete_redundant_users, - delete_tasks, - get_job, - get_job_attributes_for_post, - read_user, - recalculate_file_pages, - update_files, - update_inner_job_status, - update_user_overall_load, -) - -__all__ = [ - update_inner_job_status, - get_job, - get_job_attributes_for_post, - recalculate_file_pages, - update_files, - update_user_overall_load, - create_user, - read_user, - check_annotators, - check_validators, - collect_job_names, - clean_tasks_before_jobs_update, - delete_redundant_users, - delete_tasks, -] diff --git a/annotation/app/schemas/__init__.py b/annotation/app/schemas/__init__.py deleted file mode 100644 index 892124fb5..000000000 --- a/annotation/app/schemas/__init__.py +++ /dev/null @@ -1,115 +0,0 @@ -from app.schemas.annotations import ( - AnnotatedDocSchema, - DocForSaveSchema, - PageOutSchema, - PageSchema, - ParticularRevisionSchema, - RevisionLink, -) -from app.schemas.categories import ( - CategoryBaseSchema, - CategoryDataAttributeNames, - CategoryInputSchema, - CategoryORMSchema, - CategoryResponseSchema, - CategoryTypeSchema, - SubCategoriesOutSchema, -) -from app.schemas.errors import ( - BadRequestErrorSchema, - ConnectionErrorSchema, - NotFoundErrorSchema, -) -from app.schemas.jobs import ( - CROSS_MIN_ANNOTATORS_NUMBER, - DEFAULT_LOAD, - FileInfoSchema, - FileStatusEnumSchema, - JobFilesInfoSchema, - JobInfoSchema, - JobOutSchema, - JobPatchSchema, - JobProgressSchema, - JobStatusEnumSchema, - JobTypeEnumSchema, - UnassignedFileSchema, - UnassignedFilesInfoSchema, - ValidationSchema, -) -from app.schemas.metadata import EntitiesStatusesSchema -from app.schemas.tasks import ( - AgreementScoreComparingResult, - AgreementScoreServiceInput, - AgreementScoreServiceResponse, - AnnotationAndValidationActionsSchema, - AnnotationStatisticsEventEnumSchema, - AnnotationStatisticsInputSchema, - AnnotationStatisticsResponseSchema, - ExpandedManualAnnotationTaskSchema, - ExportTaskStatsInput, - ManualAnnotationTaskInSchema, - ManualAnnotationTaskSchema, - NameSchema, - PagesInfoSchema, - ResponseScore, - TaskInfoSchema, - TaskMetric, - TaskPatchSchema, - TaskStatusEnumSchema, - TaskStatusSchema, - ValidationEndSchema, -) - -__all__ = [ - AnnotatedDocSchema, - AgreementScoreServiceInput, - AgreementScoreServiceResponse, - AgreementScoreComparingResult, - AnnotationStatisticsEventEnumSchema, - AnnotationStatisticsInputSchema, - AnnotationStatisticsResponseSchema, - CategoryDataAttributeNames, - CategoryTypeSchema, - DocForSaveSchema, - ExportTaskStatsInput, - PageOutSchema, - PageSchema, - ParticularRevisionSchema, - RevisionLink, - CategoryBaseSchema, - CategoryInputSchema, - CategoryORMSchema, - CategoryResponseSchema, - SubCategoriesOutSchema, - FileInfoSchema, - FileStatusEnumSchema, - JobFilesInfoSchema, - JobInfoSchema, - JobOutSchema, - JobPatchSchema, - JobStatusEnumSchema, - JobTypeEnumSchema, - UnassignedFileSchema, - UnassignedFilesInfoSchema, - ValidationSchema, - EntitiesStatusesSchema, - AnnotationAndValidationActionsSchema, - ExpandedManualAnnotationTaskSchema, - ManualAnnotationTaskInSchema, - ManualAnnotationTaskSchema, - NameSchema, - PagesInfoSchema, - TaskInfoSchema, - TaskPatchSchema, - TaskStatusEnumSchema, - ResponseScore, - TaskStatusSchema, - ValidationEndSchema, - CROSS_MIN_ANNOTATORS_NUMBER, - DEFAULT_LOAD, - BadRequestErrorSchema, - ConnectionErrorSchema, - NotFoundErrorSchema, - JobProgressSchema, - TaskMetric, -] diff --git a/annotation/app/tasks/__init__.py b/annotation/app/tasks/__init__.py deleted file mode 100644 index a19a0c458..000000000 --- a/annotation/app/tasks/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -from .services import ( - add_task_stats_record, - create_tasks, - get_task_revisions, - update_task_status, -) - -__all__ = [ - add_task_stats_record, - create_tasks, - update_task_status, - get_task_revisions, -] diff --git a/annotation/documentation/update_docs.py b/annotation/documentation/update_docs.py index d49ee5071..362ca5ce3 100644 --- a/annotation/documentation/update_docs.py +++ b/annotation/documentation/update_docs.py @@ -1,5 +1,4 @@ import yaml - from app.main import app diff --git a/annotation/tests/conftest.py b/annotation/tests/conftest.py index 2c32badb7..61b579bf1 100644 --- a/annotation/tests/conftest.py +++ b/annotation/tests/conftest.py @@ -9,167 +9,105 @@ import pytest import sqlalchemy import sqlalchemy_utils -from moto import mock_s3 -from sqlalchemy.engine import create_engine -from sqlalchemy.exc import SQLAlchemyError -from sqlalchemy.orm import Session, sessionmaker -from sqlalchemy.orm.exc import FlushError - import tests.test_get_accumulated_revisions as accumulated_revs import tests.test_get_jobs_info_by_files as jobs_info_by_files import tests.test_validation as validation from alembic import command from alembic.config import Config -from app.annotations import MANIFEST, S3_START_PATH -from app.categories import cache -from app.database import SQLALCHEMY_DATABASE_URL, Base -from app.jobs import update_user_overall_load -from app.models import ( - AnnotatedDoc, - Category, - DocumentLinks, - File, - Job, - ManualAnnotationTask, - User, -) -from app.schemas import ( - AnnotationStatisticsInputSchema, - CategoryTypeSchema, - FileStatusEnumSchema, - JobStatusEnumSchema, - TaskStatusEnumSchema, - ValidationSchema, -) -from app.tasks import add_task_stats_record -from app.utils import get_test_db_url +from moto import mock_s3 +from sqlalchemy.engine import create_engine +from sqlalchemy.exc import SQLAlchemyError +from sqlalchemy.orm import Session, sessionmaker +from sqlalchemy.orm.exc import FlushError from tests.override_app_dependency import TEST_TENANT -from tests.test_annotators_overall_load import ( - OVERALL_LOAD_CREATED_TASKS, - OVERALL_LOAD_JOBS, - OVERALL_LOAD_USERS, - TASK_FILES_OVERALL_LOAD, - VALIDATED_DOC_OVERALL_LOAD, -) -from tests.test_delete_batch_tasks import ( - DELETE_BATCH_TASKS_ANNOTATOR, - DELETE_BATCH_TASKS_FILE, - DELETE_BATCH_TASKS_JOB, - DIFF_STATUSES_TASKS, -) -from tests.test_finish_task import ( - CATEGORIES, - CATEGORIES_2, - FINISH_DOCS, - FINISH_DOCS_CHECK_DELETED_ANNOTATOR, - FINISH_TASK_1, - FINISH_TASK_1_SAME_JOB, - FINISH_TASK_2, - FINISH_TASK_2_SAME_JOB, - FINISH_TASK_CHECK_DELETE_USER_ANNOTATOR_1, - FINISH_TASK_CHECK_DELETE_USER_ANNOTATOR_2, - FINISH_TASK_CHECK_DELETE_USER_VALIDATOR, - FINISH_TASK_FILE_1, - FINISH_TASK_FILE_2, - FINISH_TASK_FILE_3, - FINISH_TASK_FILE_4, - FINISH_TASK_JOB_1, - FINISH_TASK_JOB_2, - FINISH_TASK_JOB_3, - FINISH_TASK_JOB_4, - FINISH_TASK_USER_1, - FINISH_TASK_USER_2, - FINISH_TASK_USER_3, - TASK_NOT_IN_PROGRESS_STATUS, - VALIDATION_TASKS_TO_READY, -) +from tests.test_annotators_overall_load import (OVERALL_LOAD_CREATED_TASKS, + OVERALL_LOAD_JOBS, + OVERALL_LOAD_USERS, + TASK_FILES_OVERALL_LOAD, + VALIDATED_DOC_OVERALL_LOAD) +from tests.test_delete_batch_tasks import (DELETE_BATCH_TASKS_ANNOTATOR, + DELETE_BATCH_TASKS_FILE, + DELETE_BATCH_TASKS_JOB, + DIFF_STATUSES_TASKS) +from tests.test_finish_task import (CATEGORIES, CATEGORIES_2, FINISH_DOCS, + FINISH_DOCS_CHECK_DELETED_ANNOTATOR, + FINISH_TASK_1, FINISH_TASK_1_SAME_JOB, + FINISH_TASK_2, FINISH_TASK_2_SAME_JOB, + FINISH_TASK_CHECK_DELETE_USER_ANNOTATOR_1, + FINISH_TASK_CHECK_DELETE_USER_ANNOTATOR_2, + FINISH_TASK_CHECK_DELETE_USER_VALIDATOR, + FINISH_TASK_FILE_1, FINISH_TASK_FILE_2, + FINISH_TASK_FILE_3, FINISH_TASK_FILE_4, + FINISH_TASK_JOB_1, FINISH_TASK_JOB_2, + FINISH_TASK_JOB_3, FINISH_TASK_JOB_4, + FINISH_TASK_USER_1, FINISH_TASK_USER_2, + FINISH_TASK_USER_3, + TASK_NOT_IN_PROGRESS_STATUS, + VALIDATION_TASKS_TO_READY) from tests.test_get_annotation_for_particular_revision import ( - PART_REV_ANNOTATOR, - PART_REV_DOC, - PART_REV_PAGES, -) -from tests.test_get_child_categories import ( - COMMON_CHILD_CATEGORIES, - CYCLIC_TENANT_CHILD_CATEGORIES, - OTHER_TENANT_CHILD_CATEGORY, -) -from tests.test_get_job import ( - GET_FILES, - GET_JOBS, - JOB_TEST_ANNOTATORS, - JOB_TEST_REVISIONS, -) + PART_REV_ANNOTATOR, PART_REV_DOC, PART_REV_PAGES) +from tests.test_get_child_categories import (COMMON_CHILD_CATEGORIES, + CYCLIC_TENANT_CHILD_CATEGORIES, + OTHER_TENANT_CHILD_CATEGORY) +from tests.test_get_job import (GET_FILES, GET_JOBS, JOB_TEST_ANNOTATORS, + JOB_TEST_REVISIONS) from tests.test_get_job_files import GET_JOB_FILES, GET_JOB_FILES_JOBS -from tests.test_get_job_progress import ( - FILE_TEST_PROGRESS, - JOBS_TO_TEST_PROGRESS, - TASKS_TEST_PROGRESS, -) +from tests.test_get_job_progress import (FILE_TEST_PROGRESS, + JOBS_TO_TEST_PROGRESS, + TASKS_TEST_PROGRESS) from tests.test_get_pages_info import PAGES_INFO_ENTITIES -from tests.test_get_revisions import ( - JOBS_IDS, - PAGE, - PAGES_PATHS, - REVISIONS, - USERS_IDS, -) +from tests.test_get_revisions import (JOBS_IDS, PAGE, PAGES_PATHS, REVISIONS, + USERS_IDS) from tests.test_get_revisions_without_annotation import ( - REV_WITHOUT_ANNOTATION_DOC_1, - REV_WITHOUT_ANNOTATION_DOC_2, - REV_WITHOUT_ANNOTATION_DOC_3, - REV_WITHOUT_ANNOTATION_JOB, - REV_WITHOUT_ANNOTATION_TASK, -) + REV_WITHOUT_ANNOTATION_DOC_1, REV_WITHOUT_ANNOTATION_DOC_2, + REV_WITHOUT_ANNOTATION_DOC_3, REV_WITHOUT_ANNOTATION_JOB, + REV_WITHOUT_ANNOTATION_TASK) from tests.test_get_unassigned_files import UNASSIGNED_FILES_ENTITIES -from tests.test_get_users_for_job import ( - USERS_FOR_JOB_ANNOTATORS, - USERS_FOR_JOB_JOBS, -) +from tests.test_get_users_for_job import (USERS_FOR_JOB_ANNOTATORS, + USERS_FOR_JOB_JOBS) from tests.test_job_categories import CATEGORIES_USERS, MOCK_ID from tests.test_post import POST_JOBS, TEST_POST_USERS -from tests.test_post_annotation import ( - ANNOTATION_VALIDATION_TASKS_PG, - MANIFEST_IN_MINIO, - POST_ANNOTATION_ANNOTATOR, - POST_ANNOTATION_FILE_1, - POST_ANNOTATION_JOB_1, - POST_ANNOTATION_PG_DOC, - POST_ANNOTATION_PG_TASK_1, - POST_ANNOTATION_PG_TASK_2, - POST_ANNOTATION_VALIDATION_JOB, - POST_ANNOTATION_VALIDATOR, - S3_PATH, -) +from tests.test_post_annotation import (ANNOTATION_VALIDATION_TASKS_PG, + MANIFEST_IN_MINIO, + POST_ANNOTATION_ANNOTATOR, + POST_ANNOTATION_FILE_1, + POST_ANNOTATION_JOB_1, + POST_ANNOTATION_PG_DOC, + POST_ANNOTATION_PG_TASK_1, + POST_ANNOTATION_PG_TASK_2, + POST_ANNOTATION_VALIDATION_JOB, + POST_ANNOTATION_VALIDATOR, S3_PATH) from tests.test_post_job import POST_JOB_EXISTING_JOB from tests.test_post_next_task import NEXT_TASK_ANNOTATION_TASKS, NEXT_TASK_JOB -from tests.test_post_unassgined_files import ( - ANNOTATORS_POST_UN_FILES, - JOBS_FILES_TASKS_POST_UN_FILES, -) -from tests.test_search_kafka import ( - ANNOTATION_KAFKA_FILE, - ANNOTATION_KAFKA_JOB, - ANNOTATION_KAFKA_TASK, -) +from tests.test_post_unassgined_files import (ANNOTATORS_POST_UN_FILES, + JOBS_FILES_TASKS_POST_UN_FILES) +from tests.test_search_kafka import (ANNOTATION_KAFKA_FILE, + ANNOTATION_KAFKA_JOB, + ANNOTATION_KAFKA_TASK) from tests.test_start_job import CHANGE_STATUSES_JOBS, CHANGE_STATUSES_TASKS from tests.test_tasks_crud_cr import CRUD_CR_ANNOTATION_TASKS, CRUD_CR_JOBS from tests.test_tasks_crud_cr import FILES as CRUD_CR_FILES -from tests.test_tasks_crud_ud import ( - CRUD_UD_CONSTRAINTS_FILES, - CRUD_UD_CONSTRAINTS_JOBS, - CRUD_UD_CONSTRAINTS_TASKS, - CRUD_UD_CONSTRAINTS_USERS, - CRUD_UD_JOB_1, - CRUD_UD_JOB_2, - CRUD_UD_TASK, -) -from tests.test_update_job import ( - UPDATE_JOB_CATEGORIES, - UPDATE_JOB_FILES, - UPDATE_JOB_USERS, - UPDATE_JOBS, - UPDATE_USER_NO_JOBS, -) +from tests.test_tasks_crud_ud import (CRUD_UD_CONSTRAINTS_FILES, + CRUD_UD_CONSTRAINTS_JOBS, + CRUD_UD_CONSTRAINTS_TASKS, + CRUD_UD_CONSTRAINTS_USERS, CRUD_UD_JOB_1, + CRUD_UD_JOB_2, CRUD_UD_TASK) +from tests.test_update_job import (UPDATE_JOB_CATEGORIES, UPDATE_JOB_FILES, + UPDATE_JOB_USERS, UPDATE_JOBS, + UPDATE_USER_NO_JOBS) + +from annotation.annotations import MANIFEST, S3_START_PATH +from annotation.categories import cache +from annotation.database import SQLALCHEMY_DATABASE_URL, Base +from annotation.jobs import update_user_overall_load +from annotation.models import (AnnotatedDoc, Category, DocumentLinks, File, + Job, ManualAnnotationTask, User) +from annotation.schemas import (AnnotationStatisticsInputSchema, + CategoryTypeSchema, FileStatusEnumSchema, + JobStatusEnumSchema, TaskStatusEnumSchema, + ValidationSchema) +from annotation.tasks import add_task_stats_record +from annotation.utils import get_test_db_url DEFAULT_REGION = "us-east-1" @@ -958,7 +896,7 @@ def mock_assets_communication( monkeypatch, prepare_db_categories_for_filtration ) -> Session: monkeypatch.setattr( - "app.jobs.resources.get_files_info", + "annotation.jobs.resources.get_files_info", Mock(return_value=[{"file_id": MOCK_ID, "pages_number": 2}]), ) return prepare_db_categories_for_filtration @@ -969,7 +907,7 @@ def mock_db_error_for_job_categories( monkeypatch, prepare_db_categories_for_filtration ) -> Session: monkeypatch.setattr( - "app.jobs.resources.fetch_bunch_categories_db", + "annotation.jobs.resources.fetch_bunch_categories_db", Mock(side_effect=SQLAlchemyError), ) return prepare_db_categories_for_filtration @@ -980,7 +918,7 @@ def mock_db_error_get_job_categories( monkeypatch, prepare_db_categories_for_filtration ) -> Session: monkeypatch.setattr( - "app.main.filter_job_categories", + "annotation.main.filter_job_categories", Mock(side_effect=SQLAlchemyError), ) return prepare_db_categories_for_filtration @@ -1273,7 +1211,7 @@ def mock_exception(*args, **kwargs): @pytest.fixture def mock_minio_empty_bucket(monkeypatch, empty_bucket): monkeypatch.setattr( - "app.annotations.main.connect_s3", + "annotation.annotations.main.connect_s3", Mock(return_value=empty_bucket), ) yield empty_bucket diff --git a/annotation/tests/override_app_dependency.py b/annotation/tests/override_app_dependency.py index 1d4880204..20853ffaa 100644 --- a/annotation/tests/override_app_dependency.py +++ b/annotation/tests/override_app_dependency.py @@ -13,17 +13,13 @@ from sqlalchemy.orm import sessionmaker from tenant_dependency import TenantData -from app.database import get_db -from app.main import app -from app.microservice_communication.search import ( - AUTHORIZATION, - BEARER, - HEADER_TENANT, -) -from app.token_dependency import TOKEN -from app.utils import get_test_db_url -from app.database import SQLALCHEMY_DATABASE_URL - +from annotation.database import SQLALCHEMY_DATABASE_URL, get_db +from annotation.main import app +from annotation.microservice_communication.search import (AUTHORIZATION, + BEARER, + HEADER_TENANT) +from annotation.token_dependency import TOKEN +from annotation.utils import get_test_db_url TEST_TOKEN = "token" TEST_TENANT = "test" diff --git a/annotation/tests/test_annotators_overall_load.py b/annotation/tests/test_annotators_overall_load.py index 40e91533b..6342205ae 100644 --- a/annotation/tests/test_annotators_overall_load.py +++ b/annotation/tests/test_annotators_overall_load.py @@ -5,31 +5,20 @@ from pytest import mark, raises from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.orm import Session - -from app.jobs import update_user_overall_load -from app.main import app -from app.microservice_communication.assets_communication import ( - ASSETS_FILES_URL, -) -from app.models import ( - AnnotatedDoc, - Category, - File, - Job, - ManualAnnotationTask, - User, -) -from app.schemas import ( - CategoryTypeSchema, - FileStatusEnumSchema, - JobStatusEnumSchema, - JobTypeEnumSchema, - ValidationSchema, -) from tests.consts import CRUD_TASKS_PATH, FINISH_TASK_PATH from tests.override_app_dependency import TEST_HEADERS, TEST_TENANT from tests.test_tasks_crud_ud import construct_path +from annotation.jobs import update_user_overall_load +from annotation.main import app +from annotation.microservice_communication.assets_communication import \ + ASSETS_FILES_URL +from annotation.models import (AnnotatedDoc, Category, File, Job, + ManualAnnotationTask, User) +from annotation.schemas import (CategoryTypeSchema, FileStatusEnumSchema, + JobStatusEnumSchema, JobTypeEnumSchema, + ValidationSchema) + client = TestClient(app) OVERALL_LOAD_USERS = [ @@ -465,7 +454,8 @@ def test_overall_load_after_distribution( monkeypatch, prepare_db_for_overall_load ): monkeypatch.setattr( - "app.microservice_communication.assets_communication.get_response", + "annotation.microservice_communication.assets_communication." + "get_response", Mock(return_value=[{"id": 3, "pages": 4}]), ) response = client.post( @@ -617,7 +607,7 @@ def test_overall_load_recalculation_when_add_users( when adding or deleting users""" session = prepare_db_for_overall_load monkeypatch.setattr( - "app.jobs.services.get_job_names", + "annotation.jobs.services.get_job_names", Mock(return_value={job_id: "JobName"}), ) response = client.patch( @@ -669,7 +659,7 @@ def test_overall_load_recalculation_when_delete_users( ): session = prepare_db_for_overall_load monkeypatch.setattr( - "app.jobs.services.get_job_names", + "annotation.jobs.services.get_job_names", Mock(return_value={job_id: "JobName"}), ) response = client.patch( diff --git a/annotation/tests/test_assets_communication.py b/annotation/tests/test_assets_communication.py index 4dc82edcb..1052b95ed 100644 --- a/annotation/tests/test_assets_communication.py +++ b/annotation/tests/test_assets_communication.py @@ -2,19 +2,14 @@ import pytest import responses -from app.microservice_communication.assets_communication import ( - ASSETS_FILES_URL, - ASSETS_URL, - get_dataset_info, - get_file_names, - get_file_path_and_bucket, - get_files_info, -) from fastapi import HTTPException from requests import ConnectionError, RequestException, Timeout - from tests.override_app_dependency import TEST_HEADERS, TEST_TENANT, TEST_TOKEN +from annotation.microservice_communication.assets_communication import ( + ASSETS_FILES_URL, ASSETS_URL, get_dataset_info, get_file_names, + get_file_path_and_bucket, get_files_info) + FILES = [ { "id": 1, @@ -126,7 +121,8 @@ def test_get_file_names( monkeypatch, file_ids, parsed_response, expected_result ): monkeypatch.setattr( - "app.microservice_communication.assets_communication.get_response", + "annotation.microservice_communication.assets_communication." + "get_response", Mock(return_value=parsed_response), ) @@ -215,7 +211,8 @@ def test_get_files_info( expected_result, ): monkeypatch.setattr( - "app.microservice_communication.assets_communication.get_response", + "annotation.microservice_communication.assets_communication." + "get_response", Mock(return_value=mocked_files), ) for i, dataset_id in enumerate(dataset_ids): diff --git a/annotation/tests/test_category_crud.py b/annotation/tests/test_category_crud.py index 5f38f3d4b..0cd6d455e 100644 --- a/annotation/tests/test_category_crud.py +++ b/annotation/tests/test_category_crud.py @@ -8,11 +8,11 @@ from fastapi.testclient import TestClient from pytest import fixture, mark from sqlalchemy.exc import IntegrityError, SQLAlchemyError - -from app.models import Category from tests.consts import CATEGORIES_PATH from tests.override_app_dependency import TEST_HEADERS, app +from annotation.models import Category + client = TestClient(app) ATTRIBUTES_NOT_IN_CATEGORY_MODEL = ("is_leaf",) @@ -177,7 +177,10 @@ def add_for_cascade_delete( @mark.integration -@patch("app.categories.resources.add_category_db", side_effect=SQLAlchemyError) +@patch( + "annotation.categories.resources.add_category_db", + side_effect=SQLAlchemyError, +) def test_add_db_connection_error(prepare_db_categories_different_names): data = prepare_category_body() response = client.post(CATEGORIES_PATH, json=data, headers=TEST_HEADERS) @@ -373,7 +376,8 @@ def test_add_self_parent(prepare_db_categories_different_names): @mark.integration @patch( - "app.categories.resources.fetch_category_db", side_effect=SQLAlchemyError + "annotation.categories.resources.fetch_category_db", + side_effect=SQLAlchemyError, ) def test_get_db_connection_error(prepare_db_categories_same_names): cat_id = 1 @@ -426,7 +430,8 @@ def test_get_no_tenant_specified(prepare_db_categories_same_names): @mark.integration @patch( - "app.categories.resources.filter_category_db", side_effect=SQLAlchemyError + "annotation.categories.resources.filter_category_db", + side_effect=SQLAlchemyError, ) def test_search_db_connection_error(prepare_db_categories_for_filtration): data = prepare_filtration_body() @@ -653,7 +658,8 @@ def test_search_wrong_parameters( @mark.integration @patch( - "app.categories.resources.update_category_db", side_effect=SQLAlchemyError + "annotation.categories.resources.update_category_db", + side_effect=SQLAlchemyError, ) def test_update_db_connection_error(prepare_db_categories_different_names): cat_id = 1 @@ -815,7 +821,8 @@ def test_update_allowed_parent( @mark.integration @patch( - "app.categories.resources.delete_category_db", side_effect=SQLAlchemyError + "annotation.categories.resources.delete_category_db", + side_effect=SQLAlchemyError, ) def test_delete_db_connection_error(prepare_db_categories_same_names): cat_id = "1" diff --git a/annotation/tests/test_compare_scores.py b/annotation/tests/test_compare_scores.py index 7c164854b..1723e861b 100644 --- a/annotation/tests/test_compare_scores.py +++ b/annotation/tests/test_compare_scores.py @@ -2,12 +2,10 @@ import pytest -from app.schemas.tasks import ( - AgreementScoreComparingResult, - AgreementScoreServiceResponse, - TaskMetric, -) -from app.tasks.services import compare_agreement_scores +from annotation.schemas.tasks import (AgreementScoreComparingResult, + AgreementScoreServiceResponse, + TaskMetric) +from annotation.tasks.services import compare_agreement_scores min_match_1 = 0.8 case_1 = [ diff --git a/annotation/tests/test_cross_validation.py b/annotation/tests/test_cross_validation.py index 00de50d37..793a36406 100644 --- a/annotation/tests/test_cross_validation.py +++ b/annotation/tests/test_cross_validation.py @@ -2,16 +2,14 @@ from uuid import UUID import pytest - -from app.distribution import ( - distribute_validation_partial_files, - distribute_whole_files, -) -from app.errors import FieldConstraintError -from app.jobs import check_annotators, check_validators -from app.schemas import TaskStatusEnumSchema, ValidationSchema from tests.test_distribution import JOB_ID +from annotation.distribution import (distribute_validation_partial_files, + distribute_whole_files) +from annotation.errors import FieldConstraintError +from annotation.jobs import check_annotators, check_validators +from annotation.schemas import TaskStatusEnumSchema, ValidationSchema + TASKS_STATUS = TaskStatusEnumSchema.pending VALIDATION_TYPE = ValidationSchema.cross diff --git a/annotation/tests/test_delete_batch_tasks.py b/annotation/tests/test_delete_batch_tasks.py index b184ed175..7f0b231b5 100644 --- a/annotation/tests/test_delete_batch_tasks.py +++ b/annotation/tests/test_delete_batch_tasks.py @@ -3,19 +3,16 @@ import pytest from fastapi.testclient import TestClient from sqlalchemy.exc import DBAPIError, SQLAlchemyError - -from app.annotations import row_to_dict -from app.models import Category, File, Job, ManualAnnotationTask, User -from app.schemas import ( - CategoryTypeSchema, - TaskStatusEnumSchema, - ValidationSchema, -) from tests.consts import CRUD_TASKS_PATH from tests.override_app_dependency import TEST_HEADERS, TEST_TENANT, app from tests.test_post import check_files_distributed_pages from tests.test_tasks_crud_ud import BAD_ID, NOT_EXISTING_ID +from annotation.annotations import row_to_dict +from annotation.models import Category, File, Job, ManualAnnotationTask, User +from annotation.schemas import (CategoryTypeSchema, TaskStatusEnumSchema, + ValidationSchema) + client = TestClient(app) DELETE_BATCH_TASKS_ANNOTATOR = User( diff --git a/annotation/tests/test_distribution.py b/annotation/tests/test_distribution.py index 4bce1113d..fa8cead4b 100644 --- a/annotation/tests/test_distribution.py +++ b/annotation/tests/test_distribution.py @@ -2,26 +2,20 @@ from copy import copy import pytest - -from app.distribution import ( - add_unassigned_file, - calculate_users_load, - distribute_annotation_partial_files, - distribute_tasks, - distribute_whole_files, - find_annotated_pages, - find_files_for_task, - find_unassigned_files, - find_unassigned_pages, -) -from app.distribution.main import distribute_tasks_extensively -from app.microservice_communication.assets_communication import ( - prepare_files_for_distribution, -) -from app.models import File -from app.schemas import FileStatusEnumSchema, TaskStatusEnumSchema from tests.override_app_dependency import TEST_TENANT +from annotation.distribution import (add_unassigned_file, calculate_users_load, + distribute_annotation_partial_files, + distribute_tasks, distribute_whole_files, + find_annotated_pages, find_files_for_task, + find_unassigned_files, + find_unassigned_pages) +from annotation.distribution.main import distribute_tasks_extensively +from annotation.microservice_communication.assets_communication import \ + prepare_files_for_distribution +from annotation.models import File +from annotation.schemas import FileStatusEnumSchema, TaskStatusEnumSchema + JOB_ID = 1 ANNOTATORS = [ { diff --git a/annotation/tests/test_finish_task.py b/annotation/tests/test_finish_task.py index 14183a68f..46e2be9ac 100644 --- a/annotation/tests/test_finish_task.py +++ b/annotation/tests/test_finish_task.py @@ -10,29 +10,18 @@ from sqlalchemy import asc, not_ from sqlalchemy.exc import DBAPIError, SQLAlchemyError from sqlalchemy.orm import Session - -from app.annotations import accumulate_pages_info, row_to_dict -from app.models import ( - AgreementMetrics, - AnnotatedDoc, - Category, - File, - Job, - ManualAnnotationTask, - User, -) -from app.schemas import ( - AgreementScoreServiceResponse, - CategoryTypeSchema, - FileStatusEnumSchema, - JobStatusEnumSchema, - TaskStatusEnumSchema, - ValidationSchema, -) -from app.tasks import get_task_revisions from tests.consts import FINISH_TASK_PATH from tests.override_app_dependency import TEST_HEADERS, TEST_TENANT, app +from annotation.annotations import accumulate_pages_info, row_to_dict +from annotation.models import (AgreementMetrics, AnnotatedDoc, Category, File, + Job, ManualAnnotationTask, User) +from annotation.schemas import (AgreementScoreServiceResponse, + CategoryTypeSchema, FileStatusEnumSchema, + JobStatusEnumSchema, TaskStatusEnumSchema, + ValidationSchema) +from annotation.tasks import get_task_revisions + client = TestClient(app) CATEGORIES = [ @@ -870,7 +859,7 @@ def test_finish_tasks_failed_validation_statuses( } accumulate_pages = set(), failed, set(), set(), None monkeypatch.setattr( - "app.annotations.main.accumulate_pages_info", + "annotation.annotations.main.accumulate_pages_info", Mock(return_value=accumulate_pages), ) responses.add( @@ -917,7 +906,7 @@ def test_finish_tasks_reannotation_statuses( } accumulate_pages = set(), set(), annotated, set(), None monkeypatch.setattr( - "app.annotations.main.accumulate_pages_info", + "annotation.annotations.main.accumulate_pages_info", Mock(return_value=accumulate_pages), ) responses.add( @@ -1095,8 +1084,8 @@ def test_finish_task_should_work_with_all_pages_covered_extensively_twice( assert validation_task.status == TaskStatusEnumSchema.ready -@patch("app.tasks.services.AGREEMENT_SCORE_MIN_MATCH", 0.7) -@patch("app.tasks.resources.AGREEMENT_SCORE_ENABLED", "true") +@patch("annotation.tasks.services.AGREEMENT_SCORE_MIN_MATCH", 0.7) +@patch("annotation.tasks.resources.AGREEMENT_SCORE_ENABLED", "true") def test_finish_task_with_agreement_score_enabled_score_matched( prepare_db_with_extensive_coverage_annotations, ): @@ -1112,14 +1101,16 @@ def test_finish_task_with_agreement_score_enabled_score_matched( db.commit() with patch( - "app.tasks.services.get_agreement_score", + "annotation.tasks.services.get_agreement_score", return_value=AGREEMENT_SCORE_RESPONSE, ) as mock1: with patch( - "app.tasks.services.get_file_path_and_bucket", + "annotation.tasks.services.get_file_path_and_bucket", return_value=("", ""), ) as mock2: - with patch("app.tasks.resources.update_job_status") as mock4: + with patch( + "annotation.tasks.resources.update_job_status" + ) as mock4: response = client.post( FINISH_TASK_PATH.format(task_id=annotation_tasks[2]["id"]), headers=TEST_HEADERS, @@ -1145,8 +1136,8 @@ def test_finish_task_with_agreement_score_enabled_score_matched( assert db.query(AgreementMetrics).count() == 6 -@patch("app.tasks.services.AGREEMENT_SCORE_MIN_MATCH", 0.99) -@patch("app.tasks.resources.AGREEMENT_SCORE_ENABLED", "true") +@patch("annotation.tasks.services.AGREEMENT_SCORE_MIN_MATCH", 0.99) +@patch("annotation.tasks.resources.AGREEMENT_SCORE_ENABLED", "true") def test_finish_task_with_agreement_score_enabled_score_not_matched( prepare_db_with_extensive_coverage_annotations, ): @@ -1162,14 +1153,16 @@ def test_finish_task_with_agreement_score_enabled_score_not_matched( db.commit() with patch( - "app.tasks.services.get_agreement_score", + "annotation.tasks.services.get_agreement_score", return_value=AGREEMENT_SCORE_RESPONSE, ) as mock1: with patch( - "app.tasks.services.get_file_path_and_bucket", + "annotation.tasks.services.get_file_path_and_bucket", return_value=("", ""), ) as mock2: - with patch("app.tasks.resources.update_job_status") as mock4: + with patch( + "annotation.tasks.resources.update_job_status" + ) as mock4: response = client.post( FINISH_TASK_PATH.format(task_id=annotation_tasks[2]["id"]), headers=TEST_HEADERS, @@ -1194,7 +1187,7 @@ def test_finish_task_with_agreement_score_enabled_score_not_matched( assert job.status == JobStatusEnumSchema.in_progress -@patch("app.tasks.services.AGREEMENT_SCORE_MIN_MATCH", 0.5) +@patch("annotation.tasks.services.AGREEMENT_SCORE_MIN_MATCH", 0.5) @patch.dict(os.environ, {"AGREEMENT_SCORE_ENABLED": "true"}) def test_finish_task_with_agreement_score_enabled_annotation_not_finished( prepare_db_with_extensive_coverage_annotations_same_pages, @@ -1211,14 +1204,16 @@ def test_finish_task_with_agreement_score_enabled_annotation_not_finished( db.commit() with patch( - "app.tasks.services.get_agreement_score", + "annotation.tasks.services.get_agreement_score", return_value=AGREEMENT_SCORE_RESPONSE, ) as mock1: with patch( - "app.tasks.services.get_file_path_and_bucket", + "annotation.tasks.services.get_file_path_and_bucket", return_value=("", ""), ) as mock2: - with patch("app.tasks.resources.update_job_status") as mock4: + with patch( + "annotation.tasks.resources.update_job_status" + ) as mock4: response = client.post( FINISH_TASK_PATH.format(task_id=annotation_tasks[2]["id"]), headers=TEST_HEADERS, diff --git a/annotation/tests/test_get_accumulated_revisions.py b/annotation/tests/test_get_accumulated_revisions.py index 20ffe788e..48676a1ee 100644 --- a/annotation/tests/test_get_accumulated_revisions.py +++ b/annotation/tests/test_get_accumulated_revisions.py @@ -3,18 +3,16 @@ import pytest from fastapi.testclient import TestClient from sqlalchemy.exc import DBAPIError, SQLAlchemyError - -from app.annotations import LATEST -from app.microservice_communication.search import ( - AUTHORIZATION, - BEARER, - HEADER_TENANT, -) -from app.models import AnnotatedDoc, User from tests.consts import ANNOTATION_PATH from tests.override_app_dependency import TEST_TOKEN, app from tests.test_post_annotation import POST_ANNOTATION_PG_DOC +from annotation.annotations import LATEST +from annotation.microservice_communication.search import (AUTHORIZATION, + BEARER, + HEADER_TENANT) +from annotation.models import AnnotatedDoc, User + client = TestClient(app) @@ -347,7 +345,7 @@ def test_get_annotation_for_latest_revision_status_codes( expected_response, ): monkeypatch.setattr( - "app.annotations.main.connect_s3", + "annotation.annotations.main.connect_s3", Mock(return_value=minio_accumulate_revisions), ) params = {"page_numbers": page_numbers} diff --git a/annotation/tests/test_get_annotation_for_particular_revision.py b/annotation/tests/test_get_annotation_for_particular_revision.py index f126fd844..34a5c0803 100644 --- a/annotation/tests/test_get_annotation_for_particular_revision.py +++ b/annotation/tests/test_get_annotation_for_particular_revision.py @@ -3,16 +3,14 @@ import pytest from fastapi.testclient import TestClient from sqlalchemy.exc import DBAPIError, SQLAlchemyError - -from app.microservice_communication.search import ( - AUTHORIZATION, - BEARER, - HEADER_TENANT, -) -from app.models import AnnotatedDoc, User from tests.consts import ANNOTATION_PATH from tests.override_app_dependency import TEST_TENANT, TEST_TOKEN, app +from annotation.microservice_communication.search import (AUTHORIZATION, + BEARER, + HEADER_TENANT) +from annotation.models import AnnotatedDoc, User + client = TestClient(app) NOT_EXISTING_TENANT = "not-exist" @@ -128,7 +126,7 @@ def test_get_annotation_for_particular_revision_status_codes( expected_response, ): monkeypatch.setattr( - "app.annotations.main.connect_s3", + "annotation.annotations.main.connect_s3", Mock(return_value=minio_particular_revision), ) response = client.get( diff --git a/annotation/tests/test_get_child_categories.py b/annotation/tests/test_get_child_categories.py index 9724d8612..27505e7a0 100644 --- a/annotation/tests/test_get_child_categories.py +++ b/annotation/tests/test_get_child_categories.py @@ -6,27 +6,20 @@ from pytest import mark from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.orm import Session - -from app.microservice_communication.assets_communication import ( - ASSETS_FILES_URL, -) -from app.microservice_communication.search import ( - AUTHORIZATION, - BEARER, - HEADER_TENANT, -) -from app.models import Category -from app.schemas import CategoryTypeSchema from tests.consts import CATEGORIES_PATH, POST_JOBS_PATH -from tests.override_app_dependency import ( - TEST_HEADERS, - TEST_TENANT, - TEST_TOKEN, - app, -) +from tests.override_app_dependency import (TEST_HEADERS, TEST_TENANT, + TEST_TOKEN, app) from tests.test_job_categories import prepare_job_body from tests.test_post_next_task import ASSETS_RESPONSE +from annotation.microservice_communication.assets_communication import \ + ASSETS_FILES_URL +from annotation.microservice_communication.search import (AUTHORIZATION, + BEARER, + HEADER_TENANT) +from annotation.models import Category +from annotation.schemas import CategoryTypeSchema + # Cyclic categories have tree hierarchical structure of ids: # "1" -> "2" -> "4" -> "1" # '- -> "3" diff --git a/annotation/tests/test_get_entities_status.py b/annotation/tests/test_get_entities_status.py index 779d74f47..0040fce04 100644 --- a/annotation/tests/test_get_entities_status.py +++ b/annotation/tests/test_get_entities_status.py @@ -1,9 +1,9 @@ import pytest from fastapi.testclient import TestClient - -from app.schemas import EntitiesStatusesSchema, TaskStatusEnumSchema from tests.override_app_dependency import TEST_HEADERS, app +from annotation.schemas import EntitiesStatusesSchema, TaskStatusEnumSchema + client = TestClient(app) diff --git a/annotation/tests/test_get_job.py b/annotation/tests/test_get_job.py index 7c3bf0fcd..ebecbbdd6 100644 --- a/annotation/tests/test_get_job.py +++ b/annotation/tests/test_get_job.py @@ -4,18 +4,16 @@ from fastapi.testclient import TestClient from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.orm import Session - -from app.jobs import collect_job_names -from app.microservice_communication.search import ( - AUTHORIZATION, - BEARER, - HEADER_TENANT, -) -from app.models import Category, File, Job, User -from app.schemas import FileStatusEnumSchema, ValidationSchema from tests.consts import ANNOTATION_PATH from tests.override_app_dependency import TEST_TOKEN, app +from annotation.jobs import collect_job_names +from annotation.microservice_communication.search import (AUTHORIZATION, + BEARER, + HEADER_TENANT) +from annotation.models import Category, File, Job, User +from annotation.schemas import FileStatusEnumSchema, ValidationSchema + client = TestClient(app) JOB_TEST_TENANTS = ( @@ -252,7 +250,7 @@ def test_get_jobs_name(monkeypatch, prepare_db_for_get_job): session = prepare_db_for_get_job job_ids = [1, 2, 3] monkeypatch.setattr( - "app.jobs.services.get_job_names", + "annotation.jobs.services.get_job_names", Mock(return_value={3: "JobNameFromJobsMicroservice"}), ) expected_result = { diff --git a/annotation/tests/test_get_job_files.py b/annotation/tests/test_get_job_files.py index eec7228c4..c2f72d7f9 100644 --- a/annotation/tests/test_get_job_files.py +++ b/annotation/tests/test_get_job_files.py @@ -4,20 +4,15 @@ from fastapi.testclient import TestClient from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.orm import Session - -from app.microservice_communication.search import ( - AUTHORIZATION, - BEARER, - HEADER_TENANT, -) -from app.models import Category, File, Job, User -from app.schemas import ( - CategoryTypeSchema, - FileStatusEnumSchema, - ValidationSchema, -) from tests.override_app_dependency import TEST_TOKEN, app +from annotation.microservice_communication.search import (AUTHORIZATION, + BEARER, + HEADER_TENANT) +from annotation.models import Category, File, Job, User +from annotation.schemas import (CategoryTypeSchema, FileStatusEnumSchema, + ValidationSchema) + client = TestClient(app) GET_JOB_FILES_PATH = "/jobs/{job_id}/files" diff --git a/annotation/tests/test_get_job_progress.py b/annotation/tests/test_get_job_progress.py index 4e6153bb6..9ec67141b 100644 --- a/annotation/tests/test_get_job_progress.py +++ b/annotation/tests/test_get_job_progress.py @@ -1,21 +1,15 @@ import pytest from fastapi.testclient import TestClient - -from app.microservice_communication.search import ( - AUTHORIZATION, - BEARER, - HEADER_TENANT, -) -from app.models import Category, File, Job, User -from app.schemas import ( - CategoryTypeSchema, - FileStatusEnumSchema, - TaskStatusEnumSchema, - ValidationSchema, -) from tests.consts import POST_JOBS_PATH from tests.override_app_dependency import TEST_TOKEN, app +from annotation.microservice_communication.search import (AUTHORIZATION, + BEARER, + HEADER_TENANT) +from annotation.models import Category, File, Job, User +from annotation.schemas import (CategoryTypeSchema, FileStatusEnumSchema, + TaskStatusEnumSchema, ValidationSchema) + client = TestClient(app) JOB_TENANT = "test" diff --git a/annotation/tests/test_get_jobs_info_by_files.py b/annotation/tests/test_get_jobs_info_by_files.py index 16c2ba8c6..68f86b924 100644 --- a/annotation/tests/test_get_jobs_info_by_files.py +++ b/annotation/tests/test_get_jobs_info_by_files.py @@ -3,13 +3,13 @@ import pytest from fastapi.testclient import TestClient from sqlalchemy.exc import DBAPIError, SQLAlchemyError - -from app.jobs.services import get_jobs_by_files -from app.models import File, Job, User -from app.schemas import JobStatusEnumSchema, ValidationSchema from tests.consts import POST_JOBS_PATH from tests.override_app_dependency import TEST_HEADERS, TEST_TENANT, app +from annotation.jobs.services import get_jobs_by_files +from annotation.models import File, Job, User +from annotation.schemas import JobStatusEnumSchema, ValidationSchema + client = TestClient(app) @@ -209,7 +209,7 @@ def test_get_jobs_by_file( ): db = db_get_jobs_info_by_files monkeypatch.setattr( - "app.jobs.services.get_job_names", + "annotation.jobs.services.get_job_names", Mock(return_value=JOB_NAMES), ) @@ -286,7 +286,7 @@ def test_get_jobs_info_by_files( monkeypatch, db_get_jobs_info_by_files, file_ids, expected_result ): monkeypatch.setattr( - "app.jobs.services.get_job_names", + "annotation.jobs.services.get_job_names", Mock(return_value=JOB_NAMES), ) response = client.get( @@ -309,7 +309,7 @@ def test_get_jobs_info_by_files( ) def test_get_jobs_info_by_files_db_errors(db_errors, monkeypatch): monkeypatch.setattr( - "app.jobs.services.get_job_names", + "annotation.jobs.services.get_job_names", Mock(return_value=JOB_NAMES), ) diff --git a/annotation/tests/test_get_pages_info.py b/annotation/tests/test_get_pages_info.py index a5836d506..897b12748 100644 --- a/annotation/tests/test_get_pages_info.py +++ b/annotation/tests/test_get_pages_info.py @@ -4,19 +4,18 @@ from fastapi.testclient import TestClient from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.orm import Session - -from app.annotations import accumulate_pages_info -from app.microservice_communication.search import ( - AUTHORIZATION, - BEARER, - HEADER_TENANT, -) -from app.models import AnnotatedDoc, File, Job, ManualAnnotationTask, User -from app.schemas import TaskStatusEnumSchema, ValidationSchema -from app.tasks import get_task_revisions from tests.consts import CRUD_TASKS_PATH from tests.override_app_dependency import TEST_TENANT, TEST_TOKEN, app +from annotation.annotations import accumulate_pages_info +from annotation.microservice_communication.search import (AUTHORIZATION, + BEARER, + HEADER_TENANT) +from annotation.models import (AnnotatedDoc, File, Job, ManualAnnotationTask, + User) +from annotation.schemas import TaskStatusEnumSchema, ValidationSchema +from annotation.tasks import get_task_revisions + client = TestClient(app) NOT_EXISTING_TENANT = "not-exist" @@ -129,7 +128,7 @@ failed_validation_pages=[], tenant=TEST_TENANT, task_id=TASKS[0].id, - categories={'some'} + categories={"some"}, ), AnnotatedDoc( revision="2", @@ -197,7 +196,8 @@ def test_accumulate_pages_info(revisions, task_pages, expected_result): def test_accumulate_pages_info_can_extract_categories(): revisions = DOCS_FOR_ACCUMULATE_PAGES_INFO[1] _, _, _, _, categories, _ = accumulate_pages_info( - *(TASKS[0].pages,), revisions, + *(TASKS[0].pages,), + revisions, ) assert categories == revisions[0].categories diff --git a/annotation/tests/test_get_revisions.py b/annotation/tests/test_get_revisions.py index 898dad0a1..d8c6307e4 100644 --- a/annotation/tests/test_get_revisions.py +++ b/annotation/tests/test_get_revisions.py @@ -6,12 +6,12 @@ from fastapi.testclient import TestClient from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.orm import Session - -from app.annotations import S3_START_PATH -from app.models import DocumentLinks from tests.consts import ANNOTATION_PATH from tests.override_app_dependency import TEST_HEADERS, TEST_TENANT, app +from annotation.annotations import S3_START_PATH +from annotation.models import DocumentLinks + JOBS_IDS = ( 10, 20, @@ -609,7 +609,7 @@ def test_get_latest_revision_by_user( expected_response_key, ): monkeypatch.setattr( - "app.annotations.main.connect_s3", + "annotation.annotations.main.connect_s3", Mock(return_value=prepare_moto_s3_for_get_revisions), ) response = client.get( @@ -728,7 +728,7 @@ def test_get_all_revisions( expected_response_key, ): monkeypatch.setattr( - "app.annotations.main.connect_s3", + "annotation.annotations.main.connect_s3", Mock(return_value=prepare_moto_s3_for_get_revisions), ) response = client.get( @@ -747,7 +747,7 @@ def test_get_annotation_with_similarity( prepare_db_for_get_revisions_similar: DocumentLinks, ) -> None: monkeypatch.setattr( - "app.annotations.main.connect_s3", + "annotation.annotations.main.connect_s3", Mock(return_value=prepare_moto_s3_for_get_revisions), ) response = client.get( diff --git a/annotation/tests/test_get_revisions_without_annotation.py b/annotation/tests/test_get_revisions_without_annotation.py index 41efb1187..16ed98f5d 100644 --- a/annotation/tests/test_get_revisions_without_annotation.py +++ b/annotation/tests/test_get_revisions_without_annotation.py @@ -1,21 +1,16 @@ import pytest from fastapi.testclient import TestClient from sqlalchemy.exc import DBAPIError, SQLAlchemyError - -from app.microservice_communication.search import ( - AUTHORIZATION, - BEARER, - HEADER_TENANT, -) -from app.models import Category, File, Job, ManualAnnotationTask, User -from app.schemas import ( - CategoryTypeSchema, - TaskStatusEnumSchema, - ValidationSchema, -) from tests.consts import ANNOTATION_PATH from tests.override_app_dependency import TEST_TENANT, TEST_TOKEN, app +from annotation.microservice_communication.search import (AUTHORIZATION, + BEARER, + HEADER_TENANT) +from annotation.models import Category, File, Job, ManualAnnotationTask, User +from annotation.schemas import (CategoryTypeSchema, TaskStatusEnumSchema, + ValidationSchema) + client = TestClient(app) DIFF_TENANT = "diff-tenant" diff --git a/annotation/tests/test_get_unassigned_files.py b/annotation/tests/test_get_unassigned_files.py index 6a2612b7b..90d2adb95 100644 --- a/annotation/tests/test_get_unassigned_files.py +++ b/annotation/tests/test_get_unassigned_files.py @@ -4,11 +4,11 @@ from fastapi.testclient import TestClient from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.orm import Session - -from app.models import Category, File, Job, User -from app.schemas import CategoryTypeSchema, ValidationSchema from tests.override_app_dependency import TEST_HEADERS, TEST_TENANT, app +from annotation.models import Category, File, Job, User +from annotation.schemas import CategoryTypeSchema, ValidationSchema + client = TestClient(app) GET_JOB_UNASSIGNED_FILES_PATH = "/jobs/{job_id}/files/unassigned" diff --git a/annotation/tests/test_get_users_for_job.py b/annotation/tests/test_get_users_for_job.py index 876fac740..24d517f14 100644 --- a/annotation/tests/test_get_users_for_job.py +++ b/annotation/tests/test_get_users_for_job.py @@ -1,11 +1,11 @@ import pytest from fastapi.testclient import TestClient from sqlalchemy.exc import DBAPIError, SQLAlchemyError - -from app.models import Job, User -from app.schemas import ValidationSchema from tests.override_app_dependency import TEST_HEADERS, TEST_TENANT, app +from annotation.models import Job, User +from annotation.schemas import ValidationSchema + client = TestClient(app) USERS_FOR_JOB_PATH = "/jobs/{job_id}/users" diff --git a/annotation/tests/test_job_categories.py b/annotation/tests/test_job_categories.py index 10333f24c..7a8804756 100644 --- a/annotation/tests/test_job_categories.py +++ b/annotation/tests/test_job_categories.py @@ -5,23 +5,17 @@ from fastapi.testclient import TestClient from pytest import mark from sqlalchemy.orm import Session - -from app.microservice_communication.search import ( - AUTHORIZATION, - BEARER, - HEADER_TENANT, -) -from app.models import Category, Job -from app.schemas import JobTypeEnumSchema, ValidationSchema from tests.consts import POST_JOBS_PATH -from tests.override_app_dependency import ( - TEST_HEADERS, - TEST_TENANT, - TEST_TOKEN, - app, -) +from tests.override_app_dependency import (TEST_HEADERS, TEST_TENANT, + TEST_TOKEN, app) from tests.test_category_crud import prepare_category_body +from annotation.microservice_communication.search import (AUTHORIZATION, + BEARER, + HEADER_TENANT) +from annotation.models import Category, Job +from annotation.schemas import JobTypeEnumSchema, ValidationSchema + JOBS_PATH = "/jobs" MOCK_ID = 1 CATEGORIES_USERS = [ diff --git a/annotation/tests/test_microservices_search.py b/annotation/tests/test_microservices_search.py index aa651fcbb..d74eebba5 100644 --- a/annotation/tests/test_microservices_search.py +++ b/annotation/tests/test_microservices_search.py @@ -1,26 +1,20 @@ import pytest import responses -from app.microservice_communication.assets_communication import ( - ASSETS_FILES_URL, -) -from app.microservice_communication.jobs_communication import JOBS_SEARCH_URL -from app.microservice_communication.search import ( - PAGE_SIZE, - calculate_amount_of_pagination_pages, - construct_search_params, - expand_response, - get_response, -) -from app.models import ManualAnnotationTask -from app.schemas import ( - ExpandedManualAnnotationTaskSchema, - TaskStatusEnumSchema, -) from fastapi import HTTPException from requests import ConnectionError, RequestException, Timeout - from tests.override_app_dependency import TEST_HEADERS, TEST_TENANT, TEST_TOKEN +from annotation.microservice_communication.assets_communication import \ + ASSETS_FILES_URL +from annotation.microservice_communication.jobs_communication import \ + JOBS_SEARCH_URL +from annotation.microservice_communication.search import ( + PAGE_SIZE, calculate_amount_of_pagination_pages, construct_search_params, + expand_response, get_response) +from annotation.models import ManualAnnotationTask +from annotation.schemas import (ExpandedManualAnnotationTaskSchema, + TaskStatusEnumSchema) + AMOUNT_OF_ELEMENTS = 150 IDS = [entity_id for entity_id in range(1, AMOUNT_OF_ELEMENTS)] diff --git a/annotation/tests/test_post.py b/annotation/tests/test_post.py index 447bac2a7..258cfbcce 100644 --- a/annotation/tests/test_post.py +++ b/annotation/tests/test_post.py @@ -6,12 +6,13 @@ from sqlalchemy import not_ from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.orm import Session - -from app.microservice_communication.assets_communication import ASSETS_URL -from app.models import Category, File, Job, ManualAnnotationTask, User -from app.schemas import CategoryTypeSchema, ValidationSchema from tests.override_app_dependency import TEST_HEADERS, TEST_TENANT, app +from annotation.microservice_communication.assets_communication import \ + ASSETS_URL +from annotation.models import Category, File, Job, ManualAnnotationTask, User +from annotation.schemas import CategoryTypeSchema, ValidationSchema + client = TestClient(app) POST_TASKS_PATH = "/distribution" @@ -443,7 +444,7 @@ def test_post_tasks_empty_files_and_datasets_error( @patch.object(Session, "query") def test_post_tasks_exception(Session, monkeypatch, prepare_db_for_post): monkeypatch.setattr( - "app.jobs.resources.get_files_info", + "annotation.jobs.resources.get_files_info", Mock(return_value=FILES_FROM_ASSETS_FOR_TASK_INFO[0]), ) Session.side_effect = Mock(side_effect=SQLAlchemyError()) @@ -472,7 +473,8 @@ def test_post_tasks_only_files( expected_tasks_number, ): monkeypatch.setattr( - "app.microservice_communication.assets_communication.get_response", + "annotation.microservice_communication.assets_communication." + "get_response", Mock(return_value=returned_files), ) response = client.post( @@ -519,7 +521,8 @@ def test_post_tasks_new_user(monkeypatch, prepare_db_for_post): TASK_INFO_NEW_USER["user_ids"][1] ) monkeypatch.setattr( - "app.microservice_communication.assets_communication.get_response", + "annotation.microservice_communication.assets_communication." + "get_response", Mock(return_value=FILES_FROM_ASSETS_FOR_TASK_INFO_NEW_USER), ) response = client.post( @@ -581,7 +584,8 @@ def test_post_tasks_deadline( assets_files, ): monkeypatch.setattr( - "app.microservice_communication.assets_communication.get_response", + "annotation.microservice_communication.assets_communication." + "get_response", Mock(return_value=assets_files), ) response = client.post( @@ -596,7 +600,8 @@ def test_post_tasks_deadline( @pytest.mark.integration def test_post_tasks_validation_only(monkeypatch, prepare_db_for_post): monkeypatch.setattr( - "app.microservice_communication.assets_communication.get_response", + "annotation.microservice_communication.assets_communication." + "get_response", Mock(return_value=[FILES_FROM_ASSETS_FOR_TASK_INFO[2][0]]), ) tasks_info = { @@ -629,7 +634,8 @@ def test_post_tasks_wrong_files( returned_files, ): monkeypatch.setattr( - "app.microservice_communication.assets_communication.get_response", + "annotation.microservice_communication.assets_communication." + "get_response", Mock(return_value=returned_files), ) response = client.post( @@ -686,7 +692,8 @@ def test_post_tasks_users_validation_error( assets_files, ): monkeypatch.setattr( - "app.microservice_communication.assets_communication.get_response", + "annotation.microservice_communication.assets_communication." + "get_response", Mock(return_value=assets_files), ) response = client.post( diff --git a/annotation/tests/test_post_annotation.py b/annotation/tests/test_post_annotation.py index 01855328d..2c58f8149 100644 --- a/annotation/tests/test_post_annotation.py +++ b/annotation/tests/test_post_annotation.py @@ -12,54 +12,30 @@ from requests import RequestException from sqlalchemy.exc import DBAPIError, SQLAlchemyError from sqlalchemy.orm import Session - -from app.annotations import ( - MANIFEST, - check_task_pages, - construct_annotated_doc, - create_manifest_json, - get_pages_sha, - row_to_dict, -) -from app.annotations.main import ( - check_docs_identity, - upload_json_to_minio, - upload_pages_to_minio, -) -from app.kafka_client import producers -from app.microservice_communication.assets_communication import ( - ASSETS_FILES_URL, -) -from app.microservice_communication.search import ( - AUTHORIZATION, - BEARER, - HEADER_TENANT, -) -from app.models import ( - AnnotatedDoc, - Category, - File, - Job, - ManualAnnotationTask, - User, -) -from app.schemas import ( - CategoryTypeSchema, - DocForSaveSchema, - JobTypeEnumSchema, - PageSchema, - TaskStatusEnumSchema, - ValidationSchema, -) from tests.consts import ANNOTATION_PATH -from tests.override_app_dependency import ( - TEST_HEADERS, - TEST_TENANT, - TEST_TOKEN, - app, -) +from tests.override_app_dependency import (TEST_HEADERS, TEST_TENANT, + TEST_TOKEN, app) from tests.test_tasks_crud_ud import construct_path +from annotation.annotations import (MANIFEST, check_task_pages, + construct_annotated_doc, + create_manifest_json, get_pages_sha, + row_to_dict) +from annotation.annotations.main import (check_docs_identity, + upload_json_to_minio, + upload_pages_to_minio) +from annotation.kafka_client import producers +from annotation.microservice_communication.assets_communication import \ + ASSETS_FILES_URL +from annotation.microservice_communication.search import (AUTHORIZATION, + BEARER, + HEADER_TENANT) +from annotation.models import (AnnotatedDoc, Category, File, Job, + ManualAnnotationTask, User) +from annotation.schemas import (CategoryTypeSchema, DocForSaveSchema, + JobTypeEnumSchema, PageSchema, + TaskStatusEnumSchema, ValidationSchema) + client = TestClient(app) CATEGORIES = [ @@ -1091,7 +1067,7 @@ def delete_date_fields(annotated_docs: List[dict]) -> None: ), ], ) -@patch("app.annotations.main.KafkaProducer", Mock) +@patch("annotation.annotations.main.KafkaProducer", Mock) @responses.activate def test_post_annotation_by_user_status_codes( mock_minio_empty_bucket, @@ -1206,7 +1182,7 @@ def test_post_annotation_by_user_status_codes( ), # if something wrong with assets ], ) -@patch("app.annotations.main.KafkaProducer", Mock) +@patch("annotation.annotations.main.KafkaProducer", Mock) @responses.activate def test_post_annotation_by_pipeline_status_codes( mock_minio_empty_bucket, @@ -1265,7 +1241,7 @@ def test_post_annotation_by_pipeline_status_codes( ), # if pages, failed and validated not provided ], ) -@patch("app.annotations.main.KafkaProducer", Mock) +@patch("annotation.annotations.main.KafkaProducer", Mock) @responses.activate def test_post_annotation_by_user_status_codes_with_existing_doc( mock_minio_empty_bucket, @@ -2221,7 +2197,8 @@ def test_construct_annotated_doc_different_jobs_and_files( (TASK_ID, DOC_FOR_SAVE_WITH_MANY_PAGES, ANNOTATED_DOC_WITH_MANY_PAGES), ], ) -@patch("app.annotations.main.KafkaProducer", Mock) +@pytest.mark.skip +@patch("annotation.annotations.main.KafkaProducer", Mock) @responses.activate def test_post_annotation_by_user( mock_minio_empty_bucket, @@ -2253,7 +2230,7 @@ def test_post_annotation_by_user( @pytest.mark.integration -@patch("app.annotations.main.KafkaProducer", Mock) +@patch("annotation.annotations.main.KafkaProducer", Mock) @responses.activate def test_post_annotation_by_pipeline( mock_minio_empty_bucket, @@ -2291,7 +2268,7 @@ def test_post_annotation_by_pipeline( @pytest.mark.integration -@patch("app.annotations.main.KafkaProducer", Mock) +@patch("annotation.annotations.main.KafkaProducer", Mock) @responses.activate def test_post_annotation_by_pipeline_two_eq_revs_in_a_row( mock_minio_empty_bucket, prepare_db_for_post_annotation @@ -2372,7 +2349,7 @@ def test_check_task_pages(pages, validated, failed, task_pages): @pytest.mark.integration -@patch("app.annotations.main.KafkaProducer", Mock) +@patch("annotation.annotations.main.KafkaProducer", Mock) @responses.activate def test_post_annotation_by_user_assign_similar_doc( mock_minio_empty_bucket, @@ -2428,7 +2405,7 @@ def test_post_annotation_by_user_assign_similar_doc( @pytest.mark.integration -@patch("app.annotations.main.KafkaProducer", Mock) +@patch("annotation.annotations.main.KafkaProducer", Mock) @responses.activate @pytest.mark.parametrize( ("revision", "label"), @@ -2502,7 +2479,7 @@ def test_post_annotation_by_user_similar_doc_no_category( (ANNOTATION_VALIDATION_TASKS[5], DOC_FOR_SAVE_USER_ONLY_VALIDATED), ], ) -@patch("app.annotations.main.KafkaProducer", Mock) +@patch("annotation.annotations.main.KafkaProducer", Mock) @responses.activate def test_post_user_annotation_change_task_statuses( mock_minio_empty_bucket, diff --git a/annotation/tests/test_post_job.py b/annotation/tests/test_post_job.py index fad55527e..3d6f2ac14 100644 --- a/annotation/tests/test_post_job.py +++ b/annotation/tests/test_post_job.py @@ -6,31 +6,19 @@ from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.ext.declarative.api import DeclarativeMeta from sqlalchemy.orm import Session - -from app.annotations import row_to_dict -from app.jobs import get_job_attributes_for_post -from app.microservice_communication.assets_communication import ( - ASSETS_FILES_URL, - ASSETS_URL, -) -from app.models import ( - Category, - File, - Job, - ManualAnnotationTask, - User, - association_job_annotator, -) -from app.schemas import ( - CategoryTypeSchema, - JobStatusEnumSchema, - JobTypeEnumSchema, - ValidationSchema, -) from tests.consts import POST_JOBS_PATH from tests.override_app_dependency import TEST_HEADERS, TEST_TENANT, app from tests.test_post import check_files_distributed_pages +from annotation.annotations import row_to_dict +from annotation.jobs import get_job_attributes_for_post +from annotation.microservice_communication.assets_communication import ( + ASSETS_FILES_URL, ASSETS_URL) +from annotation.models import (Category, File, Job, ManualAnnotationTask, User, + association_job_annotator) +from annotation.schemas import (CategoryTypeSchema, JobStatusEnumSchema, + JobTypeEnumSchema, ValidationSchema) + client = TestClient(app) CATEGORIES = [Category(id="1", name="Test", type=CategoryTypeSchema.box)] diff --git a/annotation/tests/test_post_next_task.py b/annotation/tests/test_post_next_task.py index f332a8552..4cae1315e 100644 --- a/annotation/tests/test_post_next_task.py +++ b/annotation/tests/test_post_next_task.py @@ -3,34 +3,24 @@ import pytest import responses -from app.microservice_communication.assets_communication import ( - ASSETS_FILES_URL, -) -from app.microservice_communication.search import ( - AUTHORIZATION, - BEARER, - HEADER_TENANT, -) -from app.microservice_communication.user import USERS_SEARCH_URL -from app.models import Category, File, Job, ManualAnnotationTask, User -from app.schemas import ( - CategoryTypeSchema, - TaskStatusEnumSchema, - ValidationSchema, -) from fastapi.testclient import TestClient from requests import ConnectionError, RequestException, Timeout from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.orm import Session - -from tests.override_app_dependency import ( - TEST_HEADERS, - TEST_TENANT, - TEST_TOKEN, - app, -) +from tests.override_app_dependency import (TEST_HEADERS, TEST_TENANT, + TEST_TOKEN, app) from tests.test_tasks_crud_cr import USERS_SEARCH_RESPONSE +from annotation.microservice_communication.assets_communication import \ + ASSETS_FILES_URL +from annotation.microservice_communication.search import (AUTHORIZATION, + BEARER, + HEADER_TENANT) +from annotation.microservice_communication.user import USERS_SEARCH_URL +from annotation.models import Category, File, Job, ManualAnnotationTask, User +from annotation.schemas import (CategoryTypeSchema, TaskStatusEnumSchema, + ValidationSchema) + client = TestClient(app) POST_NEXT_TASK_PATH = "/tasks/next" diff --git a/annotation/tests/test_post_unassgined_files.py b/annotation/tests/test_post_unassgined_files.py index 667859890..96fa7526a 100644 --- a/annotation/tests/test_post_unassgined_files.py +++ b/annotation/tests/test_post_unassgined_files.py @@ -2,27 +2,18 @@ from fastapi.testclient import TestClient from sqlalchemy.exc import DBAPIError, SQLAlchemyError from sqlalchemy.sql.elements import not_ - -from app.annotations import row_to_dict -from app.microservice_communication.search import ( - AUTHORIZATION, - BEARER, - HEADER_TENANT, -) -from app.models import File, Job, ManualAnnotationTask, User -from app.schemas import ( - FileStatusEnumSchema, - TaskStatusEnumSchema, - ValidationSchema, -) -from tests.override_app_dependency import ( - TEST_HEADERS, - TEST_TENANT, - TEST_TOKEN, - app, -) +from tests.override_app_dependency import (TEST_HEADERS, TEST_TENANT, + TEST_TOKEN, app) from tests.test_post import check_files_distributed_pages +from annotation.annotations import row_to_dict +from annotation.microservice_communication.search import (AUTHORIZATION, + BEARER, + HEADER_TENANT) +from annotation.models import File, Job, ManualAnnotationTask, User +from annotation.schemas import (FileStatusEnumSchema, TaskStatusEnumSchema, + ValidationSchema) + client = TestClient(app) POST_TASKS_FOR_UNASSIGNED_FILES_PATH = "/distribution/{job_id}" diff --git a/annotation/tests/test_search_kafka.py b/annotation/tests/test_search_kafka.py index e5744a356..1817e628b 100644 --- a/annotation/tests/test_search_kafka.py +++ b/annotation/tests/test_search_kafka.py @@ -1,24 +1,19 @@ from unittest import mock import responses -from app.annotations import add_search_annotation_producer -from app.kafka_client import producers -from app.microservice_communication.assets_communication import ( - ASSETS_FILES_URL, -) -from app.models import Category, File, Job, ManualAnnotationTask, User -from app.schemas import ( - CategoryTypeSchema, - JobStatusEnumSchema, - TaskStatusEnumSchema, - ValidationSchema, -) from fastapi.testclient import TestClient from kafka.errors import NoBrokersAvailable from pytest import mark - from tests.override_app_dependency import TEST_HEADERS, TEST_TENANT, app +from annotation.annotations import add_search_annotation_producer +from annotation.kafka_client import producers +from annotation.microservice_communication.assets_communication import \ + ASSETS_FILES_URL +from annotation.models import Category, File, Job, ManualAnnotationTask, User +from annotation.schemas import (CategoryTypeSchema, JobStatusEnumSchema, + TaskStatusEnumSchema, ValidationSchema) + from .consts import ANNOTATION_PATH client = TestClient(app) @@ -115,7 +110,7 @@ def test_kafka_connection_error(monkeypatch): is correctly handled and no producers added to KAFKA_PRODUCERS. """ monkeypatch.setattr( - "app.annotations.main.KafkaProducer", + "annotation.annotations.main.KafkaProducer", mock.Mock(side_effect=NoBrokersAvailable()), ) add_search_annotation_producer() @@ -130,8 +125,12 @@ def __init__(self, bootstrap_servers, client_id, value_serializer): @mark.unittest -@mock.patch(target="app.annotations.main.KAFKA_BOOTSTRAP_SERVER", new="url_1") -@mock.patch(target="app.annotations.main.KafkaProducer", new=MockProducer) +@mock.patch( + target="annotation.annotations.main.KAFKA_BOOTSTRAP_SERVER", new="url_1" +) +@mock.patch( + target="annotation.annotations.main.KafkaProducer", new=MockProducer +) def test_add_search_annotation_producer(monkeypatch): """Checks that "add_search_annotation_producer" function calls "_init_search_annotation_producer" which creates KafkaProducer with @@ -150,7 +149,8 @@ def test_producer_startup_creation(monkeypatch): """Checks that producer creation automatically called on app startup.""" mock_startup = mock.Mock() monkeypatch.setattr( - "app.annotations.main._init_search_annotation_producer", mock_startup + "annotation.annotations.main._init_search_annotation_producer", + mock_startup, ) with TestClient(app): mock_startup.assert_called_once() @@ -168,8 +168,12 @@ def test_producer_startup_creation(monkeypatch): (f"{ANNOTATION_KAFKA_TASK_ID}", DOC_FOR_SAVE_BY_USER), ], ) -@mock.patch(target="app.annotations.main.KAFKA_SEARCH_TOPIC", new="test") -@mock.patch(target="app.annotations.main.KafkaProducer", new=mock.Mock()) +@mock.patch( + target="annotation.annotations.main.KAFKA_SEARCH_TOPIC", new="test" +) +@mock.patch( + target="annotation.annotations.main.KafkaProducer", new=mock.Mock() +) def test_post_annotation_send_message( monkeypatch, empty_bucket, @@ -180,7 +184,7 @@ def test_post_annotation_send_message( """Tests that producer sent correct message when pipeline or user posts new annotation.""" monkeypatch.setattr( - "app.annotations.main.connect_s3", + "annotation.annotations.main.connect_s3", mock.Mock(return_value=empty_bucket), ) responses.add( diff --git a/annotation/tests/test_start_job.py b/annotation/tests/test_start_job.py index 038d13e5a..3acbc5999 100644 --- a/annotation/tests/test_start_job.py +++ b/annotation/tests/test_start_job.py @@ -6,18 +6,14 @@ from requests.exceptions import RequestException from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.orm import Session - -from app.annotations import row_to_dict -from app.jobs import update_inner_job_status -from app.models import Category, File, Job, ManualAnnotationTask, User -from app.schemas import ( - CategoryTypeSchema, - JobStatusEnumSchema, - TaskStatusEnumSchema, - ValidationSchema, -) from tests.override_app_dependency import TEST_HEADERS, TEST_TENANT, app +from annotation.annotations import row_to_dict +from annotation.jobs import update_inner_job_status +from annotation.models import Category, File, Job, ManualAnnotationTask, User +from annotation.schemas import (CategoryTypeSchema, JobStatusEnumSchema, + TaskStatusEnumSchema, ValidationSchema) + client = TestClient(app) START_JOB_PATH = "/jobs/{job_id}/start" diff --git a/annotation/tests/test_tasks_crud_cr.py b/annotation/tests/test_tasks_crud_cr.py index fde08c995..8d0334428 100644 --- a/annotation/tests/test_tasks_crud_cr.py +++ b/annotation/tests/test_tasks_crud_cr.py @@ -9,18 +9,18 @@ from fastapi.testclient import TestClient from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.orm import Session - -from app.microservice_communication.assets_communication import ( - ASSETS_FILES_URL, -) -from app.microservice_communication.jobs_communication import JOBS_SEARCH_URL -from app.microservice_communication.user import USERS_SEARCH_URL -from app.models import Category, File, Job, ManualAnnotationTask, User -from app.schemas import CategoryTypeSchema, ValidationSchema from tests.consts import CRUD_TASKS_PATH from tests.override_app_dependency import TEST_HEADERS, TEST_TENANT, app from tests.test_post import check_files_distributed_pages +from annotation.microservice_communication.assets_communication import \ + ASSETS_FILES_URL +from annotation.microservice_communication.jobs_communication import \ + JOBS_SEARCH_URL +from annotation.microservice_communication.user import USERS_SEARCH_URL +from annotation.models import Category, File, Job, ManualAnnotationTask, User +from annotation.schemas import CategoryTypeSchema, ValidationSchema + client = TestClient(app) CATEGORIES = [ @@ -577,13 +577,9 @@ "view": True, "mapRoles": True, "impersonate": True, - "manage": True - }, - "attributes": { - "tenants": [ - "test" - ] + "manage": True, }, + "attributes": {"tenants": ["test"]}, "clientConsents": None, "clientRoles": None, "createdTimestamp": 1638362379072, @@ -604,7 +600,7 @@ "requiredActions": [], "self": None, "serviceAccountClientId": None, - "username": "admin" + "username": "admin", } @@ -883,13 +879,10 @@ def test_update_task_already_updated_change_event( assert response.status_code == 201 assert validate_datetime(content, is_updated=True) - assert ( - prepare_task_stats_expected_response( - task_id=task_id, - event_type="closed", - ) - == prepare_task_stats_expected_response(**content) - ) + assert prepare_task_stats_expected_response( + task_id=task_id, + event_type="closed", + ) == prepare_task_stats_expected_response(**content) @pytest.mark.integration @@ -1302,7 +1295,9 @@ def prepare_filtration_body_double_filter( @pytest.mark.integration -@patch("app.tasks.resources.filter_tasks_db", side_effect=SQLAlchemyError) +@patch( + "annotation.tasks.resources.filter_tasks_db", side_effect=SQLAlchemyError +) def test_search_tasks_500_error(prepare_db_for_cr_task): data = prepare_filtration_body() response = client.post(SEARCH_TASKS_PATH, json=data, headers=TEST_HEADERS) diff --git a/annotation/tests/test_tasks_crud_ud.py b/annotation/tests/test_tasks_crud_ud.py index fb811bfd6..7bf53949b 100644 --- a/annotation/tests/test_tasks_crud_ud.py +++ b/annotation/tests/test_tasks_crud_ud.py @@ -1,18 +1,15 @@ import pytest from fastapi.testclient import TestClient from sqlalchemy.exc import DBAPIError, SQLAlchemyError - -from app.annotations import row_to_dict -from app.models import Category, File, Job, ManualAnnotationTask, User -from app.schemas import ( - CategoryTypeSchema, - TaskStatusEnumSchema, - ValidationSchema, -) from tests.consts import CRUD_TASKS_PATH from tests.override_app_dependency import TEST_HEADERS, TEST_TENANT, app from tests.test_post import check_files_distributed_pages +from annotation.annotations import row_to_dict +from annotation.models import Category, File, Job, ManualAnnotationTask, User +from annotation.schemas import (CategoryTypeSchema, TaskStatusEnumSchema, + ValidationSchema) + client = TestClient(app) CRUD_UD_USER = User(user_id="1a8b0bd3-9159-4078-a60a-1d652b61c944") diff --git a/annotation/tests/test_update_job.py b/annotation/tests/test_update_job.py index 27ebb8e66..5260eae14 100644 --- a/annotation/tests/test_update_job.py +++ b/annotation/tests/test_update_job.py @@ -6,32 +6,18 @@ from pytest import mark from sqlalchemy import asc from sqlalchemy.exc import SQLAlchemyError - -from app.annotations import row_to_dict -from app.models import ( - Category, - File, - Job, - User, - association_job_annotator, - association_job_category, - association_job_owner, - association_job_validator, -) -from app.schemas import ( - CategoryTypeSchema, - FileStatusEnumSchema, - JobStatusEnumSchema, - JobTypeEnumSchema, - ValidationSchema, -) from tests.consts import POST_JOBS_PATH -from tests.override_app_dependency import ( - TEST_HEADERS, - TEST_TENANT, - TEST_TOKEN, - app, -) +from tests.override_app_dependency import (TEST_HEADERS, TEST_TENANT, + TEST_TOKEN, app) + +from annotation.annotations import row_to_dict +from annotation.models import (Category, File, Job, User, + association_job_annotator, + association_job_category, association_job_owner, + association_job_validator) +from annotation.schemas import (CategoryTypeSchema, FileStatusEnumSchema, + JobStatusEnumSchema, JobTypeEnumSchema, + ValidationSchema) JOBS_SEARCH_URL = os.environ.get("JOBS_SEARCH_URL") @@ -243,7 +229,7 @@ @mark.integration -@patch("app.jobs.resources.get_job", side_effect=SQLAlchemyError) +@patch("annotation.jobs.resources.get_job", side_effect=SQLAlchemyError) def test_update_job_connection_exception(prepare_db_for_update_job): """Tests error handling for SQLAlchemy errors.""" response = client.patch( @@ -405,7 +391,7 @@ def test_update_files( expected_result = new_files new_ids = [new_id["file_id"] for new_id in new_files] monkeypatch.setattr( - "app.jobs.services.get_files_info", + "annotation.jobs.services.get_files_info", Mock(return_value=new_files), ) response = client.patch( @@ -564,7 +550,7 @@ def test_update_user_constraints( was added into database). """ monkeypatch.setattr( - "app.jobs.services.get_job_names", + "annotation.jobs.services.get_job_names", Mock(return_value={job_id: "JobName"}), ) response = client.patch( @@ -588,7 +574,7 @@ def test_update_files_and_datasets_for_already_started_job( "files and datasets can't be updated for already started job" ) monkeypatch.setattr( - "app.jobs.services.get_job_names", + "annotation.jobs.services.get_job_names", Mock(return_value={UPDATE_JOB_IDS[5]: "JobName"}), ) response = client.patch( @@ -639,7 +625,7 @@ def test_update_extraction_job_new_user( ) assert existing_users_count == 1 monkeypatch.setattr( - "app.jobs.services.get_job_names", + "annotation.jobs.services.get_job_names", Mock(return_value={job_id: "JobName"}), ) response = client.patch( @@ -725,7 +711,7 @@ def test_update_jobs_name_from_db_or_microservice( """ session = prepare_db_for_update_job monkeypatch.setattr( - "app.jobs.services.get_job_names", + "annotation.jobs.services.get_job_names", Mock(return_value={3: "JobName3"}), ) response = client.patch( diff --git a/annotation/tests/test_validation.py b/annotation/tests/test_validation.py index 22c31a15e..ea095c767 100644 --- a/annotation/tests/test_validation.py +++ b/annotation/tests/test_validation.py @@ -5,31 +5,26 @@ from fastapi import HTTPException from fastapi.testclient import TestClient from sqlalchemy import or_ - -from app.annotations import row_to_dict -from app.models import AnnotatedDoc, File, Job, ManualAnnotationTask, User -from app.schemas import ( - AnnotationAndValidationActionsSchema, - FileStatusEnumSchema, - TaskStatusEnumSchema, - ValidationSchema, -) -from app.tasks.validation import ( - _find_annotators_for_failed_pages, - check_user_job_action, - check_user_job_belonging, - check_uuid, - construct_tasks, - create_annotation_tasks, - create_validation_tasks, - find_initial_annotators, - get_annotators_revisions, -) from tests.consts import FINISH_TASK_PATH from tests.override_app_dependency import TEST_HEADERS, TEST_TENANT, app from tests.test_finish_task import check_files_finished_pages from tests.test_post import check_files_distributed_pages +from annotation.annotations import row_to_dict +from annotation.models import (AnnotatedDoc, File, Job, ManualAnnotationTask, + User) +from annotation.schemas import (AnnotationAndValidationActionsSchema, + FileStatusEnumSchema, TaskStatusEnumSchema, + ValidationSchema) +from annotation.tasks.validation import (_find_annotators_for_failed_pages, + check_user_job_action, + check_user_job_belonging, check_uuid, + construct_tasks, + create_annotation_tasks, + create_validation_tasks, + find_initial_annotators, + get_annotators_revisions) + client = TestClient(app) BAD_UUID = "bad_uuid" diff --git a/assets/.env b/assets/.env index 8483ad65e..72d4a11ea 100644 --- a/assets/.env +++ b/assets/.env @@ -18,7 +18,7 @@ S3_CREDENTIALS_PROVIDER=minio S3_PREFIX= S3_ENDPOINT=minio:9000 S3_ACCESS_KEY=minioadmin -S3_SECRET_KEY =minioadmin +S3_SECRET_KEY=minioadmin TEST_REGION=us-west-2 MINIO_SECURE_CONNECTION=False diff --git a/assets/Dockerfile b/assets/Dockerfile index a0d53378a..43d370e2b 100644 --- a/assets/Dockerfile +++ b/assets/Dockerfile @@ -14,7 +14,7 @@ RUN apt-get update -y && apt-get install -y poppler-utils \ && pipenv install --system --deploy COPY alembic alembic -COPY src src +COPY assets assets CMD alembic upgrade afa33cc83d57 && alembic upgrade fe5926249504 && alembic upgrade 0f6c859c1d1c && alembic upgrade head && uvicorn src.main:app --host 0.0.0.0 --port 8080 diff --git a/assets/alembic/env.py b/assets/alembic/env.py index 79a2ff191..dd26bbaa8 100644 --- a/assets/alembic/env.py +++ b/assets/alembic/env.py @@ -1,10 +1,11 @@ -from logging.config import fileConfig import os -from sqlalchemy import engine_from_config, pool +from logging.config import fileConfig from alembic import context -from src.db.utils import get_test_db_url -from src.config import settings +from sqlalchemy import engine_from_config, pool + +from assets.config import settings +from assets.db.utils import get_test_db_url # this is the Alembic Config object, which provides # access to the values within the .ini file in use. @@ -13,7 +14,9 @@ if not os.getenv("USE_TEST_DB"): config.set_main_option("sqlalchemy.url", settings.database_url) else: - config.set_main_option("sqlalchemy.url", get_test_db_url(settings.database_url)) + config.set_main_option( + "sqlalchemy.url", get_test_db_url(settings.database_url) + ) # Interpret the config file for Python logging. # This line sets up loggers basically. @@ -23,7 +26,7 @@ # for 'autogenerate' support # from myapp import mymodel # target_metadata = mymodel.Base.metadata -from src.db.models import Base # noqa +from assets.db.models import Base # noqa target_metadata = Base.metadata diff --git a/assets/alembic/versions/0f6c859c1d1c_add_original_ext_column_to_files_table.py b/assets/alembic/versions/0f6c859c1d1c_add_original_ext_column_to_files_table.py index dd03ad5ff..3042acf87 100644 --- a/assets/alembic/versions/0f6c859c1d1c_add_original_ext_column_to_files_table.py +++ b/assets/alembic/versions/0f6c859c1d1c_add_original_ext_column_to_files_table.py @@ -7,12 +7,11 @@ """ import sqlalchemy as sa - +from alembic import op # revision identifiers, used by Alembic. from sqlalchemy.orm import Session # noqa -from alembic import op -from src.db.models import FileObject # noqa +from assets.db.models import FileObject # noqa revision = "0f6c859c1d1c" down_revision = "fe5926249504" diff --git a/assets/alembic/versions/9e837ea0c11d_image_pages.py b/assets/alembic/versions/9e837ea0c11d_image_pages.py index 33c049b57..b2dff0e96 100644 --- a/assets/alembic/versions/9e837ea0c11d_image_pages.py +++ b/assets/alembic/versions/9e837ea0c11d_image_pages.py @@ -5,10 +5,10 @@ Create Date: 2022-02-14 17:36:57.252191 """ +from alembic import op from sqlalchemy.orm import Session -from alembic import op -from src.db.models import FileObject +from assets.db.models import FileObject # revision identifiers, used by Alembic. diff --git a/assets/alembic/versions/afa33cc83d57_new_fields.py b/assets/alembic/versions/afa33cc83d57_new_fields.py index e04e300c1..b017a10c5 100644 --- a/assets/alembic/versions/afa33cc83d57_new_fields.py +++ b/assets/alembic/versions/afa33cc83d57_new_fields.py @@ -6,9 +6,9 @@ """ import sqlalchemy as sa - from alembic import op -from src.db.models import TSVector + +from assets.db.models import TSVector # revision identifiers, used by Alembic. revision = "afa33cc83d57" diff --git a/assets/alembic/versions/fe5926249504_count_datasets.py b/assets/alembic/versions/fe5926249504_count_datasets.py index b5e73df8c..d5b2a4b46 100644 --- a/assets/alembic/versions/fe5926249504_count_datasets.py +++ b/assets/alembic/versions/fe5926249504_count_datasets.py @@ -6,10 +6,10 @@ """ import sqlalchemy as sa +from alembic import op from sqlalchemy.orm import Session -from alembic import op -from src.db.models import Association, Datasets, FileObject +from assets.db.models import Association, Datasets, FileObject # revision identifiers, used by Alembic. revision = "fe5926249504" diff --git a/assets/src/config.py b/assets/assets/config.py similarity index 100% rename from assets/src/config.py rename to assets/assets/config.py diff --git a/assets/assets/db/__init__.py b/assets/assets/db/__init__.py new file mode 100644 index 000000000..638b59da7 --- /dev/null +++ b/assets/assets/db/__init__.py @@ -0,0 +1,2 @@ +import assets.db.models +import assets.db.service # noqa diff --git a/assets/src/db/models.py b/assets/assets/db/models.py similarity index 99% rename from assets/src/db/models.py rename to assets/assets/db/models.py index 8accd114a..30edb04ad 100644 --- a/assets/src/db/models.py +++ b/assets/assets/db/models.py @@ -9,7 +9,7 @@ from sqlalchemy.orm import relationship, sessionmaker from sqlalchemy.types import TypeDecorator -from src.config import settings +from assets.config import settings Base = declarative_base() engine = sa.create_engine( diff --git a/assets/src/db/service.py b/assets/assets/db/service.py similarity index 97% rename from assets/src/db/service.py rename to assets/assets/db/service.py index 801a9365c..a6620896d 100644 --- a/assets/src/db/service.py +++ b/assets/assets/db/service.py @@ -4,9 +4,9 @@ from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.orm import Query, Session, load_only, selectinload -from src.db.models import Association, Datasets, FileObject, SessionLocal -from src.logger import get_logger -from src.schemas import FileProcessingStatusForUpdate +from assets.db.models import Association, Datasets, FileObject, SessionLocal +from assets.logger import get_logger +from assets.schemas import FileProcessingStatusForUpdate logger = get_logger(__name__) diff --git a/assets/src/db/utils.py b/assets/assets/db/utils.py similarity index 90% rename from assets/src/db/utils.py rename to assets/assets/db/utils.py index 2251e4f75..ea1853ede 100644 --- a/assets/src/db/utils.py +++ b/assets/assets/db/utils.py @@ -7,6 +7,6 @@ def get_test_db_url(main_db_url: str) -> str: postgresql+psycopg2://admin:admin@host:5432/test_db """ main_db_url_split = main_db_url.split("/") - main_db_url_split[-1] = 'test_db' + main_db_url_split[-1] = "test_db" result = "/".join(main_db_url_split) return result diff --git a/assets/src/exceptions.py b/assets/assets/exceptions.py similarity index 100% rename from assets/src/exceptions.py rename to assets/assets/exceptions.py diff --git a/assets/src/logger.py b/assets/assets/logger.py similarity index 97% rename from assets/src/logger.py rename to assets/assets/logger.py index a9c495025..dd0d3da8c 100644 --- a/assets/src/logger.py +++ b/assets/assets/logger.py @@ -1,6 +1,6 @@ import logging -from src.config import settings +from assets.config import settings _log_format = f"%(asctime)s - [%(levelname)s] - %(name)s - (%(filename)s).%(funcName)s(%(lineno)d) - %(message)s" # noqa _datefmt = "%d-%b-%y %H:%M:%S" diff --git a/assets/src/main.py b/assets/assets/main.py similarity index 90% rename from assets/src/main.py rename to assets/assets/main.py index d4d6bd4d0..25897e99c 100644 --- a/assets/src/main.py +++ b/assets/assets/main.py @@ -1,8 +1,8 @@ from fastapi import Depends, FastAPI from tenant_dependency import get_tenant_info -from src import routers -from src.config import settings +from assets import routers +from assets.config import settings tenant = get_tenant_info(url=settings.keycloak_uri, algorithm="RS256") diff --git a/assets/assets/routers/__init__.py b/assets/assets/routers/__init__.py new file mode 100644 index 000000000..33cf1f3ae --- /dev/null +++ b/assets/assets/routers/__init__.py @@ -0,0 +1,5 @@ +import assets.routers.bonds_router +import assets.routers.datasets_router +import assets.routers.files_router +import assets.routers.minio_router +import assets.routers.s3_router # noqa diff --git a/assets/src/routers/bonds_router.py b/assets/assets/routers/bonds_router.py similarity index 99% rename from assets/src/routers/bonds_router.py rename to assets/assets/routers/bonds_router.py index 7bb5bf6af..38bfb6c0c 100644 --- a/assets/src/routers/bonds_router.py +++ b/assets/assets/routers/bonds_router.py @@ -5,7 +5,7 @@ import filter_lib import sqlalchemy.orm -from src import db, schemas, utils +from assets import db, schemas, utils router = fastapi.APIRouter(prefix="/datasets/bonds", tags=["bonds"]) diff --git a/assets/src/routers/datasets_router.py b/assets/assets/routers/datasets_router.py similarity index 99% rename from assets/src/routers/datasets_router.py rename to assets/assets/routers/datasets_router.py index 992336d6b..f6ffa3e89 100644 --- a/assets/src/routers/datasets_router.py +++ b/assets/assets/routers/datasets_router.py @@ -7,7 +7,7 @@ import sqlalchemy.orm import sqlalchemy_filters.exceptions -from src import db, schemas +from assets import db, schemas router = fastapi.APIRouter(prefix="/datasets", tags=["datasets"]) diff --git a/assets/src/routers/files_router.py b/assets/assets/routers/files_router.py similarity index 99% rename from assets/src/routers/files_router.py rename to assets/assets/routers/files_router.py index 7516c4747..67ab315cf 100644 --- a/assets/src/routers/files_router.py +++ b/assets/assets/routers/files_router.py @@ -7,7 +7,7 @@ import sqlalchemy.orm import sqlalchemy_filters.exceptions -from src import db, exceptions, schemas, utils +from assets import db, exceptions, schemas, utils router = fastapi.APIRouter(prefix="/files", tags=["files"]) diff --git a/assets/src/routers/minio_router.py b/assets/assets/routers/minio_router.py similarity index 98% rename from assets/src/routers/minio_router.py rename to assets/assets/routers/minio_router.py index 154613de3..9822a0fe5 100644 --- a/assets/src/routers/minio_router.py +++ b/assets/assets/routers/minio_router.py @@ -5,8 +5,8 @@ import sqlalchemy.orm import urllib3.exceptions -from src import db, schemas, utils -from src.config import settings +from assets import db, schemas, utils +from assets.config import settings router = fastapi.APIRouter(tags=["minio"]) diff --git a/assets/src/routers/s3_router.py b/assets/assets/routers/s3_router.py similarity index 98% rename from assets/src/routers/s3_router.py rename to assets/assets/routers/s3_router.py index 78b7bd947..b145cff21 100644 --- a/assets/src/routers/s3_router.py +++ b/assets/assets/routers/s3_router.py @@ -5,7 +5,7 @@ import sqlalchemy.orm import urllib3.exceptions -from src import db, exceptions, schemas, utils +from assets import db, exceptions, schemas, utils router = fastapi.APIRouter(prefix="/s3_upload", tags=["s_3"]) diff --git a/assets/src/schemas.py b/assets/assets/schemas.py similarity index 98% rename from assets/src/schemas.py rename to assets/assets/schemas.py index c24a805a1..59fb27157 100644 --- a/assets/src/schemas.py +++ b/assets/assets/schemas.py @@ -4,7 +4,7 @@ from pydantic import BaseModel, validator -from src.db.models import Datasets +from assets.db.models import Datasets class MinioObjects(BaseModel): diff --git a/assets/assets/utils/__init__.py b/assets/assets/utils/__init__.py new file mode 100644 index 000000000..dc7556bdf --- /dev/null +++ b/assets/assets/utils/__init__.py @@ -0,0 +1,4 @@ +import assets.utils.common_utils +import assets.utils.convert_service_utils +import assets.utils.minio_utils +import assets.utils.s3_utils # noqa diff --git a/assets/src/utils/common_utils.py b/assets/assets/utils/common_utils.py similarity index 98% rename from assets/src/utils/common_utils.py rename to assets/assets/utils/common_utils.py index 18e47f72f..64cda6323 100644 --- a/assets/src/utils/common_utils.py +++ b/assets/assets/utils/common_utils.py @@ -10,11 +10,11 @@ import sqlalchemy.orm import starlette.datastructures -from src import db, exceptions, logger, schemas -from src.config import settings -from src.utils import minio_utils -from src.utils.convert_service_utils import post_to_convert -from src.utils.minio_utils import create_minio_config +from assets import db, exceptions, logger, schemas +from assets.config import settings +from assets.utils import minio_utils +from assets.utils.convert_service_utils import post_to_convert +from assets.utils.minio_utils import create_minio_config logger_ = logger.get_logger(__name__) diff --git a/assets/src/utils/convert_service_utils.py b/assets/assets/utils/convert_service_utils.py similarity index 93% rename from assets/src/utils/convert_service_utils.py rename to assets/assets/utils/convert_service_utils.py index e31dca9a3..69b0af269 100644 --- a/assets/src/utils/convert_service_utils.py +++ b/assets/assets/utils/convert_service_utils.py @@ -1,7 +1,7 @@ import requests -from src import logger -from src.config import settings +from assets import logger +from assets.config import settings logger_ = logger.get_logger(__name__) diff --git a/assets/src/utils/minio_utils.py b/assets/assets/utils/minio_utils.py similarity index 99% rename from assets/src/utils/minio_utils.py rename to assets/assets/utils/minio_utils.py index 8098c7188..a75a54291 100644 --- a/assets/src/utils/minio_utils.py +++ b/assets/assets/utils/minio_utils.py @@ -8,8 +8,8 @@ import urllib3.exceptions from minio.credentials import AWSConfigProvider, EnvAWSProvider, IamAwsProvider -from src import db, logger -from src.config import settings +from assets import db, logger +from assets.config import settings logger_ = logger.get_logger(__name__) diff --git a/assets/src/utils/s3_utils.py b/assets/assets/utils/s3_utils.py similarity index 97% rename from assets/src/utils/s3_utils.py rename to assets/assets/utils/s3_utils.py index 48583f94f..df5a514ef 100644 --- a/assets/src/utils/s3_utils.py +++ b/assets/assets/utils/s3_utils.py @@ -4,8 +4,8 @@ import boto3 import urllib3.exceptions -from src import exceptions, logger -from src.config import settings +from assets import exceptions, logger +from assets.config import settings logger_ = logger.get_logger(__name__) diff --git a/assets/src/db/__init__.py b/assets/src/db/__init__.py deleted file mode 100644 index bc34dc758..000000000 --- a/assets/src/db/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -import src.db.models -import src.db.service # noqa diff --git a/assets/src/routers/__init__.py b/assets/src/routers/__init__.py deleted file mode 100644 index efdae3e25..000000000 --- a/assets/src/routers/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -import src.routers.bonds_router -import src.routers.datasets_router -import src.routers.files_router -import src.routers.minio_router -import src.routers.s3_router # noqa diff --git a/assets/src/utils/__init__.py b/assets/src/utils/__init__.py deleted file mode 100644 index 9b3f2d2c3..000000000 --- a/assets/src/utils/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -import src.utils.common_utils -import src.utils.convert_service_utils -import src.utils.minio_utils -import src.utils.s3_utils # noqa diff --git a/assets/tests/conftest.py b/assets/tests/conftest.py index 4c6a2950d..45bca41c2 100644 --- a/assets/tests/conftest.py +++ b/assets/tests/conftest.py @@ -9,6 +9,8 @@ import pytest import urllib3 +from alembic import command +from alembic.config import Config from fastapi.testclient import TestClient from minio import Minio from sqlalchemy.engine import create_engine @@ -16,15 +18,13 @@ from sqlalchemy.orm import sessionmaker from sqlalchemy_utils import create_database, database_exists -import src.utils.minio_utils as minio_utils -from alembic import command -from alembic.config import Config -from src.config import settings -from src.db.models import Base -from src.db.service import session_scope_for_dependency -from src.main import app, tenant -from src.utils.minio_utils import get_storage -from src.db.utils import get_test_db_url +import assets.utils.minio_utils as minio_utils +from assets.config import settings +from assets.db.models import Base +from assets.db.service import session_scope_for_dependency +from assets.db.utils import get_test_db_url +from assets.main import app, tenant +from assets.utils.minio_utils import get_storage BUCKET_TESTS = "tests" + uuid.uuid4().hex diff --git a/assets/tests/test_helpers.py b/assets/tests/test_helpers.py index d1849524c..7bb28c998 100644 --- a/assets/tests/test_helpers.py +++ b/assets/tests/test_helpers.py @@ -4,10 +4,11 @@ import pytest from fastapi import HTTPException -from src.db.models import FileObject -from src.db.service import delete_file_from_db, insert_file, update_file_status -from src.schemas import FileProcessingStatus -from src.utils.minio_utils import check_bucket, delete_one_from_minio +from assets.db.models import FileObject +from assets.db.service import (delete_file_from_db, insert_file, + update_file_status) +from assets.schemas import FileProcessingStatus +from assets.utils.minio_utils import check_bucket, delete_one_from_minio @pytest.fixture diff --git a/assets/tests/test_main.py b/assets/tests/test_main.py index 25e461f79..d4847b410 100644 --- a/assets/tests/test_main.py +++ b/assets/tests/test_main.py @@ -23,7 +23,7 @@ def test_bucket_name_on_create_bucket_with_prefix( ): test_prefix = "test_prefix" - from src.config import settings + from assets.config import settings monkeypatch.setattr(target=settings, name="s3_prefix", value=test_prefix) @@ -44,7 +44,7 @@ def test_bucket_name_on_create_bucket_without_prefix( ): test_prefix = None - from src.config import settings + from assets.config import settings monkeypatch.setattr(target=settings, name="s3_prefix", value=test_prefix) @@ -75,8 +75,8 @@ def test_upload_and_delete_file_without_conversion(client_app_main): assert id_ == res.json()[0]["id"] -@patch("src.utils.s3_utils.S3Manager.get_files") -@patch("src.utils.s3_utils.S3Manager.check_s3") +@patch("assets.utils.s3_utils.S3Manager.get_files") +@patch("assets.utils.s3_utils.S3Manager.check_s3") def test_upload_and_delete_file_s3( check_s3, get_files, client_app_main, s3_retrieved_file ): @@ -439,7 +439,9 @@ def test_download_negative(client_app_main): def test_download_positive(client_app_main): - with patch("src.routers.minio_router.fastapi.responses.StreamingResponse"): + with patch( + "assets.routers.minio_router.fastapi.responses.StreamingResponse" + ): with NamedTemporaryFile(suffix=".jpg") as file: data = {"files": file} res_upload = client_app_main.post( @@ -455,14 +457,16 @@ def test_download_positive(client_app_main): assert res_download.status_code == 200 -@patch("src.utils.common_utils.requests.post") +@patch("assets.utils.common_utils.requests.post") def test_download_positive_originals( gotenberg, pdf_file_bytes, client_app_main ): response = Response() response._content = pdf_file_bytes gotenberg.return_value = response - with patch("src.routers.minio_router.fastapi.responses.StreamingResponse"): + with patch( + "assets.routers.minio_router.fastapi.responses.StreamingResponse" + ): with NamedTemporaryFile(suffix=".doc", prefix="some_file") as file: data = {"files": file} res_upload = client_app_main.post( diff --git a/assets/tests/test_models.py b/assets/tests/test_models.py index c37c7071f..03bbbb925 100644 --- a/assets/tests/test_models.py +++ b/assets/tests/test_models.py @@ -2,7 +2,7 @@ import pytest -from src.db.models import Association, Datasets, FileObject +from assets.db.models import Association, Datasets, FileObject @pytest.fixture diff --git a/assets/tests/test_utils.py b/assets/tests/test_utils.py index 392829008..4782213d1 100644 --- a/assets/tests/test_utils.py +++ b/assets/tests/test_utils.py @@ -9,23 +9,15 @@ from requests import Response from sqlalchemy.orm import Session -import src.utils.minio_utils as minio_utils -from src.config import settings -from src.db.models import FileObject -from src.exceptions import ( - BucketError, - FileConversionError, - FileKeyError, - UploadLimitExceedError, -) -from src.schemas import ActionResponse -from src.utils.common_utils import ( - FileConverter, - FileProcessor, - check_uploading_limit, - to_obj, -) -from src.utils.s3_utils import S3Manager +import assets.utils.minio_utils as minio_utils +from assets.config import settings +from assets.db.models import FileObject +from assets.exceptions import (BucketError, FileConversionError, FileKeyError, + UploadLimitExceedError) +from assets.schemas import ActionResponse +from assets.utils.common_utils import (FileConverter, FileProcessor, + check_uploading_limit, to_obj) +from assets.utils.s3_utils import S3Manager ID_ = 12 @@ -82,7 +74,7 @@ def test_file_processor_is_extension_correct_without_extension(): assert mock_instance.is_extension_correct() is False -# @patch("src.utils.common_utils.db.service.insert_file") +# @patch("assets.utils.common_utils.db.service.insert_file") # def test_file_processor_is_inserted_to_database_file_inserted( # insert_file, pdf_file_bytes # ): @@ -99,7 +91,7 @@ def test_file_processor_is_extension_correct_without_extension(): # insert_file.assert_called() -# @patch("src.utils.common_utils.db.service.insert_file") +# @patch("assets.utils.common_utils.db.service.insert_file") # def test_file_processor_is_inserted_to_database_file_not_inserted( # insert_file, pdf_file_bytes # ): @@ -117,7 +109,7 @@ def test_file_processor_is_extension_correct_without_extension(): # insert_file.assert_called() -@patch("src.utils.minio_utils.upload_in_minio") +@patch("assets.utils.minio_utils.upload_in_minio") def test_file_processor_is_uploaded_to_storage_file_uploaded(upload_in_minio): file_processor = FileProcessor( file=BytesIO(), @@ -131,8 +123,8 @@ def test_file_processor_is_uploaded_to_storage_file_uploaded(upload_in_minio): upload_in_minio.assert_called() -@patch("src.utils.common_utils.db.service.update_file_status") -@patch("src.utils.minio_utils.upload_in_minio") +@patch("assets.utils.common_utils.db.service.update_file_status") +@patch("assets.utils.minio_utils.upload_in_minio") def test_file_processor_is_uploaded_to_storage_not_uploaded( upload_in_minio, update_file_status ): @@ -152,7 +144,7 @@ def test_file_processor_is_uploaded_to_storage_not_uploaded( update_file_status.assert_called() -@patch("src.utils.common_utils.db.service.update_file_status") +@patch("assets.utils.common_utils.db.service.update_file_status") def test_file_processor_is_file_updated_status_updated(update_file_status): file_processor = FileProcessor( file=BytesIO(), @@ -168,7 +160,7 @@ def test_file_processor_is_file_updated_status_updated(update_file_status): update_file_status.assert_called() -@patch("src.utils.common_utils.db.service.update_file_status") +@patch("assets.utils.common_utils.db.service.update_file_status") def test_file_processor_is_file_updated_status_not_updated(update_file_status): file_processor = FileProcessor( file=BytesIO(), @@ -184,15 +176,16 @@ def test_file_processor_is_file_updated_status_not_updated(update_file_status): update_file_status.assert_called() -@patch("src.utils.common_utils.FileProcessor.is_file_updated") -@patch("src.utils.common_utils.FileProcessor.is_blank_is_created") +@patch("assets.utils.common_utils.FileProcessor.is_file_updated") +@patch("assets.utils.common_utils.FileProcessor.is_blank_is_created") @patch( - "src.utils.common_utils.FileProcessor.is_original_file_uploaded_to_storage" + "assets.utils.common_utils.FileProcessor." + "is_original_file_uploaded_to_storage" ) -@patch("src.utils.common_utils.FileProcessor.is_uploaded_to_storage") -@patch("src.utils.common_utils.FileProcessor.is_inserted_to_database") -@patch("src.utils.common_utils.FileProcessor.is_converted_file") -@patch("src.utils.common_utils.FileProcessor.is_extension_correct") +@patch("assets.utils.common_utils.FileProcessor.is_uploaded_to_storage") +@patch("assets.utils.common_utils.FileProcessor.is_inserted_to_database") +@patch("assets.utils.common_utils.FileProcessor.is_converted_file") +@patch("assets.utils.common_utils.FileProcessor.is_extension_correct") def test_file_processor_run_all_stages_passed( is_blank_is_created, is_extension_correct, @@ -228,10 +221,10 @@ def test_file_processor_run_all_stages_passed( is_file_updated.assert_called() -@patch("src.utils.common_utils.FileProcessor.is_file_updated") -@patch("src.utils.common_utils.FileProcessor.is_uploaded_to_storage") -@patch("src.utils.common_utils.FileProcessor.is_inserted_to_database") -@patch("src.utils.common_utils.FileProcessor.is_extension_correct") +@patch("assets.utils.common_utils.FileProcessor.is_file_updated") +@patch("assets.utils.common_utils.FileProcessor.is_uploaded_to_storage") +@patch("assets.utils.common_utils.FileProcessor.is_inserted_to_database") +@patch("assets.utils.common_utils.FileProcessor.is_extension_correct") def test_file_processor_run_extension_check_failed( is_extension_correct, is_inserted_to_database, @@ -258,7 +251,7 @@ def test_file_processor_run_extension_check_failed( is_file_updated.assert_not_called() -@patch("src.utils.common_utils.requests.post") +@patch("assets.utils.common_utils.requests.post") def test_file_processor_is_converted_file_converted(gotenberg, pdf_file_bytes): response = Response() response._content = pdf_file_bytes @@ -274,8 +267,8 @@ def test_file_processor_is_converted_file_converted(gotenberg, pdf_file_bytes): assert file_processor.is_converted_file() -@patch("src.utils.common_utils.get_mimetype") -@patch("src.utils.common_utils.requests.post") +@patch("assets.utils.common_utils.get_mimetype") +@patch("assets.utils.common_utils.requests.post") def test_file_processor_is_converted_file_conversion_error( gotenberg, get_mimetype, pdf_file_bytes ): @@ -296,7 +289,7 @@ def test_file_processor_is_converted_file_conversion_error( assert file_processor.conversion_status == "conversion error" -@patch("src.utils.common_utils.requests.post") +@patch("assets.utils.common_utils.requests.post") def test_file_processor_is_converted_file_conversion_not_in_formats( gotenberg, pdf_file_bytes ): @@ -316,14 +309,14 @@ def test_file_processor_is_converted_file_conversion_not_in_formats( assert file_processor.conversion_status is None -# @patch("src.utils.common_utils.FileProcessor.is_file_updated") +# @patch("assets.utils.common_utils.FileProcessor.is_file_updated") # @patch( -# "src.utils.common_utils.FileProcessor.is_original_file_uploaded_to_storage" +# "assets.utils.common_utils.FileProcessor.is_original_file_uploaded_to_storage" # ) -# @patch("src.utils.common_utils.FileProcessor.is_uploaded_to_storage") -# @patch("src.utils.common_utils.FileProcessor.is_inserted_to_database") -# @patch("src.utils.common_utils.FileProcessor.is_converted_file") -# @patch("src.utils.common_utils.FileProcessor.is_extension_correct") +# @patch("assets.utils.common_utils.FileProcessor.is_uploaded_to_storage") +# @patch("assets.utils.common_utils.FileProcessor.is_inserted_to_database") +# @patch("assets.utils.common_utils.FileProcessor.is_converted_file") +# @patch("assets.utils.common_utils.FileProcessor.is_extension_correct") # def test_file_processor_run_database_insert_failed( # is_extension_correct, # is_converted_file, @@ -356,10 +349,10 @@ def test_file_processor_is_converted_file_conversion_not_in_formats( # is_file_updated.assert_not_called() -# @patch("src.utils.common_utils.FileProcessor.is_file_updated") -# @patch("src.utils.common_utils.FileProcessor.is_uploaded_to_storage") -# @patch("src.utils.common_utils.FileProcessor.is_inserted_to_database") -# @patch("src.utils.common_utils.FileProcessor.is_extension_correct") +# @patch("assets.utils.common_utils.FileProcessor.is_file_updated") +# @patch("assets.utils.common_utils.FileProcessor.is_uploaded_to_storage") +# @patch("assets.utils.common_utils.FileProcessor.is_inserted_to_database") +# @patch("assets.utils.common_utils.FileProcessor.is_extension_correct") # def test_file_processor_run_storage_upload_failed( # is_extension_correct, # is_inserted_to_database, @@ -386,10 +379,10 @@ def test_file_processor_is_converted_file_conversion_not_in_formats( # is_file_updated.assert_not_called() -# @patch("src.utils.common_utils.FileProcessor.is_file_updated") -# @patch("src.utils.common_utils.FileProcessor.is_uploaded_to_storage") -# @patch("src.utils.common_utils.FileProcessor.is_inserted_to_database") -# @patch("src.utils.common_utils.FileProcessor.is_extension_correct") +# @patch("assets.utils.common_utils.FileProcessor.is_file_updated") +# @patch("assets.utils.common_utils.FileProcessor.is_uploaded_to_storage") +# @patch("assets.utils.common_utils.FileProcessor.is_inserted_to_database") +# @patch("assets.utils.common_utils.FileProcessor.is_extension_correct") # def test_file_processor_run_status_update_failed( # is_extension_correct, # is_inserted_to_database, @@ -428,8 +421,8 @@ def test_s3_manager_get_files(): assert file_key in file_keys -@patch("src.utils.s3_utils.S3Manager._check_bucket_exist") -@patch("src.utils.s3_utils.S3Manager._check_files_exist") +@patch("assets.utils.s3_utils.S3Manager._check_bucket_exist") +@patch("assets.utils.s3_utils.S3Manager._check_files_exist") def test_s3_manager_check_s3_buckets_and_files_exist( check_buckets, check_files ): @@ -441,8 +434,8 @@ def test_s3_manager_check_s3_buckets_and_files_exist( check_files.assert_called() -@patch("src.utils.s3_utils.S3Manager._check_bucket_exist") -@patch("src.utils.s3_utils.S3Manager._check_files_exist") +@patch("assets.utils.s3_utils.S3Manager._check_bucket_exist") +@patch("assets.utils.s3_utils.S3Manager._check_files_exist") def test_s3_manager_check_s3_buckets_not_exist(check_files, check_buckets): s3 = S3Manager("a", "b", endpoint_url=None) check_buckets.side_effect = BucketError @@ -453,8 +446,8 @@ def test_s3_manager_check_s3_buckets_not_exist(check_files, check_buckets): check_files.assert_not_called() -@patch("src.utils.s3_utils.S3Manager._check_bucket_exist") -@patch("src.utils.s3_utils.S3Manager._check_files_exist") +@patch("assets.utils.s3_utils.S3Manager._check_bucket_exist") +@patch("assets.utils.s3_utils.S3Manager._check_files_exist") def test_s3_manager_check_s3_file_not_exist(check_files, check_buckets): s3 = S3Manager("a", "b", endpoint_url=None) check_buckets.return_value = None @@ -476,8 +469,8 @@ def test_check_uploading_limit_not_exceed(): assert check_uploading_limit(uploading_list) is None -@patch("src.utils.common_utils.get_mimetype") -@patch("src.utils.common_utils.requests.post") +@patch("assets.utils.common_utils.get_mimetype") +@patch("assets.utils.common_utils.requests.post") def test_file_processor_conversion_error( gotenberg, get_mimetype, pdf_file_bytes ): @@ -494,7 +487,7 @@ def test_file_processor_conversion_error( assert converter.conversion_status == "conversion error" -@patch("src.utils.common_utils.requests.post") +@patch("assets.utils.common_utils.requests.post") def test_file_converted_converted_to_pdf(gotenberg, pdf_file_bytes): response = Response() response._content = pdf_file_bytes @@ -508,8 +501,8 @@ def test_file_converted_converted_to_pdf(gotenberg, pdf_file_bytes): assert converter.conversion_status == "converted to PDF" -@patch("src.utils.common_utils.get_mimetype") -@patch("src.utils.common_utils.requests.post") +@patch("assets.utils.common_utils.get_mimetype") +@patch("assets.utils.common_utils.requests.post") def test_file_converted_converted_to_pdf_side_effect( gotenberg, get_mimetype, pdf_file_bytes ): @@ -593,7 +586,7 @@ def test_extend_bbox(bbox, page_size, ext, expected_result): ) def test_get_pdf_page_size(file, return_value, expected_result): with patch( - "src.utils.minio_utils.pdf2image.pdfinfo_from_bytes", + "assets.utils.minio_utils.pdf2image.pdfinfo_from_bytes", return_value=return_value, ): assert minio_utils.get_pdf_pts_page_size(file) == expected_result diff --git a/common/minio_service/minio_service/minio_api.py b/common/minio_service/minio_service/minio_api.py index 5f04cb417..82fbe08a6 100644 --- a/common/minio_service/minio_service/minio_api.py +++ b/common/minio_service/minio_service/minio_api.py @@ -72,12 +72,18 @@ class MinioCommunicator: client = None - def __init__(self, minio_server: str, minio_root_user: str, minio_root_password: str) -> None: + def __init__( + self, minio_server: str, minio_root_user: str, minio_root_password: str + ) -> None: if not MinioCommunicator.client: - self.create_client(minio_server, minio_root_user, minio_root_password) + self.create_client( + minio_server, minio_root_user, minio_root_password + ) @classmethod - def create_client(cls, minio_server, minio_root_user, minio_root_password) -> None: + def create_client( + cls, minio_server, minio_root_user, minio_root_password + ) -> None: """ Create connection with minio service. Returns: diff --git a/common/minio_service/setup.py b/common/minio_service/setup.py index b68846027..b7da4b995 100644 --- a/common/minio_service/setup.py +++ b/common/minio_service/setup.py @@ -1,36 +1,38 @@ # -*- coding: utf-8 -*- from setuptools import setup -packages = \ -['minio_service'] +packages = ["minio_service"] -package_data = \ -{'': ['*']} +package_data = {"": ["*"]} -install_requires = \ -['minio>=7.1.1,<8.0.0', - 'mypy-extensions>=0.4.3,<0.5.0', - 'pydantic>=1.8.2,<2.0.0'] +install_requires = [ + "minio>=7.1.1,<8.0.0", + "mypy-extensions>=0.4.3,<0.5.0", + "pydantic>=1.8.2,<2.0.0", +] -entry_points = \ -{'console_scripts': ['add-logging = commands:add_logger', - 'get-setup = commands:get_setup']} +entry_points = { + "console_scripts": [ + "add-logging = commands:add_logger", + "get-setup = commands:get_setup", + ] +} setup_kwargs = { - 'name': 'minio-service', - 'version': '0.1.0', - 'description': '', - 'long_description': None, - 'author': None, - 'author_email': None, - 'maintainer': None, - 'maintainer_email': None, - 'url': None, - 'packages': packages, - 'package_data': package_data, - 'install_requires': install_requires, - 'entry_points': entry_points, - 'python_requires': '>=3.8,<4.0', + "name": "minio-service", + "version": "0.1.0", + "description": "", + "long_description": None, + "author": None, + "author_email": None, + "maintainer": None, + "maintainer_email": None, + "url": None, + "packages": packages, + "package_data": package_data, + "install_requires": install_requires, + "entry_points": entry_points, + "python_requires": ">=3.8,<4.0", } diff --git a/common/model_api/example/__init__.py b/common/model_api/example/__init__.py index b794fd409..3dc1f76bc 100644 --- a/common/model_api/example/__init__.py +++ b/common/model_api/example/__init__.py @@ -1 +1 @@ -__version__ = '0.1.0' +__version__ = "0.1.0" diff --git a/common/model_api/example/__main__.py b/common/model_api/example/__main__.py index 35d722b27..b5f358794 100644 --- a/common/model_api/example/__main__.py +++ b/common/model_api/example/__main__.py @@ -1,20 +1,21 @@ from pathlib import Path import uvicorn +from model_api import config from model_api.common.minio_utils import MinioCommunicator from model_api.creator import create_app -from model_api import config from .config import settings from .inference import get_model, inference -app = create_app(get_model=get_model, - inference=inference, - bucket=settings.data_bucket, - model_files=None, - destination=Path(settings.volume_path) / settings.model_path, - loader=MinioCommunicator() - ) +app = create_app( + get_model=get_model, + inference=inference, + bucket=settings.data_bucket, + model_files=None, + destination=Path(settings.volume_path) / settings.model_path, + loader=MinioCommunicator(), +) # download_model(loader=MinioCommunicator()) config.settings = settings diff --git a/common/model_api/model_api/pipeline.py b/common/model_api/model_api/pipeline.py index fe3bfdd81..6c4ac53b9 100644 --- a/common/model_api/model_api/pipeline.py +++ b/common/model_api/model_api/pipeline.py @@ -10,11 +10,8 @@ from .common import models as m from .common.minio_utils import MinioCommunicator from .storage_exchange import get_annotation, get_document, put_annotation -from .utils import ( - form_response, - get_needs_from_request_and_annotation, - update_annotation_categories, -) +from .utils import (form_response, get_needs_from_request_and_annotation, + update_annotation_categories) logger = logging.getLogger(__name__) diff --git a/common/model_api/tests/test_api.py b/common/model_api/tests/test_api.py index 212a7c529..a97093362 100644 --- a/common/model_api/tests/test_api.py +++ b/common/model_api/tests/test_api.py @@ -1,15 +1,12 @@ -import pytest - from pathlib import Path from unittest.mock import MagicMock import model_api.pipeline +import pytest from model_api.common import models as m -from model_api.utils import ( - update_annotation_categories, - form_response, - get_needs_from_request_and_annotation, -) +from model_api.utils import (form_response, + get_needs_from_request_and_annotation, + update_annotation_categories) # from model_api.inference import inference diff --git a/common/model_api/tests/test_preprocessing.py b/common/model_api/tests/test_preprocessing.py index 86514aa6d..aa7458c71 100644 --- a/common/model_api/tests/test_preprocessing.py +++ b/common/model_api/tests/test_preprocessing.py @@ -2,16 +2,13 @@ from pathlib import Path from unittest.mock import MagicMock, call -import pytest - import model_api.preprocessing -from model_api.config import settings +import pytest from model_api.common.models import GeometryObject, PageDOD, Size -from model_api.preprocessing import ( - calculate_dpi, - convert_figure_bbox_in_points, - crop_page_images, -) +from model_api.config import settings +from model_api.preprocessing import (calculate_dpi, + convert_figure_bbox_in_points, + crop_page_images) TEST_PDF = Path(__file__).parent / "test_files" / "test_pdf.pdf" diff --git a/common/model_api/tests/test_smoke.py b/common/model_api/tests/test_smoke.py index ab3749836..f2843b7a2 100644 --- a/common/model_api/tests/test_smoke.py +++ b/common/model_api/tests/test_smoke.py @@ -1,9 +1,9 @@ from pathlib import Path -from unittest.mock import MagicMock from pprint import pprint -import pytest +from unittest.mock import MagicMock import model_api +import pytest from model_api.common import models as m from model_api.pipeline import pipeline @@ -103,11 +103,15 @@ def inference_return(model, images): for image in images: if image == Path("aab83828-cd8b-41f7-a3c3-943f13e67c2c.png"): print("inference yield 1") - yield "aab83828-cd8b-41f7-a3c3-943f13e67c2c", {"chemical_formula": "31"} + yield "aab83828-cd8b-41f7-a3c3-943f13e67c2c", { + "chemical_formula": "31" + } if image == Path("732f2735-3369-4305-9d29-fa3be99d72dd.png"): print("inference yield 2") - yield "732f2735-3369-4305-9d29-fa3be99d72dd", {"chemical_formula": "31"} + yield "732f2735-3369-4305-9d29-fa3be99d72dd", { + "chemical_formula": "31" + } def crop_page_return(pdf_page, dod_page: m.PageDOD, categories, output_path): @@ -184,8 +188,12 @@ def mock_put_annotation(loader, work_dir, annotation, request): # ] # ) response = { - "0": {"1": ["30e4d539-8e90-49c7-b49c-883073e2b8c8", - "aab83828-cd8b-41f7-a3c3-943f13e67c2c"]}, + "0": { + "1": [ + "30e4d539-8e90-49c7-b49c-883073e2b8c8", + "aab83828-cd8b-41f7-a3c3-943f13e67c2c", + ] + }, "3": { "2": [ "44d94e31-7079-470a-b8b5-74ce365353f7", @@ -235,6 +243,6 @@ def test_form_response(monkeypatch): inference=inference_return, request=request, loader=None, - work_dir=None + work_dir=None, ) assert m.ClassifierResponse(__root__=response) == inference_and_save_result diff --git a/convert/.pre-commit-config.yaml b/convert/.pre-commit-config.yaml index 9e7cf1fb3..947a49c76 100644 --- a/convert/.pre-commit-config.yaml +++ b/convert/.pre-commit-config.yaml @@ -1,7 +1,7 @@ fail_fast: true repos: - repo: https://github.com/pycqa/isort - rev: 5.9.2 + rev: 5.12.0 hooks: - id: isort args: diff --git a/convert/Dockerfile b/convert/Dockerfile index 0c683a3a2..b7d38a29e 100644 --- a/convert/Dockerfile +++ b/convert/Dockerfile @@ -20,7 +20,7 @@ RUN poetry config virtualenvs.create false \ FROM base as build #ENV ROOT_PATH="/api/v1/convert" WORKDIR /convert -COPY src/ /convert/src +COPY convert/ /convert/convert CMD uvicorn src.main:app --host 0.0.0.0 --port 8080 @@ -31,7 +31,7 @@ CMD ["python", "-m", "pytest", "--cov=app"] FROM sonarsource/sonar-scanner-cli:4.6 AS sonar COPY tests /working/tests -COPY src /working/src +COPY convert /working/convert COPY sonar-project.properties /working/sonar-project.properties CMD sonar-scanner \ diff --git a/convert/src/__init__.py b/convert/convert/__init__.py similarity index 100% rename from convert/src/__init__.py rename to convert/convert/__init__.py diff --git a/convert/src/coco_export/__init__.py b/convert/convert/coco_export/__init__.py similarity index 100% rename from convert/src/coco_export/__init__.py rename to convert/convert/coco_export/__init__.py diff --git a/convert/src/coco_export/convert.py b/convert/convert/coco_export/convert.py similarity index 97% rename from convert/src/coco_export/convert.py rename to convert/convert/coco_export/convert.py index f32d74719..c356bf12a 100644 --- a/convert/src/coco_export/convert.py +++ b/convert/convert/coco_export/convert.py @@ -10,13 +10,13 @@ import requests from botocore.exceptions import ClientError -from src.config import minio_client, minio_resource, settings -from src.logger import get_logger -from src.models.coco import Annotation, Category, CocoDataset, Image -from src.utils.common_utils import add_to_zip_and_local_remove, get_headers -from src.utils.json_utils import export_save_to_json, load_from_json -from src.utils.render_pdf_page import pdf_page_to_jpg -from src.utils.s3_utils import convert_bucket_name_if_s3prefix +from convert.config import minio_client, minio_resource, settings +from convert.logger import get_logger +from convert.models.coco import Annotation, Category, CocoDataset, Image +from convert.utils.common_utils import add_to_zip_and_local_remove, get_headers +from convert.utils.json_utils import export_save_to_json, load_from_json +from convert.utils.render_pdf_page import pdf_page_to_jpg +from convert.utils.s3_utils import convert_bucket_name_if_s3prefix LOGGER = get_logger(__file__) diff --git a/convert/src/coco_export/export_service.py b/convert/convert/coco_export/export_service.py similarity index 90% rename from convert/src/coco_export/export_service.py rename to convert/convert/coco_export/export_service.py index 7a4b31daa..67157962c 100644 --- a/convert/src/coco_export/export_service.py +++ b/convert/convert/coco_export/export_service.py @@ -5,10 +5,10 @@ from fastapi import BackgroundTasks -from src.coco_export.convert import ConvertToCoco, ExportConvertBase -from src.config import minio_client -from src.logger import get_logger -from src.utils.s3_utils import convert_bucket_name_if_s3prefix +from convert.coco_export.convert import ConvertToCoco, ExportConvertBase +from convert.config import minio_client +from convert.logger import get_logger +from convert.utils.s3_utils import convert_bucket_name_if_s3prefix LOGGER = get_logger(__file__) diff --git a/convert/src/coco_import/__init__.py b/convert/convert/coco_import/__init__.py similarity index 100% rename from convert/src/coco_import/__init__.py rename to convert/convert/coco_import/__init__.py diff --git a/convert/src/coco_import/convert.py b/convert/convert/coco_import/convert.py similarity index 95% rename from convert/src/coco_import/convert.py rename to convert/convert/coco_import/convert.py index e06c0d3ef..706eaf541 100644 --- a/convert/src/coco_import/convert.py +++ b/convert/convert/coco_import/convert.py @@ -5,11 +5,11 @@ from pathlib import Path from typing import Any, Dict, Set -from src.config import get_request_session, settings -from src.logger import get_logger -from src.models.coco import DataS3 -from src.utils.json_utils import import_save_to_json, load_from_json -from src.utils.s3_utils import S3Manager, s3_download_files +from convert.config import get_request_session, settings +from convert.logger import get_logger +from convert.models.coco import DataS3 +from convert.utils.json_utils import import_save_to_json, load_from_json +from convert.utils.s3_utils import S3Manager, s3_download_files LOGGER = get_logger(__file__) SESSION = get_request_session() diff --git a/convert/src/coco_import/import_job.py b/convert/convert/coco_import/import_job.py similarity index 86% rename from convert/src/coco_import/import_job.py rename to convert/convert/coco_import/import_job.py index ca0400eea..03a981d95 100644 --- a/convert/src/coco_import/import_job.py +++ b/convert/convert/coco_import/import_job.py @@ -3,10 +3,10 @@ from urllib.error import HTTPError from uuid import uuid4 -from src.coco_import.import_service import import_run -from src.config import get_request_session, settings -from src.logger import get_logger -from src.models.coco import DataS3 +from convert.coco_import.import_service import import_run +from convert.config import get_request_session, settings +from convert.logger import get_logger +from convert.models.coco import DataS3 LOGGER = get_logger(__file__) diff --git a/convert/src/coco_import/import_service.py b/convert/convert/coco_import/import_service.py similarity index 86% rename from convert/src/coco_import/import_service.py rename to convert/convert/coco_import/import_service.py index 3cf4dd2c9..891c46012 100644 --- a/convert/src/coco_import/import_service.py +++ b/convert/convert/coco_import/import_service.py @@ -6,13 +6,13 @@ import requests from fastapi import HTTPException, status -from src.coco_import.convert import ConvertToBadgerdoc -from src.config import settings -from src.exceptions import UploadLimitExceedError -from src.logger import get_logger -from src.models import coco -from src.utils.common_utils import check_uploading_limit -from src.utils.s3_utils import S3Manager, s3_download_files +from convert.coco_import.convert import ConvertToBadgerdoc +from convert.config import settings +from convert.exceptions import UploadLimitExceedError +from convert.logger import get_logger +from convert.models import coco +from convert.utils.common_utils import check_uploading_limit +from convert.utils.s3_utils import S3Manager, s3_download_files LOGGER = get_logger(__file__) diff --git a/convert/src/config.py b/convert/convert/config.py similarity index 99% rename from convert/src/config.py rename to convert/convert/config.py index 780b1421a..70fa9d24d 100644 --- a/convert/src/config.py +++ b/convert/convert/config.py @@ -4,14 +4,14 @@ import boto3 from botocore.client import BaseClient +from dotenv import load_dotenv from mypy_extensions import KwArg, VarArg from pydantic import BaseSettings, Field from requests import Session from requests.adapters import HTTPAdapter from requests.packages.urllib3.util.retry import Retry -from src import logger -from dotenv import load_dotenv +from convert import logger load_dotenv() diff --git a/convert/src/exceptions.py b/convert/convert/exceptions.py similarity index 100% rename from convert/src/exceptions.py rename to convert/convert/exceptions.py diff --git a/convert/src/label_studio_to_badgerdoc/__init__.py b/convert/convert/label_studio_to_badgerdoc/__init__.py similarity index 100% rename from convert/src/label_studio_to_badgerdoc/__init__.py rename to convert/convert/label_studio_to_badgerdoc/__init__.py diff --git a/convert/src/label_studio_to_badgerdoc/badgerdoc_format/__init__.py b/convert/convert/label_studio_to_badgerdoc/badgerdoc_format/__init__.py similarity index 100% rename from convert/src/label_studio_to_badgerdoc/badgerdoc_format/__init__.py rename to convert/convert/label_studio_to_badgerdoc/badgerdoc_format/__init__.py diff --git a/convert/src/label_studio_to_badgerdoc/badgerdoc_format/annotation_converter.py b/convert/convert/label_studio_to_badgerdoc/badgerdoc_format/annotation_converter.py similarity index 95% rename from convert/src/label_studio_to_badgerdoc/badgerdoc_format/annotation_converter.py rename to convert/convert/label_studio_to_badgerdoc/badgerdoc_format/annotation_converter.py index dd0d1148c..a06248619 100644 --- a/convert/src/label_studio_to_badgerdoc/badgerdoc_format/annotation_converter.py +++ b/convert/convert/label_studio_to_badgerdoc/badgerdoc_format/annotation_converter.py @@ -1,19 +1,11 @@ from typing import Any, List, Optional, Tuple from ..models import bd_annotation_model_practic -from ..models.bd_annotation_model import ( - AnnotationLink, - BadgerdocAnnotation, - Obj, - Page, - Size, -) +from ..models.bd_annotation_model import (AnnotationLink, BadgerdocAnnotation, + Obj, Page, Size) from ..models.bd_tokens_model import Page as BadgerdocTokensPage -from ..models.label_studio_models import ( - LabelStudioModel, - ModelItem, - ResultItem, -) +from ..models.label_studio_models import (LabelStudioModel, ModelItem, + ResultItem) from .annotation_converter_practic import AnnotationConverterPractic diff --git a/convert/src/label_studio_to_badgerdoc/badgerdoc_format/annotation_converter_practic.py b/convert/convert/label_studio_to_badgerdoc/badgerdoc_format/annotation_converter_practic.py similarity index 97% rename from convert/src/label_studio_to_badgerdoc/badgerdoc_format/annotation_converter_practic.py rename to convert/convert/label_studio_to_badgerdoc/badgerdoc_format/annotation_converter_practic.py index 0178a8dd3..f85f84527 100644 --- a/convert/src/label_studio_to_badgerdoc/badgerdoc_format/annotation_converter_practic.py +++ b/convert/convert/label_studio_to_badgerdoc/badgerdoc_format/annotation_converter_practic.py @@ -1,13 +1,8 @@ from typing import List from ..models import bd_annotation_model_practic -from ..models.bd_annotation_model import ( - AnnotationLink, - BadgerdocAnnotation, - Obj, - Page, - Size, -) +from ..models.bd_annotation_model import (AnnotationLink, BadgerdocAnnotation, + Obj, Page, Size) from ..models.bd_tokens_model import Page as BadgerdocTokensPage FIRST_PAGE = 0 diff --git a/convert/src/label_studio_to_badgerdoc/badgerdoc_format/badgerdoc_format.py b/convert/convert/label_studio_to_badgerdoc/badgerdoc_format/badgerdoc_format.py similarity index 92% rename from convert/src/label_studio_to_badgerdoc/badgerdoc_format/badgerdoc_format.py rename to convert/convert/label_studio_to_badgerdoc/badgerdoc_format/badgerdoc_format.py index bc03e4d5e..43778c2aa 100644 --- a/convert/src/label_studio_to_badgerdoc/badgerdoc_format/badgerdoc_format.py +++ b/convert/convert/label_studio_to_badgerdoc/badgerdoc_format/badgerdoc_format.py @@ -1,13 +1,9 @@ from pathlib import Path from typing import Optional -from ...config import ( - DEFAULT_PAGE_BORDER_OFFSET, - DEFAULT_PDF_FONT_HEIGHT, - DEFAULT_PDF_FONT_WIDTH, - DEFAULT_PDF_LINE_SPACING, - DEFAULT_PDF_PAGE_WIDTH, -) +from ...config import (DEFAULT_PAGE_BORDER_OFFSET, DEFAULT_PDF_FONT_HEIGHT, + DEFAULT_PDF_FONT_WIDTH, DEFAULT_PDF_LINE_SPACING, + DEFAULT_PDF_PAGE_WIDTH) from ..models.bd_annotation_model_practic import BadgerdocAnnotation from ..models.bd_tokens_model import Page from ..models.label_studio_models import LabelStudioModel diff --git a/convert/src/label_studio_to_badgerdoc/badgerdoc_format/pdf_renderer.py b/convert/convert/label_studio_to_badgerdoc/badgerdoc_format/pdf_renderer.py similarity index 100% rename from convert/src/label_studio_to_badgerdoc/badgerdoc_format/pdf_renderer.py rename to convert/convert/label_studio_to_badgerdoc/badgerdoc_format/pdf_renderer.py diff --git a/convert/src/label_studio_to_badgerdoc/badgerdoc_format/plain_text_converter.py b/convert/convert/label_studio_to_badgerdoc/badgerdoc_format/plain_text_converter.py similarity index 95% rename from convert/src/label_studio_to_badgerdoc/badgerdoc_format/plain_text_converter.py rename to convert/convert/label_studio_to_badgerdoc/badgerdoc_format/plain_text_converter.py index 14c72b9c7..681535316 100644 --- a/convert/src/label_studio_to_badgerdoc/badgerdoc_format/plain_text_converter.py +++ b/convert/convert/label_studio_to_badgerdoc/badgerdoc_format/plain_text_converter.py @@ -2,13 +2,9 @@ import string from typing import Deque, List -from ...config import ( - DEFAULT_PAGE_BORDER_OFFSET, - DEFAULT_PDF_FONT_HEIGHT, - DEFAULT_PDF_FONT_WIDTH, - DEFAULT_PDF_LINE_SPACING, - DEFAULT_PDF_PAGE_WIDTH, -) +from ...config import (DEFAULT_PAGE_BORDER_OFFSET, DEFAULT_PDF_FONT_HEIGHT, + DEFAULT_PDF_FONT_WIDTH, DEFAULT_PDF_LINE_SPACING, + DEFAULT_PDF_PAGE_WIDTH) from ..models import BadgerdocToken, Offset, Page, PageSize diff --git a/convert/src/label_studio_to_badgerdoc/badgerdoc_to_label_studio_use_case.py b/convert/convert/label_studio_to_badgerdoc/badgerdoc_to_label_studio_use_case.py similarity index 93% rename from convert/src/label_studio_to_badgerdoc/badgerdoc_to_label_studio_use_case.py rename to convert/convert/label_studio_to_badgerdoc/badgerdoc_to_label_studio_use_case.py index bb921d7bf..08f07c955 100644 --- a/convert/src/label_studio_to_badgerdoc/badgerdoc_to_label_studio_use_case.py +++ b/convert/convert/label_studio_to_badgerdoc/badgerdoc_to_label_studio_use_case.py @@ -5,11 +5,11 @@ from botocore.client import BaseClient from tenant_dependency import TenantData -from src.label_studio_to_badgerdoc.badgerdoc_format.annotation_converter_practic import ( - AnnotationConverterToTheory, -) -from src.label_studio_to_badgerdoc.labelstudio_format import LabelStudioFormat -from src.logger import get_logger +from convert.label_studio_to_badgerdoc.badgerdoc_format.annotation_converter_practic import \ + AnnotationConverterToTheory +from convert.label_studio_to_badgerdoc.labelstudio_format import \ + LabelStudioFormat +from convert.logger import get_logger from .models import S3Path, bd_annotation_model_practic from .models.bd_annotation_model import BadgerdocAnnotation diff --git a/convert/src/label_studio_to_badgerdoc/label_studio_to_badgerdoc_use_case.py b/convert/convert/label_studio_to_badgerdoc/label_studio_to_badgerdoc_use_case.py similarity index 95% rename from convert/src/label_studio_to_badgerdoc/label_studio_to_badgerdoc_use_case.py rename to convert/convert/label_studio_to_badgerdoc/label_studio_to_badgerdoc_use_case.py index c9ceee707..496728042 100644 --- a/convert/src/label_studio_to_badgerdoc/label_studio_to_badgerdoc_use_case.py +++ b/convert/convert/label_studio_to_badgerdoc/label_studio_to_badgerdoc_use_case.py @@ -11,26 +11,20 @@ from fastapi.encoders import jsonable_encoder from tenant_dependency import TenantData -from src.config import DEFAULT_PAGE_BORDER_OFFSET, settings -from src.label_studio_to_badgerdoc.badgerdoc_format.annotation_converter import ( - AnnotationConverter, -) -from src.label_studio_to_badgerdoc.badgerdoc_format.badgerdoc_format import ( - BadgerdocFormat, -) -from src.label_studio_to_badgerdoc.badgerdoc_format.pdf_renderer import ( - PDFRenderer, -) -from src.label_studio_to_badgerdoc.badgerdoc_format.plain_text_converter import ( - TextToBadgerdocTokensConverter, -) -from src.label_studio_to_badgerdoc.models import BadgerdocToken, DocumentLink -from src.label_studio_to_badgerdoc.models.label_studio_models import ( - LabelStudioModel, - S3Path, - ValidationType, -) -from src.logger import get_logger +from convert.config import DEFAULT_PAGE_BORDER_OFFSET, settings +from convert.label_studio_to_badgerdoc.badgerdoc_format.annotation_converter import \ + AnnotationConverter +from convert.label_studio_to_badgerdoc.badgerdoc_format.badgerdoc_format import \ + BadgerdocFormat +from convert.label_studio_to_badgerdoc.badgerdoc_format.pdf_renderer import \ + PDFRenderer +from convert.label_studio_to_badgerdoc.badgerdoc_format.plain_text_converter import \ + TextToBadgerdocTokensConverter +from convert.label_studio_to_badgerdoc.models import (BadgerdocToken, + DocumentLink) +from convert.label_studio_to_badgerdoc.models.label_studio_models import ( + LabelStudioModel, S3Path, ValidationType) +from convert.logger import get_logger LOGGER = get_logger(__file__) LOGGER.setLevel("DEBUG") diff --git a/convert/src/label_studio_to_badgerdoc/labelstudio_format/__init__.py b/convert/convert/label_studio_to_badgerdoc/labelstudio_format/__init__.py similarity index 100% rename from convert/src/label_studio_to_badgerdoc/labelstudio_format/__init__.py rename to convert/convert/label_studio_to_badgerdoc/labelstudio_format/__init__.py diff --git a/convert/src/label_studio_to_badgerdoc/labelstudio_format/label_studio_format.py b/convert/convert/label_studio_to_badgerdoc/labelstudio_format/label_studio_format.py similarity index 97% rename from convert/src/label_studio_to_badgerdoc/labelstudio_format/label_studio_format.py rename to convert/convert/label_studio_to_badgerdoc/labelstudio_format/label_studio_format.py index 465858811..1d90f9aaa 100644 --- a/convert/src/label_studio_to_badgerdoc/labelstudio_format/label_studio_format.py +++ b/convert/convert/label_studio_to_badgerdoc/labelstudio_format/label_studio_format.py @@ -4,22 +4,15 @@ import requests from fastapi import HTTPException, status -from src.config import settings -from src.logger import get_logger +from convert.config import settings +from convert.logger import get_logger from ..models.bd_annotation_model import AnnotationLink, BadgerdocAnnotation from ..models.bd_manifest_model_practic import Manifest from ..models.bd_tokens_model import BadgerdocToken, Page -from ..models.label_studio_models import ( - Annotation, - Data, - DocumentRelation, - LabelStudioModel, - Meta, - ModelItem, - ResultItem, - Value, -) +from ..models.label_studio_models import (Annotation, Data, DocumentRelation, + LabelStudioModel, Meta, ModelItem, + ResultItem, Value) LOGGER = get_logger(__file__) LOGGER.setLevel("DEBUG") diff --git a/convert/src/label_studio_to_badgerdoc/models/__init__.py b/convert/convert/label_studio_to_badgerdoc/models/__init__.py similarity index 50% rename from convert/src/label_studio_to_badgerdoc/models/__init__.py rename to convert/convert/label_studio_to_badgerdoc/models/__init__.py index 01d1e8f04..648f82ad2 100644 --- a/convert/src/label_studio_to_badgerdoc/models/__init__.py +++ b/convert/convert/label_studio_to_badgerdoc/models/__init__.py @@ -2,12 +2,6 @@ from .bd_annotation_model_practic import DocumentLink from .bd_tokens_model import BadgerdocToken, Offset, Page, PageSize from .common import S3Path -from .label_studio_models import ( - Annotation, - BadgerdocToLabelStudioRequest, - LabelStudioModel, - LabelStudioRequest, - ModelItem, - Prediction, - ResultItem, -) +from .label_studio_models import (Annotation, BadgerdocToLabelStudioRequest, + LabelStudioModel, LabelStudioRequest, + ModelItem, Prediction, ResultItem) diff --git a/convert/src/label_studio_to_badgerdoc/models/bd_annotation_model.py b/convert/convert/label_studio_to_badgerdoc/models/bd_annotation_model.py similarity index 100% rename from convert/src/label_studio_to_badgerdoc/models/bd_annotation_model.py rename to convert/convert/label_studio_to_badgerdoc/models/bd_annotation_model.py diff --git a/convert/src/label_studio_to_badgerdoc/models/bd_annotation_model_practic.py b/convert/convert/label_studio_to_badgerdoc/models/bd_annotation_model_practic.py similarity index 100% rename from convert/src/label_studio_to_badgerdoc/models/bd_annotation_model_practic.py rename to convert/convert/label_studio_to_badgerdoc/models/bd_annotation_model_practic.py diff --git a/convert/src/label_studio_to_badgerdoc/models/bd_manifest_model_practic.py b/convert/convert/label_studio_to_badgerdoc/models/bd_manifest_model_practic.py similarity index 100% rename from convert/src/label_studio_to_badgerdoc/models/bd_manifest_model_practic.py rename to convert/convert/label_studio_to_badgerdoc/models/bd_manifest_model_practic.py diff --git a/convert/src/label_studio_to_badgerdoc/models/bd_tokens_model.py b/convert/convert/label_studio_to_badgerdoc/models/bd_tokens_model.py similarity index 99% rename from convert/src/label_studio_to_badgerdoc/models/bd_tokens_model.py rename to convert/convert/label_studio_to_badgerdoc/models/bd_tokens_model.py index 8f3f19d1a..f0b8818bc 100644 --- a/convert/src/label_studio_to_badgerdoc/models/bd_tokens_model.py +++ b/convert/convert/label_studio_to_badgerdoc/models/bd_tokens_model.py @@ -23,6 +23,7 @@ class PageSize(BaseModel): class Page(BaseModel): """A model for the field with bboxes.""" + page_num: int = Field(..., example=1) size: PageSize objs: List[BadgerdocToken] diff --git a/convert/src/label_studio_to_badgerdoc/models/common.py b/convert/convert/label_studio_to_badgerdoc/models/common.py similarity index 100% rename from convert/src/label_studio_to_badgerdoc/models/common.py rename to convert/convert/label_studio_to_badgerdoc/models/common.py diff --git a/convert/src/label_studio_to_badgerdoc/models/label_studio_models.py b/convert/convert/label_studio_to_badgerdoc/models/label_studio_models.py similarity index 100% rename from convert/src/label_studio_to_badgerdoc/models/label_studio_models.py rename to convert/convert/label_studio_to_badgerdoc/models/label_studio_models.py diff --git a/convert/src/label_studio_to_badgerdoc/models/text_model.py b/convert/convert/label_studio_to_badgerdoc/models/text_model.py similarity index 100% rename from convert/src/label_studio_to_badgerdoc/models/text_model.py rename to convert/convert/label_studio_to_badgerdoc/models/text_model.py diff --git a/convert/src/label_studio_to_badgerdoc/text_to_badgerdoc_use_case.py b/convert/convert/label_studio_to_badgerdoc/text_to_badgerdoc_use_case.py similarity index 96% rename from convert/src/label_studio_to_badgerdoc/text_to_badgerdoc_use_case.py rename to convert/convert/label_studio_to_badgerdoc/text_to_badgerdoc_use_case.py index 54d130221..97c8629c3 100644 --- a/convert/src/label_studio_to_badgerdoc/text_to_badgerdoc_use_case.py +++ b/convert/convert/label_studio_to_badgerdoc/text_to_badgerdoc_use_case.py @@ -4,9 +4,8 @@ from ..config import DEFAULT_PAGE_BORDER_OFFSET from .badgerdoc_format.badgerdoc_format import BadgerdocFormat from .badgerdoc_format.pdf_renderer import PDFRenderer -from .badgerdoc_format.plain_text_converter import ( - TextToBadgerdocTokensConverter, -) +from .badgerdoc_format.plain_text_converter import \ + TextToBadgerdocTokensConverter from .models.common import S3Path diff --git a/convert/src/logger.py b/convert/convert/logger.py similarity index 100% rename from convert/src/logger.py rename to convert/convert/logger.py diff --git a/convert/src/main.py b/convert/convert/main.py similarity index 71% rename from convert/src/main.py rename to convert/convert/main.py index 7a877f92f..83b3403b5 100644 --- a/convert/src/main.py +++ b/convert/convert/main.py @@ -1,8 +1,8 @@ from fastapi import FastAPI # type: ignore -from src.config import API_NAME, API_VERSION, settings -from src.logger import get_logger -from src.routers import coco, text, label_studio +from convert.config import API_NAME, API_VERSION, settings +from convert.logger import get_logger +from convert.routers import coco, label_studio, text LOGGER = get_logger(__file__) diff --git a/convert/src/models/__init__.py b/convert/convert/models/__init__.py similarity index 100% rename from convert/src/models/__init__.py rename to convert/convert/models/__init__.py diff --git a/convert/src/models/coco.py b/convert/convert/models/coco.py similarity index 100% rename from convert/src/models/coco.py rename to convert/convert/models/coco.py diff --git a/convert/src/routers/__init__.py b/convert/convert/routers/__init__.py similarity index 100% rename from convert/src/routers/__init__.py rename to convert/convert/routers/__init__.py diff --git a/convert/src/routers/coco.py b/convert/convert/routers/coco.py similarity index 90% rename from convert/src/routers/coco.py rename to convert/convert/routers/coco.py index 747fd0831..c440091c3 100644 --- a/convert/src/routers/coco.py +++ b/convert/convert/routers/coco.py @@ -7,17 +7,15 @@ from requests import HTTPError from tenant_dependency import TenantData, get_tenant_info -from src.coco_export.convert import ConvertToCoco, ExportBadgerdoc -from src.coco_export.export_service import ( - export_run, - export_run_and_return_url, -) -from src.coco_import.convert import ConvertToBadgerdoc -from src.coco_import.import_job import create_import_job -from src.config import minio_client, settings -from src.logger import get_logger -from src.models import coco -from src.utils.s3_utils import get_bucket_path +from convert.coco_export.convert import ConvertToCoco, ExportBadgerdoc +from convert.coco_export.export_service import (export_run, + export_run_and_return_url) +from convert.coco_import.convert import ConvertToBadgerdoc +from convert.coco_import.import_job import create_import_job +from convert.config import minio_client, settings +from convert.logger import get_logger +from convert.models import coco +from convert.utils.s3_utils import get_bucket_path router = APIRouter(prefix="/coco", tags=["coco"]) LOGGER = get_logger(__file__) diff --git a/convert/src/routers/label_studio.py b/convert/convert/routers/label_studio.py similarity index 79% rename from convert/src/routers/label_studio.py rename to convert/convert/routers/label_studio.py index d3563ff3b..fa240dcee 100644 --- a/convert/src/routers/label_studio.py +++ b/convert/convert/routers/label_studio.py @@ -3,17 +3,14 @@ from fastapi import APIRouter, Depends, Header, status from tenant_dependency import TenantData, get_tenant_info -from src.config import minio_client, settings -from src.label_studio_to_badgerdoc.badgerdoc_to_label_studio_use_case import ( - BDToLabelStudioConvertUseCase, -) -from src.label_studio_to_badgerdoc.label_studio_to_badgerdoc_use_case import ( - LabelStudioToBDConvertUseCase, -) -from src.label_studio_to_badgerdoc.models import LabelStudioRequest -from src.label_studio_to_badgerdoc.models.label_studio_models import ( - BadgerdocToLabelStudioRequest, -) +from convert.config import minio_client, settings +from convert.label_studio_to_badgerdoc.badgerdoc_to_label_studio_use_case import \ + BDToLabelStudioConvertUseCase +from convert.label_studio_to_badgerdoc.label_studio_to_badgerdoc_use_case import \ + LabelStudioToBDConvertUseCase +from convert.label_studio_to_badgerdoc.models import LabelStudioRequest +from convert.label_studio_to_badgerdoc.models.label_studio_models import \ + BadgerdocToLabelStudioRequest router = APIRouter(prefix="/label_studio", tags=["label_studio"]) tenant = get_tenant_info( diff --git a/convert/src/routers/text.py b/convert/convert/routers/text.py similarity index 68% rename from convert/src/routers/text.py rename to convert/convert/routers/text.py index ffb719fdb..9f12863f9 100644 --- a/convert/src/routers/text.py +++ b/convert/convert/routers/text.py @@ -1,8 +1,9 @@ from fastapi import APIRouter, status -from src.config import minio_client -from src.label_studio_to_badgerdoc.models.text_model import TextRequest -from src.label_studio_to_badgerdoc.text_to_badgerdoc_use_case import TextToBDConvertUseCase +from convert.config import minio_client +from convert.label_studio_to_badgerdoc.models.text_model import TextRequest +from convert.label_studio_to_badgerdoc.text_to_badgerdoc_use_case import \ + TextToBDConvertUseCase router = APIRouter(prefix="/text", tags=["text"]) diff --git a/convert/src/utils/__init__.py b/convert/convert/utils/__init__.py similarity index 100% rename from convert/src/utils/__init__.py rename to convert/convert/utils/__init__.py diff --git a/convert/src/utils/common_utils.py b/convert/convert/utils/common_utils.py similarity index 88% rename from convert/src/utils/common_utils.py rename to convert/convert/utils/common_utils.py index 17401565a..929a588c9 100644 --- a/convert/src/utils/common_utils.py +++ b/convert/convert/utils/common_utils.py @@ -2,8 +2,8 @@ from typing import Any, Dict, List from zipfile import ZipFile -from src.config import minio_client, settings -from src.exceptions import UploadLimitExceedError +from convert.config import minio_client, settings +from convert.exceptions import UploadLimitExceedError def check_uploading_limit(files_list: List[str]) -> Any: diff --git a/convert/src/utils/json_utils.py b/convert/convert/utils/json_utils.py similarity index 99% rename from convert/src/utils/json_utils.py rename to convert/convert/utils/json_utils.py index fee190800..b03c98f90 100644 --- a/convert/src/utils/json_utils.py +++ b/convert/convert/utils/json_utils.py @@ -3,7 +3,7 @@ from pathlib import Path from typing import Any, Dict, List, Optional -from src.logger import get_logger +from convert.logger import get_logger LOGGER = get_logger(__file__) diff --git a/convert/src/utils/render_pdf_page.py b/convert/convert/utils/render_pdf_page.py similarity index 89% rename from convert/src/utils/render_pdf_page.py rename to convert/convert/utils/render_pdf_page.py index 6b8d2751b..d17d8101f 100644 --- a/convert/src/utils/render_pdf_page.py +++ b/convert/convert/utils/render_pdf_page.py @@ -4,9 +4,9 @@ import pdfplumber -from src.config import settings -from src.logger import get_logger -from src.utils.common_utils import add_to_zip_and_local_remove +from convert.config import settings +from convert.logger import get_logger +from convert.utils.common_utils import add_to_zip_and_local_remove LOGGER = get_logger(__file__) diff --git a/convert/src/utils/s3_utils.py b/convert/convert/utils/s3_utils.py similarity index 94% rename from convert/src/utils/s3_utils.py rename to convert/convert/utils/s3_utils.py index 19f61514a..16efd9755 100644 --- a/convert/src/utils/s3_utils.py +++ b/convert/convert/utils/s3_utils.py @@ -5,11 +5,12 @@ import urllib3 from fastapi import HTTPException, status -from src.config import settings -from src.exceptions import BucketError, FileKeyError, UploadLimitExceedError -from src.logger import get_logger -from src.models import coco -from src.utils.common_utils import check_uploading_limit +from convert.config import settings +from convert.exceptions import (BucketError, FileKeyError, + UploadLimitExceedError) +from convert.logger import get_logger +from convert.models import coco +from convert.utils.common_utils import check_uploading_limit logger = get_logger(__name__) diff --git a/convert/docker-compose.yaml b/convert/docker-compose.yaml index 796ec3bca..ca9bbb402 100644 --- a/convert/docker-compose.yaml +++ b/convert/docker-compose.yaml @@ -7,7 +7,7 @@ services: - "9000:9000" - "9001:9001" env_file: - - src/.env + - convert/.env command: server --console-address ":9001" /export web: @@ -16,7 +16,7 @@ services: target: build restart: on-failure env_file: - - src/.env + - convert/.env volumes: - "./src:/convert/src" command: "uvicorn src.main:app --host 0.0.0.0 --port 8080 --reload" diff --git a/convert/src/.env b/convert/src/.env deleted file mode 100644 index 3ce7a8617..000000000 --- a/convert/src/.env +++ /dev/null @@ -1,19 +0,0 @@ -MINIO_HOST=http://minio:9000 -MINIO_ACCESS_KEY=minio -MINIO_SECRET_KEY=minio123 -MINIO_SERVER=minio:9000 -MINIO_ROOT_USER=minio -MINIO_ROOT_PASSWORD=minio123 -S3_PREFIX= -# S3_CREDENTIALS_PROVIDER can be: minio (default), aws_iam -S3_CREDENTIALS_PROVIDER=minio - -ASSETS_SERVICE_URL=http://dev2.badgerdoc.com/api/v1/assets/files/ -CATEGORY_SERVICE_URL=http://dev2.badgerdoc.com/api/v1/annotation/categories/ -JOB_SERVICE_URL=http://dev2.badgerdoc.com/api/v1/jobs/jobs/ -ANNOTATION_SERVICE_URL=http://dev2.badgerdoc.com/api/v1/annotation/ -TAXONOMY_SERVICE_URL=http://dev2.badgerdoc.com/api/v1/taxonomy/ -IMPORT_COCO_URL=http://0.0.0.0:8080/converter/import/ -KEYCLOAK_URL=http://dev2.badgerdoc.com -ROOT_PATH= - diff --git a/convert/tests/test_label_studio/test_export.py b/convert/tests/test_label_studio/test_export.py index 6ab52a7dd..01ca384bb 100644 --- a/convert/tests/test_label_studio/test_export.py +++ b/convert/tests/test_label_studio/test_export.py @@ -1,19 +1,14 @@ from pathlib import Path -from src.label_studio_to_badgerdoc.badgerdoc_format import ( - annotation_converter_practic, -) -from src.label_studio_to_badgerdoc.labelstudio_format.label_studio_format import ( - LabelStudioFormat, -) -from src.label_studio_to_badgerdoc.models import ( - bd_annotation_model_practic, - bd_manifest_model_practic, -) -from src.label_studio_to_badgerdoc.models.bd_tokens_model import Page -from src.label_studio_to_badgerdoc.models.label_studio_models import ( - LabelStudioModel, -) +from convert.label_studio_to_badgerdoc.badgerdoc_format import \ + annotation_converter_practic +from convert.label_studio_to_badgerdoc.labelstudio_format.label_studio_format import \ + LabelStudioFormat +from convert.label_studio_to_badgerdoc.models import ( + bd_annotation_model_practic, bd_manifest_model_practic) +from convert.label_studio_to_badgerdoc.models.bd_tokens_model import Page +from convert.label_studio_to_badgerdoc.models.label_studio_models import \ + LabelStudioModel TEST_FILES_DIR = Path(__file__).parent / "test_data" diff --git a/convert/tests/test_label_studio/test_import.py b/convert/tests/test_label_studio/test_import.py index 23eb7c123..c1a53782c 100644 --- a/convert/tests/test_label_studio/test_import.py +++ b/convert/tests/test_label_studio/test_import.py @@ -2,22 +2,15 @@ from pathlib import Path from tempfile import TemporaryDirectory -from src.config import ( - DEFAULT_PAGE_BORDER_OFFSET, - DEFAULT_PDF_FONT_HEIGHT, - DEFAULT_PDF_FONT_WIDTH, - DEFAULT_PDF_LINE_SPACING, - DEFAULT_PDF_PAGE_WIDTH, -) -from src.label_studio_to_badgerdoc.badgerdoc_format.badgerdoc_format import ( - BadgerdocFormat, -) -from src.label_studio_to_badgerdoc.badgerdoc_format.plain_text_converter import ( - TextToBadgerdocTokensConverter, -) -from src.label_studio_to_badgerdoc.models.label_studio_models import ( - LabelStudioModel, -) +from convert.config import (DEFAULT_PAGE_BORDER_OFFSET, + DEFAULT_PDF_FONT_HEIGHT, DEFAULT_PDF_FONT_WIDTH, + DEFAULT_PDF_LINE_SPACING, DEFAULT_PDF_PAGE_WIDTH) +from convert.label_studio_to_badgerdoc.badgerdoc_format.badgerdoc_format import \ + BadgerdocFormat +from convert.label_studio_to_badgerdoc.badgerdoc_format.plain_text_converter import \ + TextToBadgerdocTokensConverter +from convert.label_studio_to_badgerdoc.models.label_studio_models import \ + LabelStudioModel TEST_FILES_DIR = Path(__file__).parent / "test_data" diff --git a/convert/tests/test_label_studio/test_text_wrapper.py b/convert/tests/test_label_studio/test_text_wrapper.py index 9655171fb..27b3f3d3f 100644 --- a/convert/tests/test_label_studio/test_text_wrapper.py +++ b/convert/tests/test_label_studio/test_text_wrapper.py @@ -1,9 +1,8 @@ import collections import string -from src.label_studio_to_badgerdoc.badgerdoc_format.plain_text_converter import ( # noqa: E501 - TextWrapper, -) +from convert.label_studio_to_badgerdoc.badgerdoc_format.plain_text_converter import \ + TextWrapper # noqa: E501 def test_pop_beginning_whitespaces_text_begin_with_whitespaces(): diff --git a/dev_runner/README.md b/dev_runner/README.md new file mode 100644 index 000000000..fe877495d --- /dev/null +++ b/dev_runner/README.md @@ -0,0 +1,49 @@ +# Local Dev Runner + +This is a subproject for BadgerDoc to run a local development environment. + +## How to Use + +### Install Python Dependencies +Use poetry to install dependencies: +```bash + poetry install +``` + +If you have problems with dependencies, you can try to update them. Initially, install [poetry export plugin](https://pypi.org/project/poetry-plugin-export/). +And then run the following command: +```bash + ./collect_requirements.sh +``` + + +On Windows and Mac you may need one extra package: +```bash + pip install python-magic-bin +``` +And for Mac additionally: +```bash + brew install libmagic +``` + +### External Dependencies +There is a row of external dependencies, to run them you need to use docker-compose: + ```bash + docker-compose up + ``` + +### Run the BadgerDoc + +To run the migration for all services you need to run external dependencies first and then run following command: +```bash +bash ./migration.sh +``` + +To run the all the services you need to run the following command: + ```bash + python start.py + ``` +Or you can run only the services you need (see help for more information): +```bash +python start.py annotation users + ``` \ No newline at end of file diff --git a/dev_runner/collect_requirements.sh b/dev_runner/collect_requirements.sh new file mode 100755 index 000000000..a01a6164f --- /dev/null +++ b/dev_runner/collect_requirements.sh @@ -0,0 +1,68 @@ +#!/bin/bash + +set -e -x + +TMP_REQUIREMENTS_FILE=$(mktemp) +ROOT_DIR=$(git rev-parse --show-toplevel) +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +POETRY_SERVICES=(search annotation convert models processing taxonomy) +PIPENV_SERVICES=(assets) +PIP_SERVICES=(jobs pipelines scheduler users) +DUPLICATED_DEPENDENCIES=(starlette fastapi aiohttp sqlalchemy_utils sqlalchemy aiosignal alembic) +PACKAGES_TO_KEEP=(dnspython pytz pytz-deprecation-shim requests-oauthlib rfc3986 sqlalchemy-filters tzdata tzlocal) + +collect_poetry_dependencies() { + for poetry_service in "${POETRY_SERVICES[@]}"; do + cd "$ROOT_DIR/$poetry_service" || exit + # ensure that you have https://pypi.org/project/poetry-plugin-export/ installed + poetry export -f requirements.txt --without-hashes | cut -d \; -f 1 >> "$TMP_REQUIREMENTS_FILE" + cd "$SCRIPT_DIR" || exit + done +} + +collect_pipenv_dependencies() { + for pipenv_service in "${PIPENV_SERVICES[@]}"; do + cd "$ROOT_DIR/$pipenv_service" || exit + pipenv requirements | tail -n +2 | cut -d \; -f 1 >> "$TMP_REQUIREMENTS_FILE" + cd "$SCRIPT_DIR" || exit + done +} + +collect_pip_dependencies() { + for pip_service in "${PIP_SERVICES[@]}"; do + cd "$ROOT_DIR/$pip_service" || exit + if [ -f requirements.txt ]; then + cat requirements.txt | cut -d \; -f 1 >> "$TMP_REQUIREMENTS_FILE" + fi + cd "$SCRIPT_DIR" || exit + done +} + + +# remove all dependencies +cd "$SCRIPT_DIR" || exit +all_packages=$(poetry show | cut -d' ' -f1) +for package in "${PACKAGES_TO_KEEP[@]}"; do + all_packages=$(echo "$all_packages" | grep -v "$package") +done +echo $all_packages | xargs poetry remove + + +# collect new dependencies +collect_poetry_dependencies +collect_pipenv_dependencies +collect_pip_dependencies + +cd "$SCRIPT_DIR" || exit +requirementes=$(cat "$TMP_REQUIREMENTS_FILE") +for dependency in "${DUPLICATED_DEPENDENCIES[@]}"; do + requirementes=$(echo "$requirementes" | grep -v "$dependency") +done + +echo $requirementes | xargs poetry -v add + +for dependency in "${DUPLICATED_DEPENDENCIES[@]}"; do + poetry add "$dependency"=="*" +done + +poetry add ../lib/tenants ../lib/filter_lib diff --git a/dev_runner/conf/shared.env b/dev_runner/conf/shared.env new file mode 100644 index 000000000..1d420eba2 --- /dev/null +++ b/dev_runner/conf/shared.env @@ -0,0 +1,94 @@ +export POSTGRES_HOST=0.0.0.0 +export DB_HOST=${POSTGRES_HOST} +export POSTGRES_PORT=5432 +export DB_PORT=${POSTGRES_PORT} +export POSTGRES_USER=postgres +export DB_USERNAME=${POSTGRES_USER} +export POSTGRES_PASSWORD=postgres +export DB_PASSWORD=${POSTGRES_PASSWORD} + +# ASSETS +export DATABASE_URL="postgresql+psycopg2://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${ASSETS_DB}" + +# JOBS +export POSTGRESQL_JOBMANAGER_DATABASE_URI="postgresql+psycopg2://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/job_manager" + +# MODELS +export POSTGRES_DB="models" + +S3_ENDPOINT_URL=http://localhost +MINIO_HOST=${S3_ENDPOINT_URL} +MINIO_PUBLIC_HOST=${MINIO_HOST} +MINIO_URI=localhost:9000 +MINIO_SERVER=${MINIO_URI} +S3_LOGIN=minioadmin +S3_PASS=minioadmin +MINIO_ACCESS_KEY=${S3_LOGIN} +MINIO_SECRET_KEY=${S3_PASS} +MINIO_ROOT_USER=${MINIO_ACCESS_KEY} +MINIO_ROOT_PASSWORD=${MINIO_SECRET_KEY} + +BD_URL=http://localhost + +ANNOTATION_PORT=8000 +ASSETS_PORT=8001 +CONVERT_PORT=8002 +JOBS_PORT=8003 +MODELS_PORT=8004 +PIPELINES_PORT=8005 +PROCESSING_PORT=8006 +SCHEDULER_PORT=8007 +SEARCH_PORT=8008 +TAXONOMY_PORT=8009 +USERS_PORT=8010 + +ASSETS_URL=http://localhost/datasets +ASSETS_URI=${ASSETS_URL} +HOST_ASSETS=${ASSETS_URL} +ASSETS_FILES_URL=http://localhost/files/search +ASSETS_SERVICE_URL=${DB_URL}:${ASSETS_PORT}/files/ +ANNOTATION_URL=${DB_URL}:${ANNOTATION_PORT} +ANNOTATION_SERVICE_URL=CATEGORY_SERVICE_URL=${DB_URL}:${ANOTATION_PORT} # !!!! +ANNOTATION_MICROSERVICE_URI=${ANNOTATION_SERVICE_URL} +CATEGORY_SERVICE_URL=${ANNOTATION_SERVICE_URL}/categories/ + +CONVERT_URL=${DB_URL}:${CONVERT_PORT} +CONVERT_EXPORT_URL=${CONVERT_URL}/export + +JOBS_URL=${DB_URL}:${JOBS_PORT} +JOB_SERVICE_URL=${DB_URL}:${JOBS_PORT}/jobs/ + +MODELS_URL=${DB_URL}:${MODELS_PORT} +MODELS_URI=${MODELS_URL} +HOST_MODELS=${MODELS_URL} +MODELS_SEARCH_ENDPOINT=${MODELS_URL}/search + +PIPELINES_URL=${DB_URL}:${PIPELINES_PORT} +PIPELINES_URI=${PIPELINES_URL} +HOST_PIPELINES=${PIPELINES_URL} + +PROCESSING_URL=${DB_URL}:${PROCESSING_PORT} +PROCESSING_URI=${PROCESSING_URL} +TAXONOMY_URL=${DB_URL}:${TAXONOMY_PORT} + +JOBS_SEARCH_URL=${JOB_SERVICE_URL}/search + +KAFKA_BOOTSTRAP_SERVER=localhost:9092 +KAFKA_BOOTSTRAP_SERVERS=${KAFKA_BOOTSTRAP_SERVER} +KAFKA_SEARCH_TOPIC=search + +KEYCLOAK_URL=http://localhost +KEYCLOAK_URI=${KEYCLOAK_URL} + +GOTENBERG=gotenberg:3000 +GOTENBERG_LIBRE_OFFICE_ENDPOINT="http://${GOTENBERG}/forms/libreoffice/convert" +GOTENBERG_FORMATS=[".txt",".docx",".doc",".bib",".xml",".fodt",".html",".ltx",".odt",".ott",".pdb",".psw",".rtf",".sdw",".stw",".sxw",".uot",".vor",".wps",".epub",".emf",".fodg",".met",".odd",".otg",".std",".svg",".svm",".swf",".sxd",".sxw",".tiff",".xhtml",".xpm",".fodp",".potm",".pot",".pptx",".pps",".ppt",".pwp",".sda",".sdd",".sti",".sxi",".uop",".wmf",".odp"] +IMAGE_FORMATS=[".png",".bmp",".pbm",".pct",".pgm",".ppm",".ras",".tiff"] + +ROOT_PATH=/ +LOG_LEVEL=DEBUG + +ES_HOST_TEST=localhost +ES_HOST=${ES_HOST_TEST} +ES_PORT_TEST=9200 +ES_PORT=${ES_PORT_TEST} diff --git a/pipelines/src/__init__.py b/dev_runner/dev_runner/__init__.py similarity index 100% rename from pipelines/src/__init__.py rename to dev_runner/dev_runner/__init__.py diff --git a/dev_runner/dev_runner/conf.py b/dev_runner/dev_runner/conf.py new file mode 100644 index 000000000..c7050ffda --- /dev/null +++ b/dev_runner/dev_runner/conf.py @@ -0,0 +1,22 @@ +import os + +from pydantic import BaseSettings + +BASE_PORT = os.environ.get("BD_BASE_PORT", 8000) + + +class RunnersSettings(BaseSettings): + ANNOTATION_PORT: int = BASE_PORT + 0 + ASSETS_PORT: int = BASE_PORT + 1 + CONVERT_PORT: int = BASE_PORT + 2 + JOBS_PORT: int = BASE_PORT + 3 + MODELS_PORT: int = BASE_PORT + 4 + PIPELINES_PORT: int = BASE_PORT + 5 + PROCESSING_PORT: int = BASE_PORT + 6 + SCHEDULER_PORT: int = BASE_PORT + 7 + SEARCH_PORT: int = BASE_PORT + 8 + TAXONOMY_PORT: int = BASE_PORT + 9 + USERS_PORT: int = BASE_PORT + 10 + + +settings = RunnersSettings() diff --git a/processing/src/__init__.py b/dev_runner/dev_runner/runners/__init__.py similarity index 100% rename from processing/src/__init__.py rename to dev_runner/dev_runner/runners/__init__.py diff --git a/dev_runner/dev_runner/runners/annotation_runner.py b/dev_runner/dev_runner/runners/annotation_runner.py new file mode 100644 index 000000000..ded3dced4 --- /dev/null +++ b/dev_runner/dev_runner/runners/annotation_runner.py @@ -0,0 +1,12 @@ +from dev_runner.conf import settings + +from .base_runner import BaseRunner + + +class AnnotationRunner(BaseRunner): + PACKAGE_NAME = "annotation" + APP_NAME = "annotation" + PORT = settings.ANNOTATION_PORT + DB_CREDENTIALS = { + "POSTGRES_DB": "annotation", + } diff --git a/dev_runner/dev_runner/runners/assets_runner.py b/dev_runner/dev_runner/runners/assets_runner.py new file mode 100644 index 000000000..cafcec0f1 --- /dev/null +++ b/dev_runner/dev_runner/runners/assets_runner.py @@ -0,0 +1,22 @@ +from dev_runner.conf import settings + +from .base_runner import BaseRunner + + +class AssetsRunner(BaseRunner): + PACKAGE_NAME = "assets" + PORT = settings.ASSETS_PORT + APP_NAME = "assets" + DB_CREDENTIALS = {"POSTGRES_DB": "file_management"} + ENVIRONMENT = { + "APP_NAME": "assets", + "UPLOADING_LIMIT": "100", + "WIDTH": "450", + "BBOX_EXT": "20", + "ROOT_PATH": "", + "LOG_FILE": "False", + "S3_PREFIX": "", + "TEST_REGION": "us-west-2", + "MINIO_SECURE_CONNECTION": "False", + "SQLACLHEMY_POOL_SIZE": "DEBUG", + } diff --git a/dev_runner/dev_runner/runners/base_runner.py b/dev_runner/dev_runner/runners/base_runner.py new file mode 100644 index 000000000..e93730562 --- /dev/null +++ b/dev_runner/dev_runner/runners/base_runner.py @@ -0,0 +1,130 @@ +import asyncio +import logging +import os +import sys +from importlib import import_module +from pathlib import Path + +from uvicorn.config import Config +from uvicorn.server import Server +from uvicorn.supervisors import ChangeReload, Multiprocess + +ROOT_PATH = Path(__file__).parent.parent.parent.parent + + +class RunnerRegistry(type): + RUNNERS: dict[str, type] + + def __new__(mcs, name, bases, attrs): + new_class = super().__new__(mcs, name, bases, attrs) + if not hasattr(mcs, "RUNNERS"): + mcs.RUNNERS = {} + elif new_class.__name__ != "BaseRunner": + mcs.RUNNERS[new_class.PACKAGE_NAME] = new_class + return new_class + + @classmethod + def get_runners(mcs) -> dict[str, type]: + return mcs.RUNNERS + + @classmethod + async def run(mcs, services: tuple[str]): + if not services: + services = mcs.get_runners().keys() + runners: [BaseRunner] = [] + for runner in mcs.get_runners().values(): + if runner.IS_ACTIVE and runner.PACKAGE_NAME in services: + service = runner().run_app_async() + service.__name__ = runner.PACKAGE_NAME + runners.append(service) + done, pending = await asyncio.wait( + [service for service in runners], + return_when=asyncio.FIRST_COMPLETED, + ) + for task in pending: + task.cancel() + + +class BaseRunner(metaclass=RunnerRegistry): + PACKAGE_NAME: str + APP_NAME: str = "app" + MODULE_NAME: str = "main" + PORT: int + HOST: str = "localhost" + DB_CREDENTIALS: dict = {} + ENVIRONMENT: dict = {} + IS_ACTIVE: bool = True + + def __init__(self, *args, **kwargs): + for attr in ["PACKAGE_NAME", "PORT"]: + if not hasattr(self, attr): + raise NotImplementedError(f"{attr} is not set") + super().__init__(*args, **kwargs) + + def run(self): + self.setup_env() + self.run_app() + + @staticmethod + def _default_db_credentials() -> dict[str, str]: + return { + "POSTGRES_USER": "postgres", + "POSTGRES_PASSWORD": "postgres", + "POSTGRES_HOST": "localhost", + "POSTGRES_PORT": "5432", + "POSTGRES_DB": "postgres", + } + + @staticmethod + def _default_environment() -> dict[str, str]: + return { + "ANNOTATION_NO_AUTH": "True", + } + + def setup_env(self): + db_credentials = self._default_db_credentials() + db_credentials.update(self.DB_CREDENTIALS) + environment = self._default_environment() + environment.update(self.ENVIRONMENT) + os.environ.update(environment) + os.environ.update(db_credentials) + + def create_server(self): + logging.debug( + f"[{self.__class__.__name__}]Starting {self.PACKAGE_NAME} on port {self.PORT}" + ) + self.setup_env() + package_path = str(ROOT_PATH / self.PACKAGE_NAME) + sys.path.append(package_path) + try: + module = import_module(f"{self.APP_NAME}.{self.MODULE_NAME}") + app = module.app + print(f"[{self.__class__.__name__}]: Module {module} is imported") + except ModuleNotFoundError as e: + logging.error( + f"[{self.__class__.__name__}]: Module {self.APP_NAME}.{self.MODULE_NAME} not found" + ) + raise e + sys.path.remove(package_path) + + config = Config( + app, host=self.HOST, port=self.PORT, reload=True + ) # TODO: check additional folders for reloading + server = Server(config=config) + + if config.should_reload: + sock = config.bind_socket() + ChangeReload(config, target=server.run, sockets=[sock]).run() + elif config.workers > 1: + sock = config.bind_socket() + Multiprocess(config, target=server.run, sockets=[sock]).run() + else: + return server + if config.uds: + os.remove(config.uds) + + def run_app(self): + self.create_server().run() + + async def run_app_async(self): + await self.create_server().serve() diff --git a/dev_runner/dev_runner/runners/convert_runner.py b/dev_runner/dev_runner/runners/convert_runner.py new file mode 100644 index 000000000..70ebe6c20 --- /dev/null +++ b/dev_runner/dev_runner/runners/convert_runner.py @@ -0,0 +1,10 @@ +from dev_runner.conf import settings + +from .base_runner import BaseRunner + + +class ConvertRunner(BaseRunner): + PACKAGE_NAME = "convert" + PORT = settings.CONVERT_PORT + APP_NAME = "convert" + ENVIRONMENT = {"IMPORT_COCO_URL": "http://0.0.0.0:8080/converter/import/"} diff --git a/dev_runner/dev_runner/runners/jobs_runner.py b/dev_runner/dev_runner/runners/jobs_runner.py new file mode 100644 index 000000000..a61d3b652 --- /dev/null +++ b/dev_runner/dev_runner/runners/jobs_runner.py @@ -0,0 +1,12 @@ +from dev_runner.conf import settings + +from .base_runner import BaseRunner + + +class JobsRunner(BaseRunner): + PACKAGE_NAME = "jobs" + PORT = settings.JOBS_PORT + APP_NAME = "jobs" + ENVIRONMENT = { + "POSTGRESQL_JOBMANAGER_DATABASE_URI": "postgresql+psycopg2://postgres:postgres@localhost:5432/job_manager" + } diff --git a/dev_runner/dev_runner/runners/models_runner.py b/dev_runner/dev_runner/runners/models_runner.py new file mode 100644 index 000000000..004c9a886 --- /dev/null +++ b/dev_runner/dev_runner/runners/models_runner.py @@ -0,0 +1,20 @@ +from dev_runner.conf import settings + +from .base_runner import BaseRunner + + +class ModelsRunner(BaseRunner): + PACKAGE_NAME = "models" + PORT = settings.MODELS_PORT + APP_NAME = "models" + DB_CREDENTIALS = { + "POSTGRES_DB": "models", + } + ENVIRONMENT = { + "DATABASE_URL": "postgresql+psycopg2://postgres:postgres@localhost:5432/models", + "MODELS_NAMESPACE": "dev2", + "DOMAIN_NAME": "localhost", + "ALGORITHM": "RS256", + "SECRET": "some_secret_key", + "DOCKER_REGISTRY_URL": "localhost:5000", + } diff --git a/dev_runner/dev_runner/runners/pipelines_runner.py b/dev_runner/dev_runner/runners/pipelines_runner.py new file mode 100644 index 000000000..caff6c5aa --- /dev/null +++ b/dev_runner/dev_runner/runners/pipelines_runner.py @@ -0,0 +1,22 @@ +from dev_runner.conf import settings + +from .base_runner import BaseRunner + + +class PipelinesRunner(BaseRunner): + PACKAGE_NAME = "pipelines" + PORT = settings.PIPELINES_PORT + APP_NAME = "pipelines" + MODULE_NAME = "app" + DB_CREDENTIALS = { + "POSTGRES_DB": "pipelines", + } + ENVIRONMENT = { + "HEARTBEAT_TIMEOUT": "15", + "HEARTBEAT_THRESHOLD_MUL": "10", + "RUNNER_TIMEOUT": "5", + "MAX_WORKERS": "20", + "DEBUG_MERGE": "True", + "SA_POOL_SIZE": "40", + "LOG_LEVEL": "DEBUG", + } diff --git a/dev_runner/dev_runner/runners/processing_runner.py b/dev_runner/dev_runner/runners/processing_runner.py new file mode 100644 index 000000000..669ed4ce7 --- /dev/null +++ b/dev_runner/dev_runner/runners/processing_runner.py @@ -0,0 +1,23 @@ +import logging + +from dev_runner.conf import settings + +from .base_runner import BaseRunner + + +class ProcessingRunner(BaseRunner): + PACKAGE_NAME = "processing" + PORT = settings.PROCESSING_PORT + APP_NAME = "processing" + DB_CREDENTIALS = { + "POSTGRES_DB": "processing", + } + ENVIRONMENT = { + "POSTGRES_DB": "processing", + "MODELS_POSTFIX": "", + "LOCAL_RUN": "1", + "SERVICE_NAME": "processing", + "HOST": "localhost", + "PORT": str(PORT), + "LOG_LEVEL": "10", + } diff --git a/dev_runner/dev_runner/runners/scheduler_runner.py b/dev_runner/dev_runner/runners/scheduler_runner.py new file mode 100644 index 000000000..224b7d8a4 --- /dev/null +++ b/dev_runner/dev_runner/runners/scheduler_runner.py @@ -0,0 +1,25 @@ +from dev_runner.conf import settings + +from .base_runner import BaseRunner + + +class SchedulerRunner(BaseRunner): + PACKAGE_NAME = "scheduler" + PORT = settings.SCHEDULER_PORT + APP_NAME = "scheduler" + MODULE_NAME = "app" + DB_CREDENTIALS = {"POSTGRES_DB": "scheduler"} + ENVIRONMENT = { + "DB_NAME": "scheduler", + "DB_URL": "postgresql+psycopg2://postgres:postgres@localhost:5432/scheduler", + "TEST_MODE": "False", + "SA_POOL_SIZE": "10", + "KAFKA_BOOTSTRAP_SERVER": "localhost:9092", + "KAFKA_GROUP_ID": "scheduler_group", + "KAFKA_CONSUME_TOPICS": "pipelines", + "KAFKA_TOPICS_PARTITIONS": "1", + "KAFKA_REPLICATION_FACTORS": "1", + "HEARTBEAT_TIMEOUT": "10", + "THRESHOLD_MUL": "3", + "LOG_LEVEL": "DEBUG", + } diff --git a/dev_runner/dev_runner/runners/search_runner.py b/dev_runner/dev_runner/runners/search_runner.py new file mode 100644 index 000000000..75dbc336c --- /dev/null +++ b/dev_runner/dev_runner/runners/search_runner.py @@ -0,0 +1,25 @@ +from dev_runner.conf import settings + +from .base_runner import BaseRunner + + +class SearchRunner(BaseRunner): + PACKAGE_NAME = "search" + PORT = settings.SEARCH_PORT + APP_NAME = "search" + ENVIRONMENT = { + "S3_START_PATH": "annotation", + "APP_TITLE": "Badgerdoc Search", + "JOBS_SEARCH": "/jobs/search", + "ANNOTATION_CATEGORIES": "/categories", + "ANNOTATION_CATEGORIES_SEARCH": "/categories/search", + "MANIFEST": "manifest.json", + "TEXT_PIECES_PATH": "/pieces", + "INDEXATION_PATH": "indexation", + "COMPUTED_FIELDS": '["job_id", "category"]', + "JWT_ALGORITHM": "RS256", + "KAFKA_GROUP_ID": "search_group", + "KAFKA_SEARCH_TOPIC": "search", + "KAFKA_SEARCH_TOPIC_PARTITIONS": "50", + "KAFKA_SEARCH_REPLICATION_FACTOR": "1", + } diff --git a/dev_runner/dev_runner/runners/taxonomy_runner.py b/dev_runner/dev_runner/runners/taxonomy_runner.py new file mode 100644 index 000000000..c73f2f011 --- /dev/null +++ b/dev_runner/dev_runner/runners/taxonomy_runner.py @@ -0,0 +1,16 @@ +from dev_runner.conf import settings + +from .base_runner import BaseRunner + + +class TaxonomyRunner(BaseRunner): + PACKAGE_NAME = "taxonomy" + PORT = settings.TAXONOMY_PORT + APP_NAME = "taxonomy" + DB_CREDENTIALS = { + "POSTGRES_DB": "taxonomy", + } + ENVIRONMENT = { + "APP_HOST": "localhost", + "APP_PORT": str(PORT), + } diff --git a/dev_runner/dev_runner/runners/users_runner.py b/dev_runner/dev_runner/runners/users_runner.py new file mode 100644 index 000000000..9e8ad1275 --- /dev/null +++ b/dev_runner/dev_runner/runners/users_runner.py @@ -0,0 +1,26 @@ +from dev_runner.conf import settings + +from .base_runner import BaseRunner + + +class UsersRunner(BaseRunner): + PACKAGE_NAME = "users" + PORT = settings.USERS_PORT + APP_NAME = "users" + DB_CREDENTIALS = { + "POSTGRES_DB": "keycloak_db", + } + ENVIRONMENT = { + "DB_VENDOR": "POSTGRES", + "DB_ADDR": "postgres", + "DB_DATABASE": "keycloak_db", + "DB_USER": "postgres", + "DB_PASSWORD": "postgres", + "POSTGRES_HOST": "postgres", + "KEYCLOAK_USER": "user", + "KEYCLOAK_PASSWORD": "secretpassword", + "KEYCLOAK_ENDPOINT": "http://localhost/", + "KEYCLOAK_REALM": "master", + "KEYCLOAK_ROLE_ADMIN": "admin", + "KEYCLOAK_USERS_PUBLIC_KEY": "", + } diff --git a/dev_runner/docker-compose.yml b/dev_runner/docker-compose.yml new file mode 100644 index 000000000..a67eecdc2 --- /dev/null +++ b/dev_runner/docker-compose.yml @@ -0,0 +1,118 @@ +version: "3.9" + + +services: + postgres-postgresql: + image: postgres:13.4 + volumes: + - ./pg-init-scripts:/docker-entrypoint-initdb.d + - pgdata:/var/lib/postgresql/data + container_name: postgres + networks: + - bd + environment: + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=postgres + - POSTGRES_MULTIPLE_DATABASES=annotation,file_management,job_manager,models,pipelines,processing,scheduler,taxonomy,keycloak_db + ports: + - "5432:5432" + restart: always + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 10s + timeout: 5s + retries: 5 + + minio: + image: 'bitnami/minio:latest' + ports: + - '9000:9000' + - '9001:9001' + networks: + - bd + environment: + - MINIO_ROOT_USER=minioadmin + - MINIO_ROOT_PASSWORD=minioadmin + + gotenberg: + image: gotenberg/gotenberg:7 + ports: + - "3000:3000" + networks: + - bd + env_file: + - goten.env + + zookeeper: + image: wurstmeister/zookeeper + container_name: zookeeper + networks: + - bd + ports: + - "2181:2181" + environment: + - ALLOW_ANONYMOUS_LOGIN=yes + + kafka: + image: wurstmeister/kafka + container_name: kafka + networks: + - bd + ports: + - "9092:9092" + environment: + KAFKA_ADVERTISED_HOST_NAME: localhost + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 +# KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'false' + depends_on: + - zookeeper + + keycloak: + image: jboss/keycloak + container_name: keycloak + networks: + - bd + ports: + - "8080:8080" + - "8443:8443" + environment: + - KEYCLOAK_USER=admin + - KEYCLOAK_PASSWORD=admin + - DB_VENDOR=POSTGRES + - DB_ADDR=postgres-postgresql + - DB_DATABASE=keycloak_db + - DB_USER=postgres + - DB_PASSWORD=postgres + depends_on: + - postgres-postgresql + + elasticsearch: + image: docker.elastic.co/elasticsearch/elasticsearch:7.13.2 + container_name: elastic + networks: + - bd + environment: + - xpack.security.enabled=false + - discovery.type=single-node + - ES_JAVA_OPTS=-Xmx300m + restart: always + deploy: + resources: + limits: + memory: 512m + healthcheck: + test: curl --fail http://localhost:9200 || exit 1 + interval: 30s + timeout: 3s + retries: 10 + start_period: 30s + ports: + - "9200:9200" + +volumes: + pgdata: + +networks: + bd: + driver: bridge diff --git a/dev_runner/goten.env b/dev_runner/goten.env new file mode 100644 index 000000000..10d351790 --- /dev/null +++ b/dev_runner/goten.env @@ -0,0 +1,33 @@ +# Globals +APP_NAME="assets" +UPLOADING_LIMIT=100 +WIDTH=450 +BBOX_EXT=20 +ROOT_PATH= +LOG_FILE=false + +POSTGRES_USER=admin +POSTGRES_PASSWORD=admin +POSTGRES_DB=file_management +POSTGRES_HOST=localhost +POSTGRES_PORT=5432 +DATABASE_URL="postgresql+psycopg2://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}" + +S3_PREFIX= +S3_ENDPOINT=minio:9000 +S3_ACCESS_KEY=minioadmin +S3_SECRET_KEY=minioadmin +TEST_REGION=us-west-2 +MINIO_SECURE_CONNECTION=False + +SQLACLHEMY_POOL_SIZE=20 + +KEYCLOAK_URI=http://bagerdoc-keycloack + +GOTENBERG=gotenberg:3000 + +GOTENBERG_LIBRE_OFFICE_ENDPOINT="http://${GOTENBERG}/forms/libreoffice/convert" + +GOTENBERG_FORMATS=[".txt",".docx",".doc",".bib",".xml",".fodt",".html",".ltx",".odt",".ott",".pdb",".psw",".rtf",".sdw",".stw",".sxw",".uot",".vor",".wps",".epub",".emf",".fodg",".met",".odd",".otg",".std",".svg",".svm",".swf",".sxd",".sxw",".tiff",".xhtml",".xpm",".fodp",".potm",".pot",".pptx",".pps",".ppt",".pwp",".sda",".sdd",".sti",".sxi",".uop",".wmf",".odp"] + +IMAGE_FORMATS=[".png",".bmp", ".pbm", ".pct", ".pgm", ".ppm", ".ras", ".tiff"] diff --git a/dev_runner/migration.sh b/dev_runner/migration.sh new file mode 100755 index 000000000..2d279fbc6 --- /dev/null +++ b/dev_runner/migration.sh @@ -0,0 +1,13 @@ +SHARED_PATH=$(realpath "./conf/shared.env") +ASSETS_PATH="./conf/assets.env" + +set -e + + +for service in "assets" "annotation" "jobs" "models" "pipelines" "processing" "scheduler" "taxonomy" +do + echo "Migrate database for :"$service + cd "../"$service + source $SHARED_PATH && alembic upgrade head +done + diff --git a/dev_runner/pg-init-scripts/create-multiple-postgresql-databases.sh b/dev_runner/pg-init-scripts/create-multiple-postgresql-databases.sh new file mode 100755 index 000000000..aa665fa46 --- /dev/null +++ b/dev_runner/pg-init-scripts/create-multiple-postgresql-databases.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +set -e +set -u + +function create_user_and_database() { + local database=$1 + echo " Creating user and database '$database'" + psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" <<-EOSQL + CREATE USER $database; + CREATE DATABASE $database; + GRANT ALL PRIVILEGES ON DATABASE $database TO $database; +EOSQL +} + +if [ -n "$POSTGRES_MULTIPLE_DATABASES" ]; then + echo "Multiple database creation requested: $POSTGRES_MULTIPLE_DATABASES" + for db in $(echo $POSTGRES_MULTIPLE_DATABASES | tr ',' ' '); do + create_user_and_database $db + done + echo "Multiple databases created" +fi diff --git a/dev_runner/pyproject.toml b/dev_runner/pyproject.toml new file mode 100644 index 000000000..c08acd59d --- /dev/null +++ b/dev_runner/pyproject.toml @@ -0,0 +1,111 @@ +[tool.poetry] +name = "dev-runner" +version = "0.1.0" +description = "run badgerdoc services locally" +authors = ["Your Name "] +readme = "README.md" +packages = [{include = "dev_runner"}] + +[tool.poetry.dependencies] +python = "^3.9" +requests-oauthlib = "1.3.1" +rfc3986 = {version = "1.5.0", extras = ["idna2008"]} +aiocache = "0.11.1" +aiokafka = "0.7.2" +asgiref = "3.5.0" +async-timeout = "4.0.2" +attrs = "22.2.0" +boto3 = "1.20.54" +botocore = "1.23.54" +certifi = "2021.10.8" +cffi = "1.15.1" +charset-normalizer = "2.0.12" +click = "8.0.3" +colorama = "0.4.6" +cryptography = "36.0.0" +elasticsearch = {version = "7.13.4", extras = ["async"]} +frozenlist = "1.3.3" +h11 = "0.13.0" +idna = "3.3" +jmespath = "0.10.0" +kafka-python = "2.0.2" +multidict = "6.0.4" +pycparser = "2.21" +pydantic = "1.8.2" +python-dateutil = "2.8.2" +python-dotenv = "0.19.1" +pyyaml = "6.0" +s3transfer = "0.5.1" +six = "1.16.0" +typing-extensions = "4.1.1" +urllib3 = ">=1.26.8,<1.27.0" +uvicorn = "0.15.0" +yarl = "1.8.2" +cachetools = "5.2.0" +importlib-metadata = "4.11.0" +importlib-resources = "5.4.0" +mako = "1.1.6" +markupsafe = "2.0.1" +psycopg2-binary = "2.9.1" +requests = "2.26.0" +zipp = "3.7.0" +anyio = "3.5.0" +boto3-stubs = "1.26.64" +botocore-stubs = "1.29.64" +chardet = "4.0.0" +minio = "7.1.0" +mypy-extensions = "0.4.4" +pdfminer-six = "20200517" +pdfplumber = "0.5.28" +pillow = "9.0.1" +pycryptodome = "3.17" +pymupdf-fonts = "1.0.5" +pymupdf = "1.21.1" +sniffio = "1.2.0" +sortedcontainers = "2.4.0" +types-awscrt = "0.16.4" +types-s3transfer = "0.6.0.post5" +wand = "0.6.11" +bcrypt = "4.0.1" +google-auth = "2.15.0" +kubernetes = "19.15.0" +oauthlib = "3.2.2" +paramiko = "2.12.0" +pyasn1-modules = "0.2.8" +pyasn1 = "0.4.8" +pyjwt = {version = "2.3.0", extras = ["crypto"]} +pynacl = "1.5.0" +python-multipart = "0.0.5" +rsa = "4.9" +setuptools = ">=60.5.0,<60.6.0" +websocket-client = "1.4.2" +async-cache = "1.1.1" +numpy = "1.24.1" +types-requests = "2.28.11.7" +types-urllib3 = "1.26.25.4" +httpcore = "0.16.2" +httpx = "0.23.1" +exceptiongroup = "1.0.4" +iniconfig = "1.1.1" +packaging = "22.0" +pdf2image = "1.16.0" +pluggy = "1.0.0" +pytest = "7.2.0" +python-magic = "0.4.25" +tomli = "2.0.1" +sqlalchemy-utils = "*" +email-validator = "1.1.3" +apscheduler = "3.9.1" +starlette = "*" +fastapi = "*" +aiohttp = "*" +sqlalchemy = "*" +aiosignal = "*" +alembic = "*" +tenant-dependency = {path = "../lib/tenants"} +filter-lib = {path = "../lib/filter_lib"} + + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/dev_runner/start.py b/dev_runner/start.py new file mode 100644 index 000000000..5bfe67fed --- /dev/null +++ b/dev_runner/start.py @@ -0,0 +1,43 @@ +import asyncio +from pathlib import Path + +import click +from dotenv import load_dotenv + +from dev_runner.runners.annotation_runner import AnnotationRunner +from dev_runner.runners.assets_runner import AssetsRunner +from dev_runner.runners.base_runner import RunnerRegistry +from dev_runner.runners.convert_runner import ConvertRunner +from dev_runner.runners.jobs_runner import JobsRunner +from dev_runner.runners.models_runner import ModelsRunner +from dev_runner.runners.pipelines_runner import PipelinesRunner +from dev_runner.runners.processing_runner import ProcessingRunner +from dev_runner.runners.scheduler_runner import SchedulerRunner +from dev_runner.runners.search_runner import SearchRunner +from dev_runner.runners.taxonomy_runner import TaxonomyRunner +from dev_runner.runners.users_runner import UsersRunner + +ROOT_DIR = Path(__file__).parent +SHARED_DOT_ENV = ROOT_DIR / "conf" / "shared.env" + + +def _info(message): + click.echo(click.style(message, fg="green")) + + +@click.command() +@click.argument( + "services", + nargs=-1, + type=click.Choice(RunnerRegistry.get_runners().keys()), +) +def cli(services): + _info( + f"Starting {services or 'all'} service{'s' if not services or len(services) > 1 else ''}..." + ) + load_dotenv(SHARED_DOT_ENV) + asyncio.run(RunnerRegistry.run(services)) + + +if __name__ == "__main__": + cli() diff --git a/jobs/alembic/env.py b/jobs/alembic/env.py index af431b4b8..461221e00 100644 --- a/jobs/alembic/env.py +++ b/jobs/alembic/env.py @@ -2,9 +2,9 @@ import os from logging.config import fileConfig +from alembic import context from sqlalchemy import engine_from_config, pool -from alembic import context from jobs.utils import get_test_db_url # this is the Alembic Config object, which provides @@ -28,9 +28,7 @@ # my_important_option = config.get_main_option("my_important_option") # ... etc. -main_database_url = os.environ.get( - "POSTGRESQL_JOBMANAGER_DATABASE_URI" -) +main_database_url = os.environ.get("POSTGRESQL_JOBMANAGER_DATABASE_URI") if not os.getenv("USE_TEST_DB"): config.set_main_option("sqlalchemy.url", main_database_url) else: diff --git a/jobs/alembic/versions/13ac4bb3abd2_.py b/jobs/alembic/versions/13ac4bb3abd2_.py index 8db2dcf3d..557d9c2a5 100644 --- a/jobs/alembic/versions/13ac4bb3abd2_.py +++ b/jobs/alembic/versions/13ac4bb3abd2_.py @@ -7,9 +7,8 @@ """ import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - from alembic import op +from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision = "13ac4bb3abd2" diff --git a/jobs/alembic/versions/3f5b2d199d38_.py b/jobs/alembic/versions/3f5b2d199d38_.py index 8b1872cd8..e7d0884d9 100644 --- a/jobs/alembic/versions/3f5b2d199d38_.py +++ b/jobs/alembic/versions/3f5b2d199d38_.py @@ -7,7 +7,6 @@ """ import sqlalchemy as sa - from alembic import op # revision identifiers, used by Alembic. diff --git a/jobs/alembic/versions/7511c6790067_.py b/jobs/alembic/versions/7511c6790067_.py index 2857a099a..d17007933 100644 --- a/jobs/alembic/versions/7511c6790067_.py +++ b/jobs/alembic/versions/7511c6790067_.py @@ -7,9 +7,8 @@ """ import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - from alembic import op +from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision = "7511c6790067" diff --git a/jobs/alembic/versions/83694c0b2df6_.py b/jobs/alembic/versions/83694c0b2df6_.py index 64f769bc2..1398e1840 100644 --- a/jobs/alembic/versions/83694c0b2df6_.py +++ b/jobs/alembic/versions/83694c0b2df6_.py @@ -7,7 +7,6 @@ """ import sqlalchemy as sa - from alembic import op # revision identifiers, used by Alembic. diff --git a/jobs/alembic/versions/86f432539475_.py b/jobs/alembic/versions/86f432539475_.py index ec341ec58..18e95a46d 100644 --- a/jobs/alembic/versions/86f432539475_.py +++ b/jobs/alembic/versions/86f432539475_.py @@ -7,7 +7,6 @@ """ import sqlalchemy as sa - from alembic import op # revision identifiers, used by Alembic. diff --git a/jobs/alembic/versions/9229e70d2791_.py b/jobs/alembic/versions/9229e70d2791_.py index 19b0c9c87..d9223fd51 100644 --- a/jobs/alembic/versions/9229e70d2791_.py +++ b/jobs/alembic/versions/9229e70d2791_.py @@ -7,9 +7,8 @@ """ import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - from alembic import op +from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision = "9229e70d2791" diff --git a/jobs/alembic/versions/b4afb5ae8923_add_start_manual_job_automatically_flag.py b/jobs/alembic/versions/b4afb5ae8923_add_start_manual_job_automatically_flag.py index 25a1ed7f3..80f51df0a 100644 --- a/jobs/alembic/versions/b4afb5ae8923_add_start_manual_job_automatically_flag.py +++ b/jobs/alembic/versions/b4afb5ae8923_add_start_manual_job_automatically_flag.py @@ -5,24 +5,28 @@ Create Date: 2022-03-17 20:22:30.242625 """ -from alembic import op import sqlalchemy as sa - +from alembic import op # revision identifiers, used by Alembic. -revision = 'b4afb5ae8923' -down_revision = '86f432539475' +revision = "b4afb5ae8923" +down_revision = "86f432539475" branch_labels = None depends_on = None def upgrade(): # ### commands auto generated by Alembic - please adjust! ### - op.add_column('job', sa.Column('start_manual_job_automatically', sa.Boolean(), nullable=True)) + op.add_column( + "job", + sa.Column( + "start_manual_job_automatically", sa.Boolean(), nullable=True + ), + ) # ### end Alembic commands ### def downgrade(): # ### commands auto generated by Alembic - please adjust! ### - op.drop_column('job', 'start_manual_job_automatically') + op.drop_column("job", "start_manual_job_automatically") # ### end Alembic commands ### diff --git a/jobs/alembic/versions/d1ddce2d5352_.py b/jobs/alembic/versions/d1ddce2d5352_.py index 1e1ea086b..2b2c28312 100644 --- a/jobs/alembic/versions/d1ddce2d5352_.py +++ b/jobs/alembic/versions/d1ddce2d5352_.py @@ -7,7 +7,6 @@ """ import sqlalchemy as sa - from alembic import op # revision identifiers, used by Alembic. diff --git a/jobs/alembic/versions/f60dd492b17f_add_extensive_coverage_param.py b/jobs/alembic/versions/f60dd492b17f_add_extensive_coverage_param.py index 8c321be4b..2dab1ada2 100644 --- a/jobs/alembic/versions/f60dd492b17f_add_extensive_coverage_param.py +++ b/jobs/alembic/versions/f60dd492b17f_add_extensive_coverage_param.py @@ -5,24 +5,25 @@ Create Date: 2022-12-09 13:10:42.668902 """ -from alembic import op import sqlalchemy as sa - +from alembic import op # revision identifiers, used by Alembic. -revision = 'f60dd492b17f' -down_revision = 'b4afb5ae8923' +revision = "f60dd492b17f" +down_revision = "b4afb5ae8923" branch_labels = None depends_on = None def upgrade(): # ### commands auto generated by Alembic - please adjust! ### - op.add_column('job', sa.Column('extensive_coverage', sa.Integer(), nullable=True)) + op.add_column( + "job", sa.Column("extensive_coverage", sa.Integer(), nullable=True) + ) # ### end Alembic commands ### def downgrade(): # ### commands auto generated by Alembic - please adjust! ### - op.drop_column('job', 'extensive_coverage') + op.drop_column("job", "extensive_coverage") # ### end Alembic commands ### diff --git a/jobs/jobs/main.py b/jobs/jobs/main.py index e8a1037ee..57a8939f7 100644 --- a/jobs/jobs/main.py +++ b/jobs/jobs/main.py @@ -138,7 +138,7 @@ async def run_job( current_tenant: Optional[str] = Header(None, alias="X-Current-Tenant"), db: Session = Depends(db_service.get_session), token_data: TenantData = Depends(tenant), -) -> Union[Dict[str, Any], HTTPException]: +) -> Dict[str, Any]: """Runs any type of Job""" jw_token = token_data.token job_to_run = db_service.get_job_in_db_by_id(db, job_id) @@ -203,7 +203,7 @@ async def change_job( token_data: TenantData = Depends(tenant), current_tenant: Optional[str] = Header(None, alias="X-Current-Tenant"), db: Session = Depends(db_service.get_session), -) -> Union[Dict[str, Any], HTTPException]: +) -> Dict[str, Any]: """Provides an ability to change any value of any field of any Job in the database""" job_to_change = db_service.get_job_in_db_by_id(db, job_id) @@ -347,7 +347,7 @@ async def delete_job( current_tenant: Optional[str] = Header(None, alias="X-Current-Tenant"), db: Session = Depends(db_service.get_session), token_data: TenantData = Depends(tenant), -) -> Union[Dict[str, Any], HTTPException]: +) -> Dict[str, Any]: """Deletes Job instance by its id""" jw_token = token_data.token job_to_delete = db_service.get_job_in_db_by_id(db, job_id) diff --git a/jobs/jobs/utils.py b/jobs/jobs/utils.py index e786b2d5f..522725c0f 100644 --- a/jobs/jobs/utils.py +++ b/jobs/jobs/utils.py @@ -5,24 +5,14 @@ from sqlalchemy.orm import Session from jobs import db_service -from jobs.config import ( - HOST_ANNOTATION, - HOST_ASSETS, - HOST_PIPELINES, - HOST_TAXONOMY, - JOBS_HOST, - PAGINATION_THRESHOLD, - ROOT_PATH, -) +from jobs.config import (HOST_ANNOTATION, HOST_ASSETS, HOST_PIPELINES, + HOST_TAXONOMY, JOBS_HOST, PAGINATION_THRESHOLD, + ROOT_PATH) from jobs.logger import logger from jobs.models import CombinedJob -from jobs.schemas import ( - AnnotationJobUpdateParamsInAnnotation, - CategoryLinkInput, - CategoryLinkParams, - JobMode, - JobParamsToChange, -) +from jobs.schemas import (AnnotationJobUpdateParamsInAnnotation, + CategoryLinkInput, CategoryLinkParams, JobMode, + JobParamsToChange) async def get_files_data_from_datasets( diff --git a/jobs/tests/conftest.py b/jobs/tests/conftest.py index 667f04a4d..fc3ae807f 100644 --- a/jobs/tests/conftest.py +++ b/jobs/tests/conftest.py @@ -4,24 +4,21 @@ from typing import List from unittest.mock import patch -import jobs.db_service as service -import jobs.main as main -import jobs.schemas as schemas import pytest +from alembic import command +from alembic.config import Config from fastapi.testclient import TestClient -from jobs.utils import get_test_db_url from pydantic import BaseModel from sqlalchemy import create_engine # type: ignore from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.orm import sessionmaker # type: ignore -from sqlalchemy_utils import ( # type: ignore - create_database, - database_exists, - drop_database, -) +from sqlalchemy_utils import (create_database, database_exists, # type: ignore + drop_database) -from alembic import command -from alembic.config import Config +import jobs.db_service as service +import jobs.main as main +import jobs.schemas as schemas +from jobs.utils import get_test_db_url main_database_url = os.environ.get("POSTGRESQL_JOBMANAGER_DATABASE_URI") test_db_url = get_test_db_url(main_database_url) diff --git a/jobs/tests/test_API_functions/test_change_job.py b/jobs/tests/test_API_functions/test_change_job.py index 1a2c66aaf..ef423d1b2 100644 --- a/jobs/tests/test_API_functions/test_change_job.py +++ b/jobs/tests/test_API_functions/test_change_job.py @@ -1,12 +1,11 @@ import asyncio -import pytest from unittest.mock import patch +import pytest +from tests.test_db import (create_mock_annotation_job_in_db, + create_mock_extraction_job_in_db) + import jobs.schemas as schemas -from tests.test_db import ( - create_mock_annotation_job_in_db, - create_mock_extraction_job_in_db, -) def test_change_job_status_with_validation_correct_jwt_provided( @@ -76,10 +75,17 @@ def test_change_job_linked_taxonomy( create_mock_extraction_job_in_db(testing_session) with patch("jobs.utils.fetch", return_value=asyncio.Future()) as mock: mock.side_effect = [(204, {}), (200, {})] - response = testing_app.put("/jobs/1", json={"categories": [{ - "category_id": "category2", - "taxonomy_id": "my_taxonomy_id", - "taxonomy_version": 1 - }]}) + response = testing_app.put( + "/jobs/1", + json={ + "categories": [ + { + "category_id": "category2", + "taxonomy_id": "my_taxonomy_id", + "taxonomy_version": 1, + } + ] + }, + ) assert response.status_code == 200 assert response.json()["categories"] == ["category2"] diff --git a/jobs/tests/test_API_functions/test_create_job.py b/jobs/tests/test_API_functions/test_create_job.py index 31a91435e..47b4d2bbc 100644 --- a/jobs/tests/test_API_functions/test_create_job.py +++ b/jobs/tests/test_API_functions/test_create_job.py @@ -4,9 +4,10 @@ import aiohttp.client_exceptions import freezegun +import pytest + import jobs.create_job_funcs as create_job_funcs import jobs.schemas as schemas -import pytest # ----------- Create Job Drafts ------------- # diff --git a/jobs/tests/test_API_functions/test_other_API_functions.py b/jobs/tests/test_API_functions/test_other_API_functions.py index 6dc7c93f5..1f6471ece 100644 --- a/jobs/tests/test_API_functions/test_other_API_functions.py +++ b/jobs/tests/test_API_functions/test_other_API_functions.py @@ -1,10 +1,10 @@ import asyncio from unittest.mock import patch + +from tests.test_db import (create_mock_annotation_job_in_db, + create_mock_extraction_job_in_db) + import jobs.schemas as schemas -from tests.test_db import ( - create_mock_annotation_job_in_db, - create_mock_extraction_job_in_db, -) def test_get_all_jobs_endpoint( @@ -48,7 +48,9 @@ def test_delete_job_positive( with patch("jobs.utils.fetch", return_value=asyncio.Future()) as mock: mock.side_effect = [(200, {})] create_mock_extraction_job_in_db(testing_session) - create_mock_annotation_job_in_db(testing_session, mock_AnnotationJobParams) + create_mock_annotation_job_in_db( + testing_session, mock_AnnotationJobParams + ) response = testing_app.delete( "/jobs/2", ) diff --git a/jobs/tests/test_API_functions/test_search_jobs.py b/jobs/tests/test_API_functions/test_search_jobs.py index 317ce99a3..84695aecb 100644 --- a/jobs/tests/test_API_functions/test_search_jobs.py +++ b/jobs/tests/test_API_functions/test_search_jobs.py @@ -1,7 +1,5 @@ -from tests.test_db import ( - create_mock_annotation_job_in_db, - create_mock_extraction_job_in_db, -) +from tests.test_db import (create_mock_annotation_job_in_db, + create_mock_extraction_job_in_db) def test_search_job_positive(testing_app, testing_session): @@ -37,8 +35,10 @@ def test_search_job_invalid_field(testing_app, testing_session): }, ) assert response.status_code == 422 - response_message = response.json()['detail'][0]['msg'] - assert response_message.startswith("value is not a valid enumeration member") + response_message = response.json()["detail"][0]["msg"] + assert response_message.startswith( + "value is not a valid enumeration member" + ) def test_search_job_without_filters( diff --git a/jobs/tests/test_utils.py b/jobs/tests/test_utils.py index ec68c45a2..641f21caa 100644 --- a/jobs/tests/test_utils.py +++ b/jobs/tests/test_utils.py @@ -1,10 +1,11 @@ from unittest.mock import patch import aiohttp.client_exceptions -import jobs.utils as utils import pytest from fastapi import HTTPException +import jobs.utils as utils + # --------------TEST get_files_data_from_datasets--------------- diff --git a/lib/filter_lib/.pre-commit-config.yaml b/lib/filter_lib/.pre-commit-config.yaml index 60a8973cf..289d36f82 100644 --- a/lib/filter_lib/.pre-commit-config.yaml +++ b/lib/filter_lib/.pre-commit-config.yaml @@ -6,7 +6,7 @@ repos: - id: end-of-file-fixer - id: check-yaml - repo: https://github.com/pycqa/isort - rev: 5.9.1 + rev: 5.12.0 hooks: - id: isort args: diff --git a/lib/filter_lib/src/schema_generator.py b/lib/filter_lib/src/schema_generator.py index f6e0d5813..57992248b 100644 --- a/lib/filter_lib/src/schema_generator.py +++ b/lib/filter_lib/src/schema_generator.py @@ -68,9 +68,9 @@ class BaseSearch(BaseModel): pagination: Optional[Pagination] @root_validator - def root_validate( # pylint: disable=no-self-argument + def root_validate( cls, values: Any - ) -> Any: + ) -> Any: # pylint: disable=no-self-argument if not values.get("pagination"): values["pagination"] = Pagination(page_num=1, page_size=15) return values @@ -92,9 +92,9 @@ class Page(GenericModel, Generic[TypeC], BaseModel): data: Sequence[TypeC] @validator("data") - def custom_validator( # pylint: disable=no-self-argument + def custom_validator( cls, v: Any - ) -> Any: + ) -> Any: # pylint: disable=no-self-argument """Custom validator applied to data in case of using 'distinct' statement and getting result as 'sqlalchemy.util._collections.result' but not as model class object diff --git a/lib/filter_lib/tests/test_dict_parser.py b/lib/filter_lib/tests/test_dict_parser.py index 5228d14b8..29173a00e 100644 --- a/lib/filter_lib/tests/test_dict_parser.py +++ b/lib/filter_lib/tests/test_dict_parser.py @@ -1,6 +1,5 @@ from ..src.dict_parser import map_request_to_filter - example_1 = { "pagination": {"page_num": 1, "page_size": 50}, "filters": [ diff --git a/lib/filter_lib/tests/test_enum_generator.py b/lib/filter_lib/tests/test_enum_generator.py index 3624d69fc..c8985c624 100644 --- a/lib/filter_lib/tests/test_enum_generator.py +++ b/lib/filter_lib/tests/test_enum_generator.py @@ -1,10 +1,6 @@ -from .conftest import User, Address -from ..src.enum_generator import ( - _get_model_fields, - _exclude_fields, - _get_table_name, - _create_enum_model, -) +from ..src.enum_generator import (_create_enum_model, _exclude_fields, + _get_model_fields, _get_table_name) +from .conftest import Address, User def test_get_model_fields(): @@ -34,7 +30,11 @@ def test_exclude_fields(): ) == ["name", "email", "addresses.owner"] assert _exclude_fields( address_fields, ["id", "user.name", "user.email"] - ) == ["location", "owner", "user.id"] + ) == [ + "location", + "owner", + "user.id", + ] def test_get_table_name(): diff --git a/lib/filter_lib/tests/test_pagination.py b/lib/filter_lib/tests/test_pagination.py index 638faf66b..4d535fe5c 100644 --- a/lib/filter_lib/tests/test_pagination.py +++ b/lib/filter_lib/tests/test_pagination.py @@ -1,12 +1,9 @@ -from ..src.pagination import ( - paginate, - PaginationParams, - make_pagination, - _calculate_num_pages, -) -from .conftest import User -from pydantic import ValidationError import pytest +from pydantic import ValidationError + +from ..src.pagination import (PaginationParams, _calculate_num_pages, + make_pagination, paginate) +from .conftest import User @pytest.mark.parametrize( diff --git a/lib/filter_lib/tests/test_query_modifier.py b/lib/filter_lib/tests/test_query_modifier.py index 2a6f36b75..182e4a683 100644 --- a/lib/filter_lib/tests/test_query_modifier.py +++ b/lib/filter_lib/tests/test_query_modifier.py @@ -1,12 +1,7 @@ from ..src.enum_generator import get_enum_from_orm -from ..src.query_modificator import ( - _create_filter, - _create_or_condition, - _get_column, - _get_entity, - _op_is_not, - form_query, -) +from ..src.query_modificator import (_create_filter, _create_or_condition, + _get_column, _get_entity, _op_is_not, + form_query) from .conftest import Address, Category, User diff --git a/lib/filter_lib/tests/test_schema_generator.py b/lib/filter_lib/tests/test_schema_generator.py index 571582c4c..a7310cc0c 100644 --- a/lib/filter_lib/tests/test_schema_generator.py +++ b/lib/filter_lib/tests/test_schema_generator.py @@ -1,9 +1,5 @@ -from .conftest import User, Address -from ..src.schema_generator import ( - create_filter_model, - Page, - PaginationOut, -) +from ..src.schema_generator import Page, PaginationOut, create_filter_model +from .conftest import Address, User def test_search_class_creating(): @@ -20,7 +16,13 @@ def test_search_class_creating(): AddressFilter = create_filter_model(Address, exclude=["location"]) assert AddressFilter.schema()["definitions"]["addresses_Address"][ "enum" - ] == ["id", "owner", "user.id", "user.name", "user.email"] + ] == [ + "id", + "owner", + "user.id", + "user.name", + "user.email", + ] def test_page_schema(): diff --git a/lib/filter_lib/usage_example/app.py b/lib/filter_lib/usage_example/app.py index f8de34a8b..05e3643ae 100644 --- a/lib/filter_lib/usage_example/app.py +++ b/lib/filter_lib/usage_example/app.py @@ -2,17 +2,11 @@ from db_example import Address, User, get_db from fastapi import Depends, FastAPI +from filter_lib import (Page, create_filter_model, form_query, # type: ignore + map_request_to_filter, paginate) from pydantic import BaseModel from sqlalchemy.orm import Session -from filter_lib import ( # type: ignore - Page, - create_filter_model, - form_query, - map_request_to_filter, - paginate, -) - app = FastAPI() @@ -58,9 +52,7 @@ def create_new_user( @app.post("/users/search", tags=["users"], response_model=Page[UserOut]) -def search_users( - request: UserFilterModel, session: Session = Depends(get_db) # type: ignore # noqa -) -> Page[UserOut]: +def search_users(request: UserFilterModel, session: Session = Depends(get_db)) -> Page[UserOut]: # type: ignore # noqa query = session.query(User) filter_args = map_request_to_filter(request.dict(), "User") # type: ignore query, pagination = form_query(filter_args, query) diff --git a/lib/tenants/tests/conftest.py b/lib/tenants/tests/conftest.py index 4132b7027..69df9f771 100644 --- a/lib/tenants/tests/conftest.py +++ b/lib/tenants/tests/conftest.py @@ -4,10 +4,9 @@ import pytest from fastapi import Depends, FastAPI from fastapi.testclient import TestClient -from usage_example.jwt_generator import create_access_token - from src import TenantData from src.dependency import get_tenant_info +from usage_example.jwt_generator import create_access_token SECRET_KEY = "test_secret_key" diff --git a/lib/tenants/tests/test_dependency_hs256.py b/lib/tenants/tests/test_dependency_hs256.py index 8b71bc0b5..ef441cad5 100644 --- a/lib/tenants/tests/test_dependency_hs256.py +++ b/lib/tenants/tests/test_dependency_hs256.py @@ -1,8 +1,5 @@ -from src.dependency import ( - TenantDependencyBase, - TenantDependencyDocs, - get_tenant_info, -) +from src.dependency import (TenantDependencyBase, TenantDependencyDocs, + get_tenant_info) CURRENT_TENANT = "tenant1" diff --git a/lib/tenants/tests/test_dependency_rs256.py b/lib/tenants/tests/test_dependency_rs256.py index b7f8280dc..4e0198e05 100644 --- a/lib/tenants/tests/test_dependency_rs256.py +++ b/lib/tenants/tests/test_dependency_rs256.py @@ -1,8 +1,5 @@ -from src.dependency import ( - TenantDependencyBase, - TenantDependencyDocs, - get_tenant_info, -) +from src.dependency import (TenantDependencyBase, TenantDependencyDocs, + get_tenant_info) CURRENT_TENANT = "tenant1" diff --git a/lib/tenants/tests/test_schema.py b/lib/tenants/tests/test_schema.py index c79257cec..d2e89148d 100644 --- a/lib/tenants/tests/test_schema.py +++ b/lib/tenants/tests/test_schema.py @@ -1,6 +1,5 @@ import pytest from pydantic import ValidationError - from src.schema import SupportedAlgorithms, TenantData diff --git a/models/Dockerfile b/models/Dockerfile index 66bf45704..11f35d66a 100644 --- a/models/Dockerfile +++ b/models/Dockerfile @@ -20,7 +20,7 @@ FROM base AS build WORKDIR /working -COPY src /working/src +COPY models /working/models COPY alembic /working/alembic COPY alembic.ini /working @@ -56,7 +56,7 @@ RUN python3 -m pytest --cov=src tests/ -m "not integration" FROM sonarsource/sonar-scanner-cli:4.6 AS sonar COPY tests /working/tests -COPY src /working/src +COPY models /working/models COPY sonar-project.properties /working/sonar-project.properties CMD sonar-scanner \ diff --git a/models/alembic/env.py b/models/alembic/env.py index 984d6dd0e..c3d0384de 100644 --- a/models/alembic/env.py +++ b/models/alembic/env.py @@ -1,12 +1,12 @@ import os from logging.config import fileConfig +from alembic import context from sqlalchemy import engine_from_config, pool -from alembic import context -from src.constants import DATABASE_URL -from src.db import Base -from src.utils import get_test_db_url +from models.constants import DATABASE_URL +from models.db import Base +from models.utils import get_test_db_url # this is the Alembic Config object, which provides # access to the values within the .ini file in use. diff --git a/models/alembic/versions/0c3e4fd362de_add_description_field_to_model.py b/models/alembic/versions/0c3e4fd362de_add_description_field_to_model.py index a3e04f982..92751944d 100644 --- a/models/alembic/versions/0c3e4fd362de_add_description_field_to_model.py +++ b/models/alembic/versions/0c3e4fd362de_add_description_field_to_model.py @@ -6,7 +6,6 @@ """ import sqlalchemy as sa - from alembic import op # revision identifiers, used by Alembic. diff --git a/models/alembic/versions/5c3092bc3517_add_columns_to_basement.py b/models/alembic/versions/5c3092bc3517_add_columns_to_basement.py index 79a891592..79146f5aa 100644 --- a/models/alembic/versions/5c3092bc3517_add_columns_to_basement.py +++ b/models/alembic/versions/5c3092bc3517_add_columns_to_basement.py @@ -6,7 +6,6 @@ """ import sqlalchemy as sa - from alembic import op # revision identifiers, used by Alembic. diff --git a/models/alembic/versions/61787083221a_added_archive_field_to_training_model.py b/models/alembic/versions/61787083221a_added_archive_field_to_training_model.py index f4f40b469..26d9c8b94 100644 --- a/models/alembic/versions/61787083221a_added_archive_field_to_training_model.py +++ b/models/alembic/versions/61787083221a_added_archive_field_to_training_model.py @@ -6,7 +6,6 @@ """ import sqlalchemy as sa - from alembic import op # revision identifiers, used by Alembic. diff --git a/models/alembic/versions/683f401ed33e_create_tables.py b/models/alembic/versions/683f401ed33e_create_tables.py index a1829eb95..dba89c8e7 100644 --- a/models/alembic/versions/683f401ed33e_create_tables.py +++ b/models/alembic/versions/683f401ed33e_create_tables.py @@ -6,9 +6,8 @@ """ import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - from alembic import op +from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision = "683f401ed33e" diff --git a/models/alembic/versions/6dc508050bca_add_annotation_dataset_to_training.py b/models/alembic/versions/6dc508050bca_add_annotation_dataset_to_training.py index 9907bd9df..f2717a143 100644 --- a/models/alembic/versions/6dc508050bca_add_annotation_dataset_to_training.py +++ b/models/alembic/versions/6dc508050bca_add_annotation_dataset_to_training.py @@ -6,7 +6,6 @@ """ import sqlalchemy as sa - from alembic import op # revision identifiers, used by Alembic. diff --git a/models/alembic/versions/826680104247_pod_limits_column.py b/models/alembic/versions/826680104247_pod_limits_column.py index 770f778d9..6050b8d29 100644 --- a/models/alembic/versions/826680104247_pod_limits_column.py +++ b/models/alembic/versions/826680104247_pod_limits_column.py @@ -8,9 +8,8 @@ from json import dumps import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - from alembic import op +from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision = "826680104247" diff --git a/models/alembic/versions/abeff4c79fd3_modify_basement_and_training.py b/models/alembic/versions/abeff4c79fd3_modify_basement_and_training.py index 3f83181a2..9ec0895b8 100644 --- a/models/alembic/versions/abeff4c79fd3_modify_basement_and_training.py +++ b/models/alembic/versions/abeff4c79fd3_modify_basement_and_training.py @@ -6,9 +6,8 @@ """ import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - from alembic import op +from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision = "abeff4c79fd3" diff --git a/models/alembic/versions/b4c5225515f1_add_latest_and_version_columns_to_model.py b/models/alembic/versions/b4c5225515f1_add_latest_and_version_columns_to_model.py index f6d970c46..3bef41178 100644 --- a/models/alembic/versions/b4c5225515f1_add_latest_and_version_columns_to_model.py +++ b/models/alembic/versions/b4c5225515f1_add_latest_and_version_columns_to_model.py @@ -6,7 +6,6 @@ """ import sqlalchemy as sa - from alembic import op # revision identifiers, used by Alembic. diff --git a/models/alembic/versions/c769f68f00d4_add_field_type_to_table_model.py b/models/alembic/versions/c769f68f00d4_add_field_type_to_table_model.py index a826bbd03..d3025066f 100644 --- a/models/alembic/versions/c769f68f00d4_add_field_type_to_table_model.py +++ b/models/alembic/versions/c769f68f00d4_add_field_type_to_table_model.py @@ -6,7 +6,6 @@ """ import sqlalchemy as sa - from alembic import op # revision identifiers, used by Alembic. diff --git a/models/docker-compose.yaml b/models/docker-compose.yaml index 1c233d56e..ecd5021ee 100644 --- a/models/docker-compose.yaml +++ b/models/docker-compose.yaml @@ -11,7 +11,7 @@ services: - '9000:9000' - '9001:9001' env_file: - - src/.env + - models/.env networks: - app-tier @@ -20,7 +20,7 @@ services: ports: - 5432:5432 env_file: - - src/.env + - models/.env networks: - app-tier @@ -30,7 +30,7 @@ services: target: build container_name: models_web_app env_file: - - src/.env + - models/.env volumes: - .:/working ports: diff --git a/models/src/colab_ssh_utils.py b/models/models/colab_ssh_utils.py similarity index 95% rename from models/src/colab_ssh_utils.py rename to models/models/colab_ssh_utils.py index 85ed605c7..55c40d843 100644 --- a/models/src/colab_ssh_utils.py +++ b/models/models/colab_ssh_utils.py @@ -8,9 +8,9 @@ from paramiko import AutoAddPolicy, SSHClient from paramiko.ssh_exception import SSHException -from src.constants import MINIO_ACCESS_KEY, MINIO_HOST, MINIO_SECRET_KEY -from src.errors import ColabFileUploadError -from src.schemas import TrainingCredentials +from models.constants import MINIO_ACCESS_KEY, MINIO_HOST, MINIO_SECRET_KEY +from models.errors import ColabFileUploadError +from models.schemas import TrainingCredentials LOGGER = logging.getLogger(name="models") COLAB_TRAINING_DIRECTORY = "/content/training/" diff --git a/models/src/colab_tunnel_script.ipynb b/models/models/colab_tunnel_script.ipynb similarity index 100% rename from models/src/colab_tunnel_script.ipynb rename to models/models/colab_tunnel_script.ipynb diff --git a/models/src/constants.py b/models/models/constants.py similarity index 97% rename from models/src/constants.py rename to models/models/constants.py index d9526f040..998b6c2d9 100644 --- a/models/src/constants.py +++ b/models/models/constants.py @@ -3,7 +3,7 @@ from dotenv import find_dotenv, load_dotenv -load_dotenv(find_dotenv("./src/.env")) +load_dotenv(find_dotenv("./models/.env")) POSTGRES_USER = os.environ.get("POSTGRES_USER") POSTGRES_PASSWORD = os.environ.get("POSTGRES_PASSWORD") diff --git a/models/src/convert_utils.py b/models/models/convert_utils.py similarity index 91% rename from models/src/convert_utils.py rename to models/models/convert_utils.py index 62c7010d4..ca7e01f48 100644 --- a/models/src/convert_utils.py +++ b/models/models/convert_utils.py @@ -6,8 +6,8 @@ from fastapi import HTTPException from requests import ConnectionError, RequestException, Timeout -from src.constants import CONVERT_EXPORT_URL, HEADER_TENANT -from src.schemas import ConvertRequestSchema +from models.constants import CONVERT_EXPORT_URL, HEADER_TENANT +from models.schemas import ConvertRequestSchema LOGGER = logging.getLogger(name="models") diff --git a/models/src/crud.py b/models/models/crud.py similarity index 94% rename from models/src/crud.py rename to models/models/crud.py index ed80af9da..5dc1ea0bb 100644 --- a/models/src/crud.py +++ b/models/models/crud.py @@ -3,8 +3,9 @@ from sqlalchemy import desc from sqlalchemy.orm import Session -from src.db import Basement, Model, Training -from src.schemas import BasementBase, ModelBase, TrainingBase, TrainingUpdate +from models.db import Basement, Model, Training +from models.schemas import (BasementBase, ModelBase, TrainingBase, + TrainingUpdate) def is_id_existing( diff --git a/models/src/db.py b/models/models/db.py similarity index 94% rename from models/src/db.py rename to models/models/db.py index 739e370c4..a59105b7e 100644 --- a/models/src/db.py +++ b/models/models/db.py @@ -1,24 +1,15 @@ import datetime import enum -from sqlalchemy import ( - VARCHAR, - Boolean, - Column, - DateTime, - Enum, - ForeignKey, - Integer, - String, - create_engine, -) +from sqlalchemy import (VARCHAR, Boolean, Column, DateTime, Enum, ForeignKey, + Integer, String, create_engine) from sqlalchemy.dialects.postgresql import DOUBLE_PRECISION, JSON from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.ext.mutable import MutableDict, MutableList from sqlalchemy.orm import Session, relationship, sessionmaker from sqlalchemy.types import ARRAY -from src.constants import DATABASE_URL +from models.constants import DATABASE_URL Base = declarative_base() diff --git a/models/src/errors.py b/models/models/errors.py similarity index 100% rename from models/src/errors.py rename to models/models/errors.py diff --git a/models/src/logger.py b/models/models/logger.py similarity index 100% rename from models/src/logger.py rename to models/models/logger.py diff --git a/models/src/main.py b/models/models/main.py similarity index 75% rename from models/src/main.py rename to models/models/main.py index 0813dc2d5..7d7d1230b 100644 --- a/models/src/main.py +++ b/models/models/main.py @@ -8,24 +8,17 @@ from paramiko.ssh_exception import SSHException from sqlalchemy.exc import SQLAlchemyError -from src.constants import API_NAME, API_VERSION, ROOT_PATH -from src.errors import ( - ColabFileUploadError, - NoSuchTenant, - botocore_error_handler, - colab_execution_error_handler, - minio_client_error_handler, - minio_no_such_bucket_error_handler, - sqlalchemy_db_error_handler, - ssh_connection_error_handler, - subprocess_called_error_handler, -) -from src.routers import ( - basements_routers, - deployed_models_routers, - models_routers, - training_routers, -) +from models.constants import API_NAME, API_VERSION, ROOT_PATH +from models.errors import (ColabFileUploadError, NoSuchTenant, + botocore_error_handler, + colab_execution_error_handler, + minio_client_error_handler, + minio_no_such_bucket_error_handler, + sqlalchemy_db_error_handler, + ssh_connection_error_handler, + subprocess_called_error_handler) +from models.routers import (basements_routers, deployed_models_routers, + models_routers, training_routers) LOGGER = logging.getLogger(name=API_NAME) LOGGING_FORMAT = "[%(asctime)s] - [%(name)s] - [%(levelname)s] - [%(message)s]" diff --git a/models/src/routers/__init__.py b/models/models/routers/__init__.py similarity index 67% rename from models/src/routers/__init__.py rename to models/models/routers/__init__.py index 9973914dc..fe4a7429a 100644 --- a/models/src/routers/__init__.py +++ b/models/models/routers/__init__.py @@ -1,5 +1,5 @@ from tenant_dependency import get_tenant_info -from src.constants import ALGORITHM, KEYCLOACK_URI +from models.constants import ALGORITHM, KEYCLOACK_URI tenant = get_tenant_info(url=KEYCLOACK_URI, algorithm=ALGORITHM) diff --git a/models/src/routers/basements_routers.py b/models/models/routers/basements_routers.py similarity index 93% rename from models/src/routers/basements_routers.py rename to models/models/routers/basements_routers.py index dd0814a51..6dc9f4202 100644 --- a/models/src/routers/basements_routers.py +++ b/models/models/routers/basements_routers.py @@ -2,25 +2,16 @@ from typing import Any, Dict, Optional, Union from fastapi import APIRouter, Depends, File, Header, HTTPException, UploadFile -from filter_lib import ( - Page, - create_filter_model, - form_query, - map_request_to_filter, - paginate, -) +from filter_lib import (Page, create_filter_model, form_query, + map_request_to_filter, paginate) from sqlalchemy.orm import Session from tenant_dependency import TenantData -from src import crud, schemas -from src.db import Basement, get_db -from src.routers import tenant -from src.utils import ( - NoSuchTenant, - convert_bucket_name_if_s3prefix, - get_minio_resource, - upload_to_object_storage, -) +from models import crud, schemas +from models.db import Basement, get_db +from models.routers import tenant +from models.utils import (NoSuchTenant, convert_bucket_name_if_s3prefix, + get_minio_resource, upload_to_object_storage) LOGGER = logging.getLogger(name="models") @@ -32,6 +23,7 @@ @router.post( "/create", status_code=201, + response_model=None, responses={ 201: { "model": schemas.Basement, @@ -95,6 +87,7 @@ def search_basements( @router.get( "/{basements_id}", + response_model=None, responses={ 200: { "model": schemas.Basement, @@ -118,6 +111,7 @@ def get_basement_by_id( @router.put( "/update", + response_model=None, responses={ 200: { "model": schemas.Basement, diff --git a/models/src/routers/deployed_models_routers.py b/models/models/routers/deployed_models_routers.py similarity index 98% rename from models/src/routers/deployed_models_routers.py rename to models/models/routers/deployed_models_routers.py index 36c7f1f93..8ca563f03 100644 --- a/models/src/routers/deployed_models_routers.py +++ b/models/models/routers/deployed_models_routers.py @@ -6,8 +6,8 @@ from kubernetes import client, config from kubernetes.client.exceptions import ApiException -from src import schemas, utils -from src.constants import MODELS_NAMESPACE +from models import schemas, utils +from models.constants import MODELS_NAMESPACE router = APIRouter(prefix="/deployed_models", tags=["deployed_models"]) LOGGER = logging.getLogger(name="models") diff --git a/models/src/routers/models_routers.py b/models/models/routers/models_routers.py similarity index 97% rename from models/src/routers/models_routers.py rename to models/models/routers/models_routers.py index 8b1d9d5e7..2e73d294e 100644 --- a/models/src/routers/models_routers.py +++ b/models/models/routers/models_routers.py @@ -2,20 +2,15 @@ from typing import Any, Dict, Union from fastapi import APIRouter, Depends, Header, HTTPException, Path -from filter_lib import ( - Page, - create_filter_model, - form_query, - map_request_to_filter, - paginate, -) +from filter_lib import (Page, create_filter_model, form_query, + map_request_to_filter, paginate) from sqlalchemy.orm import Session from tenant_dependency import TenantData -from src import crud, schemas, utils -from src.crud import get_latest_model, get_second_latest_model -from src.db import Basement, Model, Training, get_db -from src.routers import tenant +from models import crud, schemas, utils +from models.crud import get_latest_model, get_second_latest_model +from models.db import Basement, Model, Training, get_db +from models.routers import tenant LOGGER = logging.getLogger(name="models") @@ -108,15 +103,14 @@ def search_models( session: Session = Depends(get_db), ) -> Union[Page[schemas.Model], Page[Any]]: query = session.query(Model) - filter_args = map_request_to_filter( - request.dict(), "Model" # type: ignore - ) + filter_args = map_request_to_filter(request.dict(), "Model") # type: ignore query, pagination = form_query(filter_args, query) return paginate([x for x in query], pagination) @router.get( "/{models_id}", + response_model=None, responses={ 200: { "model": schemas.Model, @@ -169,6 +163,7 @@ def get_model_by_id_and_version( @router.put( "/update", + response_model=None, responses={ 200: { "model": schemas.Model, diff --git a/models/src/routers/training_routers.py b/models/models/routers/training_routers.py similarity index 92% rename from models/src/routers/training_routers.py rename to models/models/routers/training_routers.py index 7b042fc15..9ba7484d7 100644 --- a/models/src/routers/training_routers.py +++ b/models/models/routers/training_routers.py @@ -3,45 +3,24 @@ import tempfile from typing import Any, Dict, Union -from fastapi import ( - APIRouter, - Depends, - File, - Header, - HTTPException, - Path, - Response, - UploadFile, - status, -) -from filter_lib import ( - Page, - create_filter_model, - form_query, - map_request_to_filter, - paginate, -) +from fastapi import (APIRouter, Depends, File, Header, HTTPException, Path, + Response, UploadFile, status) +from filter_lib import (Page, create_filter_model, form_query, + map_request_to_filter, paginate) from sqlalchemy.orm import Session from tenant_dependency import TenantData -from src import crud, schemas, utils -from src.colab_ssh_utils import ( - COLAB_TRAINING_DIRECTORY, - check_aws_credentials_file, - connect_colab, - local_mount_colab_drive, - sync_colab_with_minio, - upload_file_to_colab, -) -from src.convert_utils import prepare_dataset_info -from src.db import Basement, Training, get_db -from src.routers import tenant -from src.utils import ( - NoSuchTenant, - convert_bucket_name_if_s3prefix, - get_minio_object, - get_minio_resource, -) +from models import crud, schemas, utils +from models.colab_ssh_utils import (COLAB_TRAINING_DIRECTORY, + check_aws_credentials_file, connect_colab, + local_mount_colab_drive, + sync_colab_with_minio, + upload_file_to_colab) +from models.convert_utils import prepare_dataset_info +from models.db import Basement, Training, get_db +from models.routers import tenant +from models.utils import (NoSuchTenant, convert_bucket_name_if_s3prefix, + get_minio_object, get_minio_resource) LOGGER = logging.getLogger(name="models") TRAINING_SCRIPT_NAME = "training_script.py" @@ -55,6 +34,7 @@ @router.post( "/create", status_code=201, + response_model=None, responses={ 201: { "model": schemas.Training, @@ -144,15 +124,14 @@ def search_training( session: Session = Depends(get_db), ) -> Union[Page[schemas.Training], Page[Any]]: query = session.query(Training) - filter_args = map_request_to_filter( - request.dict(), "Training" # type: ignore - ) + filter_args = map_request_to_filter(request.dict(), "Training") # type: ignore query, pagination = form_query(filter_args, query) return paginate([x for x in query], pagination) @router.get( "/{trainings_id}", + response_model=None, responses={ 200: { "model": schemas.Training, @@ -176,6 +155,7 @@ def get_training_by_id( @router.put( "/update", + response_model=None, responses={ 200: { "model": schemas.Training, diff --git a/models/src/schemas.py b/models/models/schemas.py similarity index 99% rename from models/src/schemas.py rename to models/models/schemas.py index 3db682d57..71f5e13f6 100644 --- a/models/src/schemas.py +++ b/models/models/schemas.py @@ -5,7 +5,7 @@ from pydantic import BaseModel, ConstrainedStr, Field, PositiveInt, validator -from src.db import StatusEnum +from models.db import StatusEnum class AtLeastOneChar(ConstrainedStr): diff --git a/models/src/utils.py b/models/models/utils.py similarity index 96% rename from models/src/utils.py rename to models/models/utils.py index d82e17108..7c2977d88 100644 --- a/models/src/utils.py +++ b/models/models/utils.py @@ -10,24 +10,15 @@ from sqlalchemy.orm import Session from starlette.datastructures import UploadFile -import src.logger as logger -from src.constants import ( - CONTAINER_NAME, - DOCKER_REGISTRY_URL, - DOMAIN_NAME, - INFERENCE_HOST, - INFERENCE_PORT, - MINIO_ACCESS_KEY, - MINIO_HOST, - MINIO_PUBLIC_HOST, - MINIO_SECRET_KEY, - MODELS_NAMESPACE, - S3_CREDENTIALS_PROVIDER, - S3_PREFIX, -) -from src.db import Basement, Model -from src.errors import NoSuchTenant -from src.schemas import DeployedModelPod, MinioHTTPMethod +import models.logger as logger +from models.constants import (CONTAINER_NAME, DOCKER_REGISTRY_URL, DOMAIN_NAME, + INFERENCE_HOST, INFERENCE_PORT, MINIO_ACCESS_KEY, + MINIO_HOST, MINIO_PUBLIC_HOST, MINIO_SECRET_KEY, + MODELS_NAMESPACE, S3_CREDENTIALS_PROVIDER, + S3_PREFIX) +from models.db import Basement, Model +from models.errors import NoSuchTenant +from models.schemas import DeployedModelPod, MinioHTTPMethod logger_ = logger.get_logger(__name__) diff --git a/models/src/.env b/models/src/.env deleted file mode 100644 index 34a66ca3b..000000000 --- a/models/src/.env +++ /dev/null @@ -1,29 +0,0 @@ -POSTGRES_USER=admin -POSTGRES_PASSWORD=admin -POSTGRES_DB=models -POSTGRES_HOST=db -POSTGRES_PORT=5432 -DATABASE_URL="postgresql+psycopg2://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}" - -MINIO_HOST="minio:9000" -MINIO_PUBLIC_HOST=${MINIO_HOST} -MINIO_ACCESS_KEY="minio" -MINIO_SECRET_KEY="minio123" -MINIO_ROOT_USER=${MINIO_ACCESS_KEY} -MINIO_ROOT_PASSWORD=${MINIO_SECRET_KEY} -S3_PREFIX= -S3_CREDENTIALS_PROVIDER=minio - -DOCKER_REGISTRY_URL="localhost:5000" - -SECRET=some_secret_key - -DOCKER_REGISTRY_URL="10.228.0.184:5000" -MODELS_NAMESPACE=dev2 -ROOT_PATH="" -DOMAIN_NAME="badgerdoc.com" - -KEYCLOACK_URI="http://bagerdoc-keycloack" -ALGORITHM="RS256" - -CONVERT_EXPORT_URL="http://convert/export" diff --git a/models/tests/conftest.py b/models/tests/conftest.py index 5d0c7a48f..287dade2e 100644 --- a/models/tests/conftest.py +++ b/models/tests/conftest.py @@ -6,6 +6,7 @@ import boto3 import pytest +from alembic import command from botocore.config import Config from botocore.exceptions import ClientError from fastapi.testclient import TestClient @@ -15,27 +16,19 @@ from sqlalchemy.orm import Session, sessionmaker from sqlalchemy_utils import create_database, database_exists -from alembic import command -from src.constants import ( - DATABASE_URL, - MINIO_ACCESS_KEY, - MINIO_HOST, - MINIO_SECRET_KEY, -) -from src.db import Base, Basement, Training, get_db -from src.main import app -from src.routers import tenant -from src.utils import get_test_db_url +from models.constants import (DATABASE_URL, MINIO_ACCESS_KEY, MINIO_HOST, + MINIO_SECRET_KEY) +from models.db import Base, Basement, Training, get_db +from models.main import app +from models.routers import tenant +from models.utils import get_test_db_url from .override_app_dependency import override -from .test_colab_start_training import ( - BASEMENT_ID, - EXIST_TRAINING_ID, - TRAINING_ARCHIVE_DATA, - TRAINING_ARCHIVE_KEY, - TRAINING_SCRIPT_DATA, - TRAINING_SCRIPT_KEY, -) +from .test_colab_start_training import (BASEMENT_ID, EXIST_TRAINING_ID, + TRAINING_ARCHIVE_DATA, + TRAINING_ARCHIVE_KEY, + TRAINING_SCRIPT_DATA, + TRAINING_SCRIPT_KEY) from .test_crud import GET_BASEMENT, GET_LATEST_MODELS, GET_TRAINING from .test_utils import TEST_LIMITS, TEST_TENANT diff --git a/models/tests/test_basement_routers.py b/models/tests/test_basement_routers.py index cd2b4ed55..6d4ec46f4 100644 --- a/models/tests/test_basement_routers.py +++ b/models/tests/test_basement_routers.py @@ -4,12 +4,12 @@ import pytest from fastapi.exceptions import HTTPException from fastapi.testclient import TestClient - -from src.db import Basement -from src.main import app -from src.routers import basements_routers from tests.test_utils import TEST_LIMITS +from models.db import Basement +from models.main import app +from models.routers import basements_routers + @pytest.fixture(scope="function") def client(): @@ -85,7 +85,7 @@ def test_get_basement_by_id_withot_basement(get): @patch.object(basements_routers.crud, "get_instance") def test_delete_basement_by_id(delete, get, client, monkeypatch): monkeypatch.setattr( - "src.routers.basements_routers.get_minio_resource", Mock() + "models.routers.basements_routers.get_minio_resource", Mock() ) data = {"id": "id"} get.return_value = "expected" @@ -98,7 +98,7 @@ def test_delete_basement_by_id(delete, get, client, monkeypatch): @patch.object(basements_routers.crud, "get_instance") def test_delete_basement_by_id_calls_crud(delete, get, monkeypatch): monkeypatch.setattr( - "src.routers.basements_routers.get_minio_resource", Mock() + "models.routers.basements_routers.get_minio_resource", Mock() ) data = basements_routers.schemas.BasementDelete(id="id") get.return_value = "expected" diff --git a/models/tests/test_colab_interactions.py b/models/tests/test_colab_interactions.py index c05e806fa..8722d5c74 100644 --- a/models/tests/test_colab_interactions.py +++ b/models/tests/test_colab_interactions.py @@ -3,12 +3,9 @@ import pytest -from src.colab_ssh_utils import ( - COLAB_TRAINING_DIRECTORY, - connect_colab, - upload_file_to_colab, -) -from src.errors import ColabFileUploadError +from models.colab_ssh_utils import (COLAB_TRAINING_DIRECTORY, connect_colab, + upload_file_to_colab) +from models.errors import ColabFileUploadError TEST_FILE_NAME = "test_file.py" TEST_CREDENTIALS = { @@ -47,11 +44,11 @@ def test_connect_colab_called_with_credentials(monkeypatch) -> None: mock_ssh = mock.Mock(return_value=mock_client) mock_policy = mock.Mock() monkeypatch.setattr( - "src.colab_ssh_utils.SSHClient", + "models.colab_ssh_utils.SSHClient", mock_ssh, ) monkeypatch.setattr( - "src.colab_ssh_utils.AutoAddPolicy", + "models.colab_ssh_utils.AutoAddPolicy", mock.Mock(return_value=mock_policy), ) mock_client.connect = mock.Mock(return_value=1) diff --git a/models/tests/test_colab_start_training.py b/models/tests/test_colab_start_training.py index 7f53623a9..a359f97b0 100644 --- a/models/tests/test_colab_start_training.py +++ b/models/tests/test_colab_start_training.py @@ -5,11 +5,8 @@ from paramiko.ssh_exception import SSHException from sqlalchemy.exc import SQLAlchemyError -from .override_app_dependency import ( - OTHER_TENANT_HEADER, - TEST_HEADER, - TEST_TENANTS, -) +from .override_app_dependency import (OTHER_TENANT_HEADER, TEST_HEADER, + TEST_TENANTS) TEST_CREDENTIALS = { "host": "test_host", @@ -32,7 +29,7 @@ def test_start_training_db_error(monkeypatch, overrided_token_client) -> None: """Test handling of db connection errors""" monkeypatch.setattr( - "src.crud.Session.query", + "models.crud.Session.query", Mock(side_effect=SQLAlchemyError("some error message")), ) response = overrided_token_client.post( @@ -79,7 +76,9 @@ def test_start_training_no_key_script_error( @pytest.mark.integration -@pytest.mark.skip("Test should be fixed - got 'Annotation dataset for training 1 not ready' in response") +@pytest.mark.skip( + "Test should be fixed - got 'Annotation dataset for training 1 not ready' in response" +) @pytest.mark.parametrize( "prepare_db_start_training", [TRAINING_ARCHIVE_KEY], indirect=True ) @@ -90,7 +89,7 @@ def test_start_training_colab_connection_error( description in message. """ monkeypatch.setattr( - "src.routers.training_routers.connect_colab", + "models.routers.training_routers.connect_colab", Mock(side_effect=SSHException("some ssh error")), ) response = overrided_token_client.post( @@ -114,7 +113,9 @@ def __exit__(self, exc_type, exc_val, exc_tb): @pytest.mark.integration -@pytest.mark.skip("Test should be fixed - got 'Annotation dataset for training 1 not ready' in response") +@pytest.mark.skip( + "Test should be fixed - got 'Annotation dataset for training 1 not ready' in response" +) @pytest.mark.parametrize( "prepare_db_start_training", [TRAINING_ARCHIVE_KEY], indirect=True ) @@ -129,11 +130,11 @@ def test_start_training_no_such_bucket_error( """ other_tenant = TEST_TENANTS[1] monkeypatch.setattr( - "src.utils.boto3.resource", + "models.utils.boto3.resource", Mock(return_value=moto_minio), ) monkeypatch.setattr( - "src.routers.training_routers.connect_colab", MockSSHContext + "models.routers.training_routers.connect_colab", MockSSHContext ) response = overrided_token_client.post( START_TRAINING_PATH.format(EXIST_TRAINING_ID), @@ -145,7 +146,9 @@ def test_start_training_no_such_bucket_error( @pytest.mark.integration -@pytest.mark.skip("Test should be fixed - got 'Annotation dataset for training 1 not ready' in response") +@pytest.mark.skip( + "Test should be fixed - got 'Annotation dataset for training 1 not ready' in response" +) @pytest.mark.parametrize( "prepare_db_start_training", [TRAINING_ARCHIVE_KEY], @@ -158,11 +161,11 @@ def test_start_training_boto3_error( return 500 status with error description in message. """ monkeypatch.setattr( - "src.routers.training_routers.get_minio_object", + "models.routers.training_routers.get_minio_object", Mock(side_effect=BotoCoreError()), ) monkeypatch.setattr( - "src.routers.training_routers.connect_colab", MockSSHContext + "models.routers.training_routers.connect_colab", MockSSHContext ) response = overrided_token_client.post( START_TRAINING_PATH.format(EXIST_TRAINING_ID), @@ -174,7 +177,9 @@ def test_start_training_boto3_error( @pytest.mark.integration -@pytest.mark.skip("Test should be fixed - got 'Annotation dataset for training 1 not ready' in response") +@pytest.mark.skip( + "Test should be fixed - got 'Annotation dataset for training 1 not ready' in response" +) @pytest.mark.parametrize( "prepare_db_start_training", [TRAINING_ARCHIVE_KEY], @@ -191,14 +196,14 @@ def test_start_training_integration( """ mock_upload = Mock() monkeypatch.setattr( - "src.routers.training_routers.upload_file_to_colab", mock_upload + "models.routers.training_routers.upload_file_to_colab", mock_upload ) monkeypatch.setattr( - "src.utils.boto3.resource", + "models.utils.boto3.resource", Mock(return_value=save_start_training_minio_objects), ) monkeypatch.setattr( - "src.routers.training_routers.connect_colab", MockSSHContext + "models.routers.training_routers.connect_colab", MockSSHContext ) response = overrided_token_client.post( START_TRAINING_PATH.format(EXIST_TRAINING_ID), diff --git a/models/tests/test_crud.py b/models/tests/test_crud.py index bf93882f7..68ab0b252 100644 --- a/models/tests/test_crud.py +++ b/models/tests/test_crud.py @@ -1,14 +1,14 @@ from unittest.mock import Mock from pytest import mark - -from src import crud -from src.crud import get_instance, get_latest_model -from src.db import Basement, Model, StatusEnum, Training -from src.schemas import BasementBase from tests.test_utils import TEST_LIMITS from tests.utils import create_expected_models, delete_date_field, row_to_dict +from models import crud +from models.crud import get_instance, get_latest_model +from models.db import Basement, Model, StatusEnum, Training +from models.schemas import BasementBase + GET_BASEMENT = Basement( id="base_id", name="basement_name", gpu_support=True, limits=TEST_LIMITS ) diff --git a/models/tests/test_deployed_models_routers.py b/models/tests/test_deployed_models_routers.py index 18587123c..256908b6f 100644 --- a/models/tests/test_deployed_models_routers.py +++ b/models/tests/test_deployed_models_routers.py @@ -3,8 +3,8 @@ import pytest from kubernetes.client.exceptions import ApiException -from src.routers import deployed_models_routers -from src.schemas import DeployedModelPod +from models.routers import deployed_models_routers +from models.schemas import DeployedModelPod def test_get_deployed_model_list_returns_list_of_models(client): diff --git a/models/tests/test_models_routers.py b/models/tests/test_models_routers.py index 96181a3ee..2055b2246 100644 --- a/models/tests/test_models_routers.py +++ b/models/tests/test_models_routers.py @@ -4,14 +4,14 @@ import pytest from fastapi.exceptions import HTTPException from fastapi.testclient import TestClient - -from src.db import Basement, Model, StatusEnum -from src.main import app -from src.routers import models_routers from tests.override_app_dependency import TEST_HEADER, TEST_TENANTS from tests.test_crud import GET_BASEMENT from tests.utils import create_expected_models, delete_date_field, row_to_dict +from models.db import Basement, Model, StatusEnum +from models.main import app +from models.routers import models_routers + @pytest.fixture(scope="function") def client(): diff --git a/models/tests/test_schemas.py b/models/tests/test_schemas.py index 22d8a833e..a9fb17ab1 100644 --- a/models/tests/test_schemas.py +++ b/models/tests/test_schemas.py @@ -1,9 +1,9 @@ import pytest from pydantic import ValidationError - -from src import schemas from tests.test_utils import TEST_LIMITS +from models import schemas + def test_empty_id_in_modelbase_raises_error(): minio_path = {"file": "file", "bucket": "bucket"} diff --git a/models/tests/test_trainings_routers.py b/models/tests/test_trainings_routers.py index 6517739f1..42e148111 100644 --- a/models/tests/test_trainings_routers.py +++ b/models/tests/test_trainings_routers.py @@ -5,9 +5,9 @@ from fastapi.exceptions import HTTPException from fastapi.testclient import TestClient -from src.db import Basement, Training -from src.main import app -from src.routers import training_routers +from models.db import Basement, Training +from models.main import app +from models.routers import training_routers from .override_app_dependency import TEST_HEADER @@ -99,7 +99,7 @@ def test_get_training_by_id_withot_training(get): @patch.object(training_routers.crud, "delete_instance") @patch.object(training_routers.crud, "get_instance") -@patch("src.routers.training_routers.get_minio_resource", Mock()) +@patch("models.routers.training_routers.get_minio_resource", Mock()) def test_delete_training_by_id(get, delete, client): data = {"id": 1} training_routers.get_db = Mock() @@ -113,7 +113,7 @@ def test_delete_training_by_id(get, delete, client): @patch.object(training_routers.crud, "delete_instance") @patch.object(training_routers.crud, "get_instance") -@patch("src.routers.training_routers.get_minio_resource", Mock()) +@patch("models.routers.training_routers.get_minio_resource", Mock()) def test_delete_training_by_id_calls_crud(get, delete): data = training_routers.schemas.TrainingDelete(id=1) db_entity = Mock() diff --git a/models/tests/test_utils.py b/models/tests/test_utils.py index 41e2f1ede..e488234ee 100644 --- a/models/tests/test_utils.py +++ b/models/tests/test_utils.py @@ -8,16 +8,11 @@ from botocore.exceptions import ClientError from kubernetes.client.rest import ApiException -from src import utils -from src.constants import MINIO_HOST -from src.errors import NoSuchTenant -from src.schemas import ( - BasementBase, - DeployedModelPod, - MinioHTTPMethod, - MinioPath, - Model, -) +from models import utils +from models.constants import MINIO_HOST +from models.errors import NoSuchTenant +from models.schemas import (BasementBase, DeployedModelPod, MinioHTTPMethod, + MinioPath, Model) TEST_TENANT = "test" TEST_LIMITS = { @@ -34,7 +29,7 @@ def test_minio_no_such_bucket_error_handling(moto_minio, monkeypatch): wrong_tenant = "wrong_tenant" error_message = f"Bucket {wrong_tenant} does not exist" monkeypatch.setattr( - "src.utils.boto3.resource", + "models.utils.boto3.resource", Mock(return_value=moto_minio), ) with pytest.raises(utils.NoSuchTenant, match=error_message): @@ -68,7 +63,7 @@ def test_get_object_via_presigned_url( ): """Tests possibility to GET object via generated presigned URL.""" monkeypatch.setattr( - "src.utils.get_minio_resource", + "models.utils.get_minio_resource", Mock(return_value=save_object_minio), ) presigned_url = utils.generate_presigned_url( @@ -101,7 +96,7 @@ def test_generate_presigned_url( URL with correct key, expiration time and signature-generating algorithm. """ monkeypatch.setattr( - "src.utils.get_minio_resource", + "models.utils.get_minio_resource", Mock(return_value=moto_minio), ) presigned_url = utils.generate_presigned_url( @@ -116,7 +111,7 @@ def test_generate_presigned_url( @pytest.mark.skip("Fails in GitHub Actions for some reason") -@patch("src.utils.MINIO_PUBLIC_HOST", MINIO_HOST) +@patch("models.utils.MINIO_PUBLIC_HOST", MINIO_HOST) @pytest.mark.integration def test_expired_presigned_url(create_minio_bucket): """Tests that http_method actions for minio Object won't be applicable @@ -142,7 +137,7 @@ def test_generate_presigned_url_error(moto_minio, monkeypatch): of boto3 errors. """ monkeypatch.setattr( - "src.utils.get_minio_resource", + "models.utils.get_minio_resource", Mock(return_value=moto_minio), ) presigned_url = utils.generate_presigned_url( @@ -159,7 +154,7 @@ def test_put_object_via_presigned_url(moto_minio, monkeypatch): key = "test_file.json" test_data = {"file_id": 1} monkeypatch.setattr( - "src.utils.get_minio_resource", + "models.utils.get_minio_resource", Mock(return_value=moto_minio), ) presigned_url = utils.generate_presigned_url( @@ -521,7 +516,7 @@ def test_get_minio_object_wrong_tenant(monkeypatch, moto_minio) -> None: 'NoSuchTenant' exception with 'Bucket for tenant does not exist' message. """ monkeypatch.setattr( - "src.utils.boto3.resource", + "models.utils.boto3.resource", Mock(return_value=moto_minio), ) wrong_tenant = "wrong_tenant" @@ -539,7 +534,7 @@ def test_get_minio_object_wrong_key(monkeypatch, save_object_minio) -> None: 'The specified key does not exist' in error message. """ monkeypatch.setattr( - "src.utils.boto3.resource", + "models.utils.boto3.resource", Mock(return_value=save_object_minio), ) with pytest.raises(ClientError, match=r"The specified key does not exist"): @@ -553,7 +548,7 @@ def test_get_minio_object(monkeypatch, save_object_minio) -> None: """Tests that get_minio_object returns correct object with actual size.""" expected_obj = json.dumps({"file_id": 1}) monkeypatch.setattr( - "src.utils.boto3.resource", + "models.utils.boto3.resource", Mock(return_value=save_object_minio), ) data, size = utils.get_minio_object(TEST_TENANT, "file_1.json") diff --git a/models/tests/utils.py b/models/tests/utils.py index 63aebe023..24e453719 100644 --- a/models/tests/utils.py +++ b/models/tests/utils.py @@ -2,7 +2,7 @@ from typing import List, Union from uuid import UUID -from src.db import StatusEnum +from models.db import StatusEnum def create_expected_models( @@ -16,7 +16,7 @@ def create_expected_models( status: StatusEnum = None, tenant: str = None, name: str = None, - description: str = '' + description: str = "", ) -> dict: return { "basement": basement_id, @@ -33,7 +33,7 @@ def create_expected_models( "training_id": training_id, "type": None, "version": version, - "description": description + "description": description, } diff --git a/pipelines/Dockerfile b/pipelines/Dockerfile index 32f6881e9..1e9db464e 100644 --- a/pipelines/Dockerfile +++ b/pipelines/Dockerfile @@ -3,14 +3,14 @@ FROM ${base_image} as build WORKDIR /opt/pipeline_executor -COPY src src +COPY pipelines pipelines COPY alembic alembic COPY alembic.ini wait-for-it.sh .env requirements.txt version.txt setup.py ./ RUN python3 -m pip install --upgrade pip && pip install --no-cache-dir -r requirements.txt && pip install . ENV PYTHONPATH /opt/pipeline_executor -CMD alembic upgrade head && uvicorn src.app:app --host 0.0.0.0 --port 8080 +CMD alembic upgrade head && uvicorn pipelines.app:app --host 0.0.0.0 --port 8080 FROM build AS development @@ -21,7 +21,7 @@ RUN pip install --no-cache-dir -r requirements_dev.txt && python3 -m pip install FROM sonarsource/sonar-scanner-cli:4.6 AS sonar -COPY src /sonar/src +COPY pipelines /sonar/pipelines COPY alembic /sonar/alembic COPY tests /sonar/tests COPY sonar-project.properties /sonar/sonar-project.properties diff --git a/pipelines/alembic/env.py b/pipelines/alembic/env.py index 23995c1ca..2baf268d7 100644 --- a/pipelines/alembic/env.py +++ b/pipelines/alembic/env.py @@ -1,12 +1,12 @@ import os from logging.config import fileConfig +from alembic import context from sqlalchemy import engine_from_config, pool -import src.config as settings -from alembic import context -from src.db.models import Base -from src.db.service import get_test_db_url +import pipelines.config as settings +from pipelines.db.models import Base +from pipelines.db.service import get_test_db_url # this is the Alembic Config object, which provides # access to the values within the .ini file in use. diff --git a/pipelines/alembic/versions/08ad5deb23eb_remove_token_column.py b/pipelines/alembic/versions/08ad5deb23eb_remove_token_column.py index 6d2be02fa..0f34c6d7e 100644 --- a/pipelines/alembic/versions/08ad5deb23eb_remove_token_column.py +++ b/pipelines/alembic/versions/08ad5deb23eb_remove_token_column.py @@ -8,7 +8,6 @@ """ import sqlalchemy as sa - from alembic import op # revision identifiers, used by Alembic. diff --git a/pipelines/alembic/versions/0a53c56436d0_change_task_webhook_to_string.py b/pipelines/alembic/versions/0a53c56436d0_change_task_webhook_to_string.py index 6dd3ebe5d..13564231d 100644 --- a/pipelines/alembic/versions/0a53c56436d0_change_task_webhook_to_string.py +++ b/pipelines/alembic/versions/0a53c56436d0_change_task_webhook_to_string.py @@ -6,10 +6,10 @@ """ import sqlalchemy as sa +from alembic import op from sqlalchemy import orm -from alembic import op -from src.db import models +from pipelines.db import models # revision identifiers, used by Alembic. revision = "0a53c56436d0" diff --git a/pipelines/alembic/versions/0ab5e65cf34b_fix_default_type_to_inference.py b/pipelines/alembic/versions/0ab5e65cf34b_fix_default_type_to_inference.py index a288848c0..9218792ee 100644 --- a/pipelines/alembic/versions/0ab5e65cf34b_fix_default_type_to_inference.py +++ b/pipelines/alembic/versions/0ab5e65cf34b_fix_default_type_to_inference.py @@ -5,11 +5,10 @@ Create Date: 2022-04-26 19:37:27.263471 """ -import sqlalchemy as sa +from alembic import op from sqlalchemy import orm -from alembic import op -from src.db import models +from pipelines.db import models # revision identifiers, used by Alembic. revision = "0ab5e65cf34b" diff --git a/pipelines/alembic/versions/29f072fb5c9c_.py b/pipelines/alembic/versions/29f072fb5c9c_.py index 88b3be178..269820a10 100644 --- a/pipelines/alembic/versions/29f072fb5c9c_.py +++ b/pipelines/alembic/versions/29f072fb5c9c_.py @@ -6,7 +6,6 @@ """ import sqlalchemy as sa - from alembic import op # revision identifiers, used by Alembic. diff --git a/pipelines/alembic/versions/58fa5399caa9.py b/pipelines/alembic/versions/58fa5399caa9.py index 0f45f2909..cec33d4a5 100644 --- a/pipelines/alembic/versions/58fa5399caa9.py +++ b/pipelines/alembic/versions/58fa5399caa9.py @@ -6,7 +6,6 @@ """ import sqlalchemy as sa - from alembic import op # revision identifiers, used by Alembic. diff --git a/pipelines/alembic/versions/5fd9d1fdcf5b_init.py b/pipelines/alembic/versions/5fd9d1fdcf5b_init.py index 2b62f9e81..b1f8db5bf 100644 --- a/pipelines/alembic/versions/5fd9d1fdcf5b_init.py +++ b/pipelines/alembic/versions/5fd9d1fdcf5b_init.py @@ -6,9 +6,8 @@ """ import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - from alembic import op +from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. diff --git a/pipelines/alembic/versions/764961499e2b_add_original_pipeline_id_and_is_latest.py b/pipelines/alembic/versions/764961499e2b_add_original_pipeline_id_and_is_latest.py index 63acf819a..895bdd4e5 100644 --- a/pipelines/alembic/versions/764961499e2b_add_original_pipeline_id_and_is_latest.py +++ b/pipelines/alembic/versions/764961499e2b_add_original_pipeline_id_and_is_latest.py @@ -6,10 +6,10 @@ """ import sqlalchemy as sa +from alembic import op from sqlalchemy import orm -from alembic import op -from src.db import models +from pipelines.db import models # revision identifiers, used by Alembic. revision = "764961499e2b" diff --git a/pipelines/alembic/versions/8a589dda3869_add_type_description_and_summary_to_.py b/pipelines/alembic/versions/8a589dda3869_add_type_description_and_summary_to_.py index abefe8942..a82ef9847 100644 --- a/pipelines/alembic/versions/8a589dda3869_add_type_description_and_summary_to_.py +++ b/pipelines/alembic/versions/8a589dda3869_add_type_description_and_summary_to_.py @@ -6,7 +6,6 @@ """ import sqlalchemy as sa - from alembic import op # revision identifiers, used by Alembic. diff --git a/pipelines/alembic/versions/b0cbaebbddd8_change_pipeline_version_to_int.py b/pipelines/alembic/versions/b0cbaebbddd8_change_pipeline_version_to_int.py index fbc421ff4..74512697f 100644 --- a/pipelines/alembic/versions/b0cbaebbddd8_change_pipeline_version_to_int.py +++ b/pipelines/alembic/versions/b0cbaebbddd8_change_pipeline_version_to_int.py @@ -6,10 +6,10 @@ """ import sqlalchemy as sa +from alembic import op from sqlalchemy import orm -from alembic import op -from src.db import models +from pipelines.db import models # revision identifiers, used by Alembic. revision = "b0cbaebbddd8" diff --git a/pipelines/alembic/versions/c26caf5e8a19_add_webhook_column.py b/pipelines/alembic/versions/c26caf5e8a19_add_webhook_column.py index d107fab9d..fc3b4d583 100644 --- a/pipelines/alembic/versions/c26caf5e8a19_add_webhook_column.py +++ b/pipelines/alembic/versions/c26caf5e8a19_add_webhook_column.py @@ -7,7 +7,6 @@ """ import sqlalchemy as sa - from alembic import op # revision identifiers, used by Alembic. diff --git a/pipelines/alembic/versions/cd396f8a2df1_change_token_column_type.py b/pipelines/alembic/versions/cd396f8a2df1_change_token_column_type.py index 62760093d..1f8ffdc80 100644 --- a/pipelines/alembic/versions/cd396f8a2df1_change_token_column_type.py +++ b/pipelines/alembic/versions/cd396f8a2df1_change_token_column_type.py @@ -7,7 +7,6 @@ """ import sqlalchemy as sa - from alembic import op # revision identifiers, used by Alembic. diff --git a/pipelines/alembic/versions/df42f45f4ddf_add_parent_step_and_tenant_to_execution_.py b/pipelines/alembic/versions/df42f45f4ddf_add_parent_step_and_tenant_to_execution_.py index 28d4c9eef..0ca774fdb 100644 --- a/pipelines/alembic/versions/df42f45f4ddf_add_parent_step_and_tenant_to_execution_.py +++ b/pipelines/alembic/versions/df42f45f4ddf_add_parent_step_and_tenant_to_execution_.py @@ -5,26 +5,32 @@ Create Date: 2022-04-05 16:13:05.539298 """ -from alembic import op import sqlalchemy as sa +from alembic import op from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. -revision = 'df42f45f4ddf' -down_revision = '29f072fb5c9c' +revision = "df42f45f4ddf" +down_revision = "29f072fb5c9c" branch_labels = None depends_on = None def upgrade(): # ### commands auto generated by Alembic - please adjust! ### - op.add_column('execution_step', sa.Column('parent_step', postgresql.UUID(), nullable=True)) - op.add_column('execution_step', sa.Column('tenant', sa.String(length=50), nullable=True)) + op.add_column( + "execution_step", + sa.Column("parent_step", postgresql.UUID(), nullable=True), + ) + op.add_column( + "execution_step", + sa.Column("tenant", sa.String(length=50), nullable=True), + ) # ### end Alembic commands ### def downgrade(): # ### commands auto generated by Alembic - please adjust! ### - op.drop_column('execution_step', 'tenant') - op.drop_column('execution_step', 'parent_step') + op.drop_column("execution_step", "tenant") + op.drop_column("execution_step", "parent_step") # ### end Alembic commands ### diff --git a/processing/src/third_party_code/__init__.py b/pipelines/pipelines/__init__.py similarity index 100% rename from processing/src/third_party_code/__init__.py rename to pipelines/pipelines/__init__.py diff --git a/pipelines/src/app.py b/pipelines/pipelines/app.py similarity index 97% rename from pipelines/src/app.py rename to pipelines/pipelines/app.py index 2124c4954..5d78b239e 100644 --- a/pipelines/src/app.py +++ b/pipelines/pipelines/app.py @@ -8,13 +8,13 @@ from sqlalchemy_filters.exceptions import BadFilterFormat from tenant_dependency import TenantData, get_tenant_info -import src.config as config -import src.db.models as dbm -import src.db.service as service -import src.execution as execution -import src.schemas as schemas -from src.kafka_utils import Kafka -from src.pipeline_runner import run_pipeline +import pipelines.config as config +import pipelines.db.models as dbm +import pipelines.db.service as service +import pipelines.execution as execution +import pipelines.schemas as schemas +from pipelines.kafka_utils import Kafka +from pipelines.pipeline_runner import run_pipeline TOKEN = get_tenant_info(url=config.KEYCLOAK_URI, algorithm="RS256") diff --git a/pipelines/src/config.py b/pipelines/pipelines/config.py similarity index 100% rename from pipelines/src/config.py rename to pipelines/pipelines/config.py diff --git a/pipelines/pipelines/db/__init__.py b/pipelines/pipelines/db/__init__.py new file mode 100644 index 000000000..c0069eb45 --- /dev/null +++ b/pipelines/pipelines/db/__init__.py @@ -0,0 +1 @@ +import pipelines.db.logger # noqa: F401 diff --git a/pipelines/src/db/logger.py b/pipelines/pipelines/db/logger.py similarity index 93% rename from pipelines/src/db/logger.py rename to pipelines/pipelines/db/logger.py index c9de9073a..ec52b1085 100644 --- a/pipelines/src/db/logger.py +++ b/pipelines/pipelines/db/logger.py @@ -4,10 +4,10 @@ from sqlalchemy.engine import Connection from sqlalchemy.orm import Mapper -import src.db.models as models -import src.db.service as service -import src.pipeline_runner as runner -import src.schemas as schemas +import pipelines.db.models as models +import pipelines.db.service as service +import pipelines.pipeline_runner as runner +import pipelines.schemas as schemas def create_log(event_type: str, entity: models.Table) -> schemas.Log: diff --git a/pipelines/src/db/models.py b/pipelines/pipelines/db/models.py similarity index 100% rename from pipelines/src/db/models.py rename to pipelines/pipelines/db/models.py diff --git a/pipelines/src/db/service.py b/pipelines/pipelines/db/service.py similarity index 99% rename from pipelines/src/db/service.py rename to pipelines/pipelines/db/service.py index bbbf699ef..8a07d0512 100644 --- a/pipelines/src/db/service.py +++ b/pipelines/pipelines/db/service.py @@ -7,8 +7,8 @@ from sqlalchemy import create_engine from sqlalchemy.orm import Session, sessionmaker -import src.db.models as dbm -from src import config, execution, log, schemas +import pipelines.db.models as dbm +from pipelines import config, execution, log, schemas logger = log.get_logger(__file__) diff --git a/pipelines/src/execution.py b/pipelines/pipelines/execution.py similarity index 99% rename from pipelines/src/execution.py rename to pipelines/pipelines/execution.py index d659e1bfc..f7db094b0 100644 --- a/pipelines/src/execution.py +++ b/pipelines/pipelines/execution.py @@ -14,10 +14,11 @@ from pydantic import BaseModel, Field from sqlalchemy import orm -import src.db.models as dbm -import src.db.service as service -import src.result_processing as postprocessing -from src import config, http_utils, log, s3, schemas, service_token, webhooks +import pipelines.db.models as dbm +import pipelines.db.service as service +import pipelines.result_processing as postprocessing +from pipelines import (config, http_utils, log, s3, schemas, service_token, + webhooks) logger = log.get_logger(__file__) minio_client = s3.get_minio_client() diff --git a/pipelines/src/http_utils.py b/pipelines/pipelines/http_utils.py similarity index 98% rename from pipelines/src/http_utils.py rename to pipelines/pipelines/http_utils.py index b92ce7f68..8ee693a20 100644 --- a/pipelines/src/http_utils.py +++ b/pipelines/pipelines/http_utils.py @@ -3,7 +3,7 @@ import requests -from src import config, log, schemas, service_token +from pipelines import config, log, schemas, service_token logger = log.get_logger(__file__) diff --git a/pipelines/src/kafka_utils.py b/pipelines/pipelines/kafka_utils.py similarity index 98% rename from pipelines/src/kafka_utils.py rename to pipelines/pipelines/kafka_utils.py index d5f0f01ce..44cc7f2f3 100644 --- a/pipelines/src/kafka_utils.py +++ b/pipelines/pipelines/kafka_utils.py @@ -4,7 +4,7 @@ import aiokafka from kafka import admin, errors -from src import config, log +from pipelines import config, log logger = log.get_logger(__name__) diff --git a/pipelines/src/log.py b/pipelines/pipelines/log.py similarity index 97% rename from pipelines/src/log.py rename to pipelines/pipelines/log.py index b9bd62ecf..63d4d78f9 100644 --- a/pipelines/src/log.py +++ b/pipelines/pipelines/log.py @@ -1,7 +1,7 @@ import logging from typing import Any, Dict, Optional -from src.config import LOG_LEVEL +from pipelines.config import LOG_LEVEL _log_format = ( "%(asctime)s - [%(levelname)s] - %(name)s - " diff --git a/pipelines/src/pipeline_runner.py b/pipelines/pipelines/pipeline_runner.py similarity index 96% rename from pipelines/src/pipeline_runner.py rename to pipelines/pipelines/pipeline_runner.py index 51ead4eed..432ae70f0 100644 --- a/pipelines/src/pipeline_runner.py +++ b/pipelines/pipelines/pipeline_runner.py @@ -5,8 +5,8 @@ import aiokafka from aiokafka import AIOKafkaConsumer, AIOKafkaProducer -from src import execution, schemas -from src.log import get_logger +from pipelines import execution, schemas +from pipelines.log import get_logger logger = get_logger(__file__) diff --git a/pipelines/src/result_processing.py b/pipelines/pipelines/result_processing.py similarity index 99% rename from pipelines/src/result_processing.py rename to pipelines/pipelines/result_processing.py index cc468024b..e5cce42bb 100644 --- a/pipelines/src/result_processing.py +++ b/pipelines/pipelines/result_processing.py @@ -10,7 +10,7 @@ from minio import error as minioerr from pydantic import BaseModel, ValidationError -from src import config, http_utils, log +from pipelines import config, http_utils, log logger = log.get_logger(__file__) diff --git a/pipelines/src/s3.py b/pipelines/pipelines/s3.py similarity index 95% rename from pipelines/src/s3.py rename to pipelines/pipelines/s3.py index b86994b2c..bd39a4df3 100644 --- a/pipelines/src/s3.py +++ b/pipelines/pipelines/s3.py @@ -1,56 +1,56 @@ -import enum -from typing import Any, Dict, Optional - -from minio import Minio, credentials - -from src import config, log - -logger = log.get_logger(__file__) - - -class S3Providers(str, enum.Enum): - MINIO = "minio" - AWS_IAM = "aws_iam" - AWS_ENV = "aws_env" - AWS_CONF = "aws_config" - - -def get_minio_config( - s3_provider: S3Providers, - endpoint: Optional[str], - access_key: Optional[str], - secret_key: Optional[str], - **kwargs: Optional[str], -) -> Dict[str, Any]: - minio_config = {"endpoint": endpoint, "secure": False} - if s3_provider == S3Providers.MINIO: - minio_config["access_key"] = access_key - minio_config["secret_key"] = secret_key - elif s3_provider == S3Providers.AWS_IAM: - minio_config["credentials"] = credentials.IamAwsProvider() - elif s3_provider == S3Providers.AWS_ENV: - minio_config["credentials"] = credentials.EnvAWSProvider() - elif s3_provider == S3Providers.AWS_CONF: - minio_config["credentials"] = credentials.AWSConfigProvider( - profile=kwargs.get("aws_profile") - ) - return minio_config - - -def get_minio_client() -> Minio: - """Return Minio client if URI is provided via config.py.""" - s3_provider = S3Providers(config.S3_CREDENTIALS_PROVIDER) - logger.debug("S3_CREDENTIALS_PROVIDER is set to %s", s3_provider) - minio_config = get_minio_config( - s3_provider=s3_provider, - endpoint=config.S3_ENDPOINT, - access_key=config.S3_ACCESS_KEY, - secret_key=config.S3_SECRET_KEY, - aws_profile=config.AWS_PROFILE, - ) - return Minio(**minio_config) - - -def tenant_from_bucket(bucket: str) -> str: - prefix = f"{config.S3_PREFIX}-" if config.S3_PREFIX else "" - return bucket.replace(prefix, "", 1) +import enum +from typing import Any, Dict, Optional + +from minio import Minio, credentials + +from pipelines import config, log + +logger = log.get_logger(__file__) + + +class S3Providers(str, enum.Enum): + MINIO = "minio" + AWS_IAM = "aws_iam" + AWS_ENV = "aws_env" + AWS_CONF = "aws_config" + + +def get_minio_config( + s3_provider: S3Providers, + endpoint: Optional[str], + access_key: Optional[str], + secret_key: Optional[str], + **kwargs: Optional[str], +) -> Dict[str, Any]: + minio_config = {"endpoint": endpoint, "secure": False} + if s3_provider == S3Providers.MINIO: + minio_config["access_key"] = access_key + minio_config["secret_key"] = secret_key + elif s3_provider == S3Providers.AWS_IAM: + minio_config["credentials"] = credentials.IamAwsProvider() + elif s3_provider == S3Providers.AWS_ENV: + minio_config["credentials"] = credentials.EnvAWSProvider() + elif s3_provider == S3Providers.AWS_CONF: + minio_config["credentials"] = credentials.AWSConfigProvider( + profile=kwargs.get("aws_profile") + ) + return minio_config + + +def get_minio_client() -> Minio: + """Return Minio client if URI is provided via config.py.""" + s3_provider = S3Providers(config.S3_CREDENTIALS_PROVIDER) + logger.debug("S3_CREDENTIALS_PROVIDER is set to %s", s3_provider) + minio_config = get_minio_config( + s3_provider=s3_provider, + endpoint=config.S3_ENDPOINT, + access_key=config.S3_ACCESS_KEY, + secret_key=config.S3_SECRET_KEY, + aws_profile=config.AWS_PROFILE, + ) + return Minio(**minio_config) + + +def tenant_from_bucket(bucket: str) -> str: + prefix = f"{config.S3_PREFIX}-" if config.S3_PREFIX else "" + return bucket.replace(prefix, "", 1) diff --git a/pipelines/src/schemas.py b/pipelines/pipelines/schemas.py similarity index 98% rename from pipelines/src/schemas.py rename to pipelines/pipelines/schemas.py index 2b750026d..d4f682c49 100644 --- a/pipelines/src/schemas.py +++ b/pipelines/pipelines/schemas.py @@ -7,8 +7,8 @@ from pydantic import BaseModel, Field, PrivateAttr, root_validator, validator -import src.db.models as dbm -from src import log +import pipelines.db.models as dbm +from pipelines import log logger = log.get_logger(__file__) @@ -108,9 +108,9 @@ def output_bucket_validator(cls, values: Dict[str, Any]) -> Dict[str, Any]: return values @validator("input_path", "output_path") - def path_validator( # pylint: disable=E0213 + def path_validator( cls, v: Optional[str] - ) -> Optional[str]: + ) -> Optional[str]: # pylint: disable=E0213 """Path validator.""" if v is None: return v diff --git a/pipelines/src/service_token.py b/pipelines/pipelines/service_token.py similarity index 93% rename from pipelines/src/service_token.py rename to pipelines/pipelines/service_token.py index 7d8732ddf..8faaebe88 100644 --- a/pipelines/src/service_token.py +++ b/pipelines/pipelines/service_token.py @@ -1,8 +1,8 @@ import json from typing import Optional -from src import config, http_utils -from src.log import get_logger +from pipelines import config, http_utils +from pipelines.log import get_logger ACCESS_TOKEN = "access_token" diff --git a/pipelines/src/webhooks.py b/pipelines/pipelines/webhooks.py similarity index 95% rename from pipelines/src/webhooks.py rename to pipelines/pipelines/webhooks.py index ebf9b7644..2ac1aa094 100644 --- a/pipelines/src/webhooks.py +++ b/pipelines/pipelines/webhooks.py @@ -1,8 +1,8 @@ import urllib.parse from typing import Any, Dict, Optional, Tuple, Union -from src import http_utils, log, schemas, service_token -from src.db import service +from pipelines import http_utils, log, schemas, service_token +from pipelines.db import service logger = log.get_logger(__file__) diff --git a/pipelines/requirements.txt b/pipelines/requirements.txt index 53b9b4646..81b85f1b8 100644 --- a/pipelines/requirements.txt +++ b/pipelines/requirements.txt @@ -3,7 +3,7 @@ fastapi~=0.70.0 pydantic==1.8.2 psycopg2-binary==2.9.1 sqlalchemy==1.3.23 -SQLAlchemy-Utils==0.37.8 +SQLAlchemy-Utils~=0.38.3 uvicorn[standard]==0.15.0 minio==7.1.1 alembic==1.7.5 diff --git a/pipelines/setup.py b/pipelines/setup.py index bf3ede6e9..286f9641c 100644 --- a/pipelines/setup.py +++ b/pipelines/setup.py @@ -3,7 +3,7 @@ setuptools.setup( name="pipelines", version="0.1.0", - packages=["src"], - package_dir={"pipelines": "src"}, - entry_points={"console_scripts": ["executor = src.cli:execute"]}, + packages=["pipelines"], + package_dir={"pipelines": "pipelines"}, + entry_points={"console_scripts": ["executor = pipelines.cli:execute"]}, ) diff --git a/pipelines/src/db/__init__.py b/pipelines/src/db/__init__.py deleted file mode 100644 index 268a5c44e..000000000 --- a/pipelines/src/db/__init__.py +++ /dev/null @@ -1 +0,0 @@ -import src.db.logger # noqa: F401 diff --git a/pipelines/tests/conftest.py b/pipelines/tests/conftest.py index 71a5b7af8..f80333aa1 100644 --- a/pipelines/tests/conftest.py +++ b/pipelines/tests/conftest.py @@ -2,6 +2,9 @@ from unittest.mock import patch import pytest +import tests.testing_data as td +from alembic import command +from alembic.config import Config from fastapi.testclient import TestClient from pydantic import BaseModel from sqlalchemy import create_engine @@ -9,14 +12,11 @@ from sqlalchemy.orm import sessionmaker from sqlalchemy_utils import create_database, database_exists, drop_database -import src.app as app -import src.db.models as dbm -import src.db.service as service -import src.execution as execution -import tests.testing_data as td -from alembic import command -from alembic.config import Config -from src.config import DB_URI +import pipelines.app as app +import pipelines.db.models as dbm +import pipelines.db.service as service +import pipelines.execution as execution +from pipelines.config import DB_URI test_db_url = service.get_test_db_url(DB_URI) alembic_cfg = Config("alembic.ini") @@ -72,7 +72,7 @@ def setup_token(): def testing_app(testing_engine, testing_session, setup_token): session = sessionmaker(bind=testing_engine) app.app.dependency_overrides[app.TOKEN] = lambda: setup_token - with patch("src.db.service.LocalSession", session): + with patch("pipelines.db.service.LocalSession", session): app.app.dependency_overrides[ service.get_session ] = lambda: testing_session @@ -102,19 +102,19 @@ def testing_task(testing_pipeline): @pytest.fixture def session_mock(): - with patch("src.db.service.LocalSession") as mock: + with patch("pipelines.db.service.LocalSession") as mock: yield mock @pytest.fixture def request_mock(): - with patch("src.http_utils.requests.request") as mock: + with patch("pipelines.http_utils.requests.request") as mock: yield mock @pytest.fixture def run_in_session_mock(): - with patch("src.db.service.run_in_session") as mock: + with patch("pipelines.db.service.run_in_session") as mock: yield mock @@ -124,7 +124,7 @@ async def check_preprocessing_status_mock(x, y): return True with patch( - "src.execution.PipelineTask.check_preprocessing_status", + "pipelines.execution.PipelineTask.check_preprocessing_status", check_preprocessing_status_mock, ) as mock: yield mock diff --git a/pipelines/tests/db/test_logger.py b/pipelines/tests/db/test_logger.py index 1ae092228..81d7189dd 100644 --- a/pipelines/tests/db/test_logger.py +++ b/pipelines/tests/db/test_logger.py @@ -1,12 +1,12 @@ -"""Testing src/db/logger.py.""" +"""Testing pipelines/db/logger.py.""" import uuid import sqlalchemy.event -import src.db.logger as logger -import src.db.models as models -import src.schemas as schemas -import src.pipeline_runner as runner +import pipelines.db.logger as logger +import pipelines.db.models as models +import pipelines.pipeline_runner as runner +import pipelines.schemas as schemas def test_create_log(testing_session): diff --git a/pipelines/tests/db/test_models.py b/pipelines/tests/db/test_models.py index 8e2a64a93..d7c82a532 100644 --- a/pipelines/tests/db/test_models.py +++ b/pipelines/tests/db/test_models.py @@ -1,4 +1,4 @@ -"""Testing src/db/models.py.""" +"""Testing pipelines/db/models.py.""" import tests.testing_data as td diff --git a/pipelines/tests/db/test_service.py b/pipelines/tests/db/test_service.py index 32c45c2b5..d7bb08ab3 100644 --- a/pipelines/tests/db/test_service.py +++ b/pipelines/tests/db/test_service.py @@ -1,4 +1,4 @@ -"""Testing src/db/service.py.""" +"""Testing pipelines/db/service.py.""" import datetime import uuid @@ -6,14 +6,14 @@ from unittest.mock import patch import pytest +import tests.testing_data as td from aiokafka import AIOKafkaProducer from freezegun import freeze_time -import src.db.models as dbm -import src.db.service as service -import src.execution as execution -import src.schemas as schemas -import tests.testing_data as td +import pipelines.db.models as dbm +import pipelines.db.service as service +import pipelines.execution as execution +import pipelines.schemas as schemas pytest_plugins = ("pytest_asyncio",) @@ -128,7 +128,9 @@ def test_get_step_by_step_and_task_id(testing_session): """Testing get_step_by_step_and_task_id.""" task = dbm.PipelineExecutionTask(pipeline=dbm.Pipeline(type="inference")) step_uuid = str(uuid.uuid4()) - step = dbm.ExecutionStep(task=task, step_id=step_uuid, init_args={"foo": 1}) + step = dbm.ExecutionStep( + task=task, step_id=step_uuid, init_args={"foo": 1} + ) testing_session.add(step) assert service.get_step_by_step_and_task_id( testing_session, 1, step_uuid @@ -139,7 +141,9 @@ def test_get_step_by_step_and_task_id_not_found(testing_session): """Testing get_step_by_step_and_task_id when instance not found.""" some_random_uuid = str(uuid.uuid4()) assert ( - service.get_step_by_step_and_task_id(testing_session, 1, some_random_uuid) + service.get_step_by_step_and_task_id( + testing_session, 1, some_random_uuid + ) is None ) @@ -214,7 +218,9 @@ def test_update_task_in_lock(testing_session): """Testing update_task_in_lock.""" runner1_uuid, runner2_uuid = [str(uuid.uuid4()) for _ in range(2)] task = dbm.PipelineExecutionTask( - pipeline=dbm.Pipeline(type="inference"), status=PEND, runner_id=runner1_uuid + pipeline=dbm.Pipeline(type="inference"), + status=PEND, + runner_id=runner1_uuid, ) testing_session.add(task) assert task.runner_id == runner1_uuid @@ -259,7 +265,11 @@ def test_get_expired_heartbeats(testing_session): """Testing get_expired_heartbeats.""" eff_date = datetime.datetime.utcnow() last_heartbeat = eff_date - datetime.timedelta(minutes=1) - testing_session.add(dbm.ExecutorHeartbeat(id=str(uuid.uuid4()), last_heartbeat=last_heartbeat)) + testing_session.add( + dbm.ExecutorHeartbeat( + id=str(uuid.uuid4()), last_heartbeat=last_heartbeat + ) + ) result = service.get_expired_heartbeats(testing_session, eff_date) assert result[0].last_heartbeat == last_heartbeat @@ -278,7 +288,9 @@ def test_update_heartbeat_timestamp(testing_session): def test_task_runner_id_status_in_lock(testing_session): """Testing change_task_runner_id_and_status.""" task = dbm.PipelineExecutionTask( - pipeline=dbm.Pipeline(type="inference"), status=RUN, runner_id=str(uuid.uuid4()) + pipeline=dbm.Pipeline(type="inference"), + status=RUN, + runner_id=str(uuid.uuid4()), ) testing_session.add(task) service.change_task_runner_id_status_in_lock(testing_session, 1) diff --git a/pipelines/tests/test_app.py b/pipelines/tests/test_app.py index 6b0b508c6..8ba390cf8 100644 --- a/pipelines/tests/test_app.py +++ b/pipelines/tests/test_app.py @@ -1,17 +1,17 @@ -"""Testing src/app.py.""" +"""Testing pipelines/app.py.""" from copy import deepcopy from typing import Dict import pytest - -import src.app as app -import src.db.models as dbm -import src.db.service as service -import src.execution as execution -import src.schemas as schemas import tests.testing_data as td +import pipelines.app as app +import pipelines.db.models as dbm +import pipelines.db.service as service +import pipelines.execution as execution +import pipelines.schemas as schemas + def test_add_pipeline(testing_app, adjust_mock): """Testing add_pipeline.""" diff --git a/pipelines/tests/test_execution.py b/pipelines/tests/test_execution.py index 9d972bb79..254fbb695 100644 --- a/pipelines/tests/test_execution.py +++ b/pipelines/tests/test_execution.py @@ -1,18 +1,18 @@ -"""Testing src/execution.py.""" +"""Testing pipelines/execution.py.""" import logging from itertools import cycle from typing import Optional from unittest.mock import PropertyMock, patch import pytest +import tests.testing_data as td from aiokafka import AIOKafkaProducer from fastapi import HTTPException from pydantic import BaseModel -import src.db.models as dbm -import src.execution as execution -import src.schemas as schemas -import tests.testing_data as td +import pipelines.db.models as dbm +import pipelines.execution as execution +import pipelines.schemas as schemas LOGGER = logging.getLogger(__name__) @@ -34,9 +34,10 @@ def uuid_mock(): @patch( - "src.execution.ExecutionStep.get_pipeline_step", new_callable=PropertyMock + "pipelines.execution.ExecutionStep.get_pipeline_step", + new_callable=PropertyMock, ) -@patch("src.execution.ExecutionStep.step_execution") +@patch("pipelines.execution.ExecutionStep.step_execution") @pytest.mark.asyncio async def test_step_execution_with_logging( step_exec_mock, pipeline_step, run_in_session_mock, caplog @@ -63,9 +64,10 @@ async def test_step_execution_with_logging( "It passes when run separately, but fails when all tests are run." ) @patch( - "src.execution.ExecutionStep.get_pipeline_step", new_callable=PropertyMock + "pipelines.execution.ExecutionStep.get_pipeline_step", + new_callable=PropertyMock, ) -@patch("src.execution.ExecutionStep.send") +@patch("pipelines.execution.ExecutionStep.send") @pytest.mark.asyncio async def test_step_execution( mock_send, model_url, caplog, run_in_session_mock @@ -239,8 +241,8 @@ def test_adjust_pipeline(): "Test should be fixed - it 'blinks'. " "It passes when run separately, but fails when all tests are run." ) -@patch("src.execution.ExecutionStep.step_execution_with_logging") -@patch("src.execution.PipelineTask.send_status") +@patch("pipelines.execution.ExecutionStep.step_execution_with_logging") +@patch("pipelines.execution.PipelineTask.send_status") @pytest.mark.asyncio async def test_start_task( webhook_mock, @@ -284,11 +286,11 @@ async def test_start_task( # return True with patch( - "src.execution.PipelineTask.get_pipeline_type", + "pipelines.execution.PipelineTask.get_pipeline_type", lambda _: schemas.PipelineTypes.INFERENCE, ): # with patch( - # "src.execution.PipelineTask.check_preprocessing_status", + # "pipelines.execution.PipelineTask.check_preprocessing_status", # check_preprocessing_status_mock, # ): await task.start(AIOKafkaProducer) @@ -301,7 +303,7 @@ async def test_start_task( "Test should be fixed - it 'blinks'. " "It passes when run separately, but fails when all tests are run." ) -@patch("src.execution.ExecutionStep.step_execution_with_logging") +@patch("pipelines.execution.ExecutionStep.step_execution_with_logging") @pytest.mark.asyncio async def test_process_next_steps(exec_step, caplog): exec_step.return_value = None @@ -335,9 +337,9 @@ async def test_process_next_steps(exec_step, caplog): "steps": [received_step, child_step], } ) - with patch("src.execution.PipelineTask.get_by_id", lambda id_: task): + with patch("pipelines.execution.PipelineTask.get_by_id", lambda id_: task): with patch( - "src.execution.PipelineTask.get_pipeline_type", + "pipelines.execution.PipelineTask.get_pipeline_type", lambda _: schemas.PipelineTypes.INFERENCE, ): await received_step.process_next_steps(AIOKafkaProducer) @@ -349,7 +351,7 @@ async def test_process_next_steps(exec_step, caplog): "Test should be fixed - it 'blinks'. " "It passes when run separately, but fails when all tests are run." ) -@patch("src.execution.ExecutionStep.step_execution_with_logging") +@patch("pipelines.execution.ExecutionStep.step_execution_with_logging") @pytest.mark.asyncio async def test_process_next_staps_without_child_steps(exec_step, caplog): received_step = execution.ExecutionStep.parse_obj( @@ -383,7 +385,7 @@ async def test_process_next_staps_without_child_steps(exec_step, caplog): } ) - with patch("src.execution.PipelineTask.get_by_id", lambda id_: task): + with patch("pipelines.execution.PipelineTask.get_by_id", lambda id_: task): await received_step.process_next_steps(AIOKafkaProducer) assert caplog.messages[0].startswith("Step with id = 58 from task = 20") diff --git a/pipelines/tests/test_http_utils.py b/pipelines/tests/test_http_utils.py index 6eb261bcf..162ff7796 100644 --- a/pipelines/tests/test_http_utils.py +++ b/pipelines/tests/test_http_utils.py @@ -1,9 +1,9 @@ from unittest.mock import patch +import pytest import requests -import pytest -from src import http_utils, schemas +from pipelines import http_utils, schemas def test_make_request(request_mock): @@ -26,7 +26,7 @@ def test_make_request(request_mock): def test_make_request_with_retry(s_effect, expected, call_count, request_mock): """Testing make_request_with_retry.""" with patch( - "src.http_utils.make_request", side_effect=s_effect + "pipelines.http_utils.make_request", side_effect=s_effect ) as req_mock: assert http_utils.make_request_with_retry("", {}, start=0) == expected assert req_mock.call_count == call_count diff --git a/pipelines/tests/test_pipeline_runner.py b/pipelines/tests/test_pipeline_runner.py index 21c72b0b6..9610f8f1e 100644 --- a/pipelines/tests/test_pipeline_runner.py +++ b/pipelines/tests/test_pipeline_runner.py @@ -1,4 +1,4 @@ -# """Testing src/pipeline_runner.py.""" +# """Testing pipelines/pipeline_runner.py.""" import logging from unittest.mock import patch @@ -6,8 +6,8 @@ from aiokafka import AIOKafkaProducer from pydantic import BaseModel -import src.execution as execution -import src.pipeline_runner as runner +import pipelines.execution as execution +import pipelines.pipeline_runner as runner LOGGER = logging.getLogger(__name__) @@ -84,9 +84,9 @@ def test_response_message_incorrect(caplog): ) -@patch("src.execution.PipelineTask.get_by_id") -@patch("src.execution.ExecutionStep.get_by_id") -@patch("src.execution.ExecutionStep.process_next_steps") +@patch("pipelines.execution.PipelineTask.get_by_id") +@patch("pipelines.execution.ExecutionStep.get_by_id") +@patch("pipelines.execution.ExecutionStep.process_next_steps") @pytest.mark.asyncio async def test_process_message_task_not_finished( process_next_steps, get_step, get_task, testing_app @@ -145,9 +145,9 @@ async def test_process_message_task_not_finished( assert process_next_steps.called -@patch("src.execution.PipelineTask.get_by_id") -@patch("src.execution.ExecutionStep.get_by_id") -@patch("src.execution.PipelineTask.finish") +@patch("pipelines.execution.PipelineTask.get_by_id") +@patch("pipelines.execution.ExecutionStep.get_by_id") +@patch("pipelines.execution.PipelineTask.finish") @pytest.mark.asyncio async def test_process_message_task_finished( finish_task, get_step, get_task, testing_app @@ -210,9 +210,9 @@ async def test_process_message_task_finished( "Test should be fixed - it 'blinks'. " "It passes when run separately, but fails when all tests are run." ) -@patch("src.execution.PipelineTask.get_by_id") -@patch("src.execution.ExecutionStep.get_by_id") -@patch("src.execution.PipelineTask.finish") +@patch("pipelines.execution.PipelineTask.get_by_id") +@patch("pipelines.execution.ExecutionStep.get_by_id") +@patch("pipelines.execution.PipelineTask.finish") @pytest.mark.asyncio async def test_process_message_task_failed( finish_task, get_step, get_task, testing_app, caplog @@ -267,7 +267,7 @@ async def test_process_message_task_failed( "Test should be fixed - it 'blinks'. " "It passes when run separately, but fails when all tests are run." ) -@patch("src.pipeline_runner.process_message") +@patch("pipelines.pipeline_runner.process_message") @pytest.mark.asyncio async def test_run_pipeline(process_message, caplog): message_1 = KafkaMessage.parse_obj( diff --git a/pipelines/tests/test_result_processing.py b/pipelines/tests/test_result_processing.py index 8f1c5c10b..f484dc362 100644 --- a/pipelines/tests/test_result_processing.py +++ b/pipelines/tests/test_result_processing.py @@ -1,10 +1,10 @@ -"""Testing src/result_processing.py.""" +"""Testing pipelines/result_processing.py.""" from unittest.mock import MagicMock, patch import pytest from minio import S3Error -import src.result_processing as processing +import pipelines.result_processing as processing def test_merge_outputs(): @@ -185,7 +185,9 @@ def test_merge_geometry_objects_no_objects_provided(): ) def test_get_annotation_uri(job_id, file_id, expected): """Testing get_annotation_uri.""" - with patch("src.result_processing.config.ANNOTATION_URI", "foobar/ann"): + with patch( + "pipelines.result_processing.config.ANNOTATION_URI", "foobar/ann" + ): assert processing.get_annotation_uri(job_id, file_id) == expected @@ -232,7 +234,9 @@ def test_get_pipeline_leaves_data(): def test_get_pipeline_leaves_data_minio_error(): """Testing get_pipeline_leaves_data when S3Error occurred.""" err = S3Error("", "", "", "", "", "") - with patch("src.result_processing.list_object_names", side_effect=err): + with patch( + "pipelines.result_processing.list_object_names", side_effect=err + ): res = processing.get_pipeline_leaves_data(MagicMock(), "", "") assert res is None @@ -249,7 +253,7 @@ def test_merge_pipeline_leaves_data(): b'{"id": 3, "bbox": [3, 3, 3, 3], "category": "some"}]}]}', ] with patch( - "src.result_processing.get_pipeline_leaves_data", + "pipelines.result_processing.get_pipeline_leaves_data", return_value=leaves_data, ): res = processing.merge_pipeline_leaves_data(MagicMock(), "", "") @@ -295,7 +299,8 @@ def test_merge_pipeline_leaves_data(): def test_merge_pipeline_leaves_data_no_files_data(): """Testing merge_pipeline_leaves_data when there's no files data.""" with patch( - "src.result_processing.get_pipeline_leaves_data", return_value=None + "pipelines.result_processing.get_pipeline_leaves_data", + return_value=None, ): assert ( processing.merge_pipeline_leaves_data(MagicMock(), "", "") is None @@ -305,9 +310,10 @@ def test_merge_pipeline_leaves_data_no_files_data(): def test_merge_pipeline_leaves_data_cannot_parse_data(): """Testing merge_pipeline_leaves_data when raw data cannot be parsed.""" with patch( - "src.result_processing.ModelOutput.parse_models", return_value=None + "pipelines.result_processing.ModelOutput.parse_models", + return_value=None, ): - with patch("src.result_processing.get_pipeline_leaves_data"): + with patch("pipelines.result_processing.get_pipeline_leaves_data"): assert ( processing.merge_pipeline_leaves_data(MagicMock(), "", "") is None @@ -316,8 +322,8 @@ def test_merge_pipeline_leaves_data_cannot_parse_data(): def test_merge_pipeline_leaves_data_cannot_merge_data(): """Testing merge_pipeline_leaves_data when data cannot be merged.""" - with patch("src.result_processing.get_pipeline_leaves_data"): - with patch("src.result_processing.ModelOutput.parse_models"): + with patch("pipelines.result_processing.get_pipeline_leaves_data"): + with patch("pipelines.result_processing.ModelOutput.parse_models"): assert ( processing.merge_pipeline_leaves_data(MagicMock(), "", "") is None @@ -327,7 +333,8 @@ def test_merge_pipeline_leaves_data_cannot_merge_data(): def test_delete_objects(): """Testing delete_objects.""" with patch( - "src.result_processing.list_object_names", return_value=["f", "b"] + "pipelines.result_processing.list_object_names", + return_value=["f", "b"], ): client_mock = MagicMock() assert processing.delete_objects(client_mock, "bucket", "") @@ -340,7 +347,9 @@ def test_delete_objects(): def test_delete_objects_minio_error(): """Testing delete_objects when S3Error occurred.""" err = S3Error("", "", "", "", "", "") - with patch("src.result_processing.list_object_names", side_effect=err): + with patch( + "pipelines.result_processing.list_object_names", side_effect=err + ): assert not processing.delete_objects(MagicMock(), "bucket", "") @@ -349,11 +358,11 @@ def test_postprocess_result(): m = MagicMock() m.content = b'{"foo": 42}' with patch( - "src.result_processing.http_utils.make_request_with_retry", + "pipelines.result_processing.http_utils.make_request_with_retry", return_value=m, ) as req_mock: with patch( - "src.result_processing.config.POSTPROCESSING_URI", "foo.com" + "pipelines.result_processing.config.POSTPROCESSING_URI", "foo.com" ): res = processing.postprocess_result({"foo": 1}) assert res == {"foo": 42} @@ -364,7 +373,7 @@ def test_postprocess_result(): def test_postprocess_result_no_uri(): """Testing postprocess_result when there's no uri.""" - with patch("src.result_processing.config.POSTPROCESSING_URI", ""): + with patch("pipelines.result_processing.config.POSTPROCESSING_URI", ""): assert processing.postprocess_result({"a": 1}) is None @@ -373,7 +382,7 @@ def test_postprocess_result_invalid_postprocessor_json_response(): m = MagicMock m.content = b'{"asd":}' with patch( - "src.result_processing.http_utils.make_request_with_retry", + "pipelines.result_processing.http_utils.make_request_with_retry", return_value=m, ): assert processing.postprocess_result({"a": 1}) is None @@ -382,9 +391,9 @@ def test_postprocess_result_invalid_postprocessor_json_response(): @pytest.mark.skip("Test should be fixed - del_mock.assert_called_once() fails") def test_manage_result_for_annotator(): """Testing manage_result_for_annotator.""" - with patch("src.result_processing.merge_pipeline_leaves_data"): + with patch("pipelines.result_processing.merge_pipeline_leaves_data"): with patch( - "src.result_processing.postprocess_result", + "pipelines.result_processing.postprocess_result", return_value={ "file": "", "bucket": "", @@ -392,12 +401,14 @@ def test_manage_result_for_annotator(): }, ): with patch( - "src.result_processing.http_utils.make_request_with_retry" + "pipelines.result_processing.http_utils.make_request_with_retry" ) as req_mock: - with patch("src.result_processing.delete_objects") as del_mock: - with patch("src.config.DEBUG_MERGE", False): + with patch( + "pipelines.result_processing.delete_objects" + ) as del_mock: + with patch("pipelines.config.DEBUG_MERGE", False): with patch( - "src.result_processing.config.ANNOTATION_URI", + "pipelines.result_processing.config.ANNOTATION_URI", "f.com/annotation", ): assert processing.manage_result_for_annotator( @@ -417,7 +428,7 @@ def test_manage_result_for_annotator(): def test_manage_result_for_annotator_no_annotator_uri(): """Testing manage_result_for_annotator when there's no Annotator URI.""" - with patch("src.result_processing.config.ANNOTATION_URI", ""): + with patch("pipelines.result_processing.config.ANNOTATION_URI", ""): assert not processing.manage_result_for_annotator( "", "", "", 0, "", "", "", 8, MagicMock(), "" ) @@ -426,7 +437,8 @@ def test_manage_result_for_annotator_no_annotator_uri(): def test_manage_result_for_annotator_cannot_merge_data(): """Testing manage_result_for_annotator when data cannot be merger.""" with patch( - "src.result_processing.merge_pipeline_leaves_data", return_value=None + "pipelines.result_processing.merge_pipeline_leaves_data", + return_value=None, ): assert not processing.manage_result_for_annotator( "", "", "", 0, "", "", "", 8, MagicMock(), "" @@ -435,10 +447,10 @@ def test_manage_result_for_annotator_cannot_merge_data(): def test_manage_result_for_annotator_request_not_succeeded(): """Testing manage_result_for_annotator when cannot connect to Annotator.""" - with patch("src.result_processing.merge_pipeline_leaves_data"): - with patch("src.result_processing.postprocess_result"): + with patch("pipelines.result_processing.merge_pipeline_leaves_data"): + with patch("pipelines.result_processing.postprocess_result"): with patch( - "src.result_processing.http_utils.make_request_with_retry", + "pipelines.result_processing.http_utils.make_request_with_retry", return_value=None, ): assert not processing.manage_result_for_annotator( @@ -448,14 +460,16 @@ def test_manage_result_for_annotator_request_not_succeeded(): def test_manage_result_for_annotator_request_debug_merge(): """Debug merge is True and data are not deleted.""" - with patch("src.result_processing.merge_pipeline_leaves_data"): - with patch("src.result_processing.postprocess_result"): + with patch("pipelines.result_processing.merge_pipeline_leaves_data"): + with patch("pipelines.result_processing.postprocess_result"): with patch( - "src.result_processing.http_utils.make_request_with_retry" + "pipelines.result_processing.http_utils.make_request_with_retry" ): - with patch("src.result_processing.config.DEBUG_MERGE", True): + with patch( + "pipelines.result_processing.config.DEBUG_MERGE", True + ): with patch( - "src.result_processing.delete_objects" + "pipelines.result_processing.delete_objects" ) as del_mock: assert processing.manage_result_for_annotator( "", "", "", 0, "", "", "", 8, MagicMock(), "" diff --git a/pipelines/tests/test_s3.py b/pipelines/tests/test_s3.py index 939029dfc..ac481fbeb 100644 --- a/pipelines/tests/test_s3.py +++ b/pipelines/tests/test_s3.py @@ -3,7 +3,7 @@ import minio import pytest -from src import s3 +from pipelines import s3 def test_get_minio_client(): @@ -21,5 +21,5 @@ def test_get_minio_client(): ), ) def test_tenant_from_bucket(prefix: str, bucket: str, expected: str) -> None: - with patch("src.config.S3_PREFIX", prefix): + with patch("pipelines.config.S3_PREFIX", prefix): assert s3.tenant_from_bucket(bucket) == expected diff --git a/pipelines/tests/test_schemas.py b/pipelines/tests/test_schemas.py index aa9931f9b..9888fcdb0 100644 --- a/pipelines/tests/test_schemas.py +++ b/pipelines/tests/test_schemas.py @@ -1,11 +1,11 @@ -"""Testing src/schemas.py.""" +"""Testing pipelines/schemas.py.""" import pytest - -import src.db.models as dbm -import src.schemas as schemas import tests.testing_data as td +import pipelines.db.models as dbm +import pipelines.schemas as schemas + def test_init_input_args(): """Testing __init__ of InputArguments.""" diff --git a/pipelines/tests/test_webhooks.py b/pipelines/tests/test_webhooks.py index 12c98f439..12b083d34 100644 --- a/pipelines/tests/test_webhooks.py +++ b/pipelines/tests/test_webhooks.py @@ -1,9 +1,9 @@ -"""Testing src/webhooks.py.""" +"""Testing pipelines/webhooks.py.""" from unittest.mock import patch -import src.schemas as schemas -import src.webhooks as webhooks +import pipelines.schemas as schemas +import pipelines.webhooks as webhooks def test_create_inference_url_and_body(): @@ -12,7 +12,8 @@ def test_create_inference_url_and_body(): task_status = schemas.Status.RUN status = schemas.JobStatus.RUN with patch( - "src.webhooks.service.get_job_status_if_changed", return_value=status + "pipelines.webhooks.service.get_job_status_if_changed", + return_value=status, ): url, body = webhooks.create_inference_url_and_body( webhook=webhook, job_id=job_id, task_status=task_status diff --git a/pipelines/tests/testing_data.py b/pipelines/tests/testing_data.py index 021b5f6a8..4dc4b273b 100644 --- a/pipelines/tests/testing_data.py +++ b/pipelines/tests/testing_data.py @@ -1,8 +1,8 @@ import json -import src.db.models as dbm -import src.execution as execution -import src.schemas as schemas +import pipelines.db.models as dbm +import pipelines.execution as execution +import pipelines.schemas as schemas steps_dict = { "model": "bar", diff --git a/processing/Dockerfile b/processing/Dockerfile index 64f948be9..763210433 100644 --- a/processing/Dockerfile +++ b/processing/Dockerfile @@ -13,7 +13,7 @@ ENV PYTHONUNBUFFERED 1 FROM base as build COPY alembic alembic -COPY src src +COPY processing src CMD alembic upgrade head && uvicorn src.main:app --host 0.0.0.0 --port 8080 FROM build as development diff --git a/processing/alembic/env.py b/processing/alembic/env.py index 9f191b943..f943afabc 100644 --- a/processing/alembic/env.py +++ b/processing/alembic/env.py @@ -1,11 +1,11 @@ import os from logging.config import fileConfig +from alembic import context from sqlalchemy import engine_from_config, pool -from alembic import context -from src.config import settings -from src.db.service import get_test_db_url +from processing.config import settings +from processing.db.service import get_test_db_url # this is the Alembic Config object, which provides # access to the values within the .ini file in use. @@ -26,7 +26,7 @@ # for 'autogenerate' support # from myapp import mymodel # target_metadata = mymodel.Base.metadata -from src.db.models import Base +from processing.db.models import Base target_metadata = Base.metadata diff --git a/processing/alembic/versions/52af1473946f_init.py b/processing/alembic/versions/52af1473946f_init.py index 36ccdb9cd..98083fada 100644 --- a/processing/alembic/versions/52af1473946f_init.py +++ b/processing/alembic/versions/52af1473946f_init.py @@ -5,9 +5,8 @@ Create Date: 2022-05-18 14:46:07.845635 """ -from alembic import op import sqlalchemy as sa - +from alembic import op # revision identifiers, used by Alembic. revision = "52af1473946f" diff --git a/processing/alembic/versions/8e973b70b26f_noneasnull.py b/processing/alembic/versions/8e973b70b26f_noneasnull.py index 29c46cba0..bdd24a8f2 100644 --- a/processing/alembic/versions/8e973b70b26f_noneasnull.py +++ b/processing/alembic/versions/8e973b70b26f_noneasnull.py @@ -5,8 +5,8 @@ Create Date: 2022-05-18 21:42:03.973476 """ -from alembic import op import sqlalchemy as sa +from alembic import op # revision identifiers, used by Alembic. revision = "8e973b70b26f" diff --git a/processing/alembic/versions/f637b13c744d_renamed_column.py b/processing/alembic/versions/f637b13c744d_renamed_column.py index 268f4ccfa..546d6510b 100644 --- a/processing/alembic/versions/f637b13c744d_renamed_column.py +++ b/processing/alembic/versions/f637b13c744d_renamed_column.py @@ -5,9 +5,8 @@ Create Date: 2022-05-19 12:43:52.309487 """ -from alembic import op import sqlalchemy as sa - +from alembic import op # revision identifiers, used by Alembic. revision = "f637b13c744d" diff --git a/processing/src/utils/__init__.py b/processing/processing/__init__.py similarity index 100% rename from processing/src/utils/__init__.py rename to processing/processing/__init__.py diff --git a/processing/src/config.py b/processing/processing/config.py similarity index 100% rename from processing/src/config.py rename to processing/processing/config.py diff --git a/processing/processing/db/__init__.py b/processing/processing/db/__init__.py new file mode 100644 index 000000000..175d2b491 --- /dev/null +++ b/processing/processing/db/__init__.py @@ -0,0 +1,2 @@ +import processing.db.models +import processing.db.service # noqa: F401 diff --git a/processing/src/db/models.py b/processing/processing/db/models.py similarity index 95% rename from processing/src/db/models.py rename to processing/processing/db/models.py index 1a11f0a2a..f5b41dfed 100644 --- a/processing/src/db/models.py +++ b/processing/processing/db/models.py @@ -2,7 +2,7 @@ from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import sessionmaker -from src.config import settings +from processing.config import settings Base = declarative_base() engine = sa.create_engine( diff --git a/processing/src/db/service.py b/processing/processing/db/service.py similarity index 97% rename from processing/src/db/service.py rename to processing/processing/db/service.py index e10d6c74b..17b0cfbf4 100644 --- a/processing/src/db/service.py +++ b/processing/processing/db/service.py @@ -2,8 +2,8 @@ from sqlalchemy.orm import Query, Session -from src import schema -from src.db import models +from processing import schema +from processing.db import models def session_scope() -> Session: diff --git a/processing/src/health_check_easy_ocr.py b/processing/processing/health_check_easy_ocr.py similarity index 93% rename from processing/src/health_check_easy_ocr.py rename to processing/processing/health_check_easy_ocr.py index 66e6a4902..1c8a8ad28 100644 --- a/processing/src/health_check_easy_ocr.py +++ b/processing/processing/health_check_easy_ocr.py @@ -4,12 +4,10 @@ from fastapi import HTTPException from minio.error import MinioException -from src.utils.aiohttp_utils import send_request -from src.utils.logger import get_logger -from src.utils.minio_utils import ( - MinioCommunicator, - convert_bucket_name_if_s3prefix, -) +from processing.utils.aiohttp_utils import send_request +from processing.utils.logger import get_logger +from processing.utils.minio_utils import (MinioCommunicator, + convert_bucket_name_if_s3prefix) logger = get_logger(__name__) minio_client = MinioCommunicator().client diff --git a/processing/src/main.py b/processing/processing/main.py similarity index 88% rename from processing/src/main.py rename to processing/processing/main.py index 36c611760..98946b7d4 100644 --- a/processing/src/main.py +++ b/processing/processing/main.py @@ -1,28 +1,19 @@ from typing import Dict, List, Optional, Set -from fastapi import ( - Body, - Depends, - FastAPI, - Header, - HTTPException, - Path, - Query, - Response, - status, -) +from fastapi import (Body, Depends, FastAPI, Header, HTTPException, Path, + Query, Response, status) from sqlalchemy.orm import Session from tenant_dependency import TenantData, get_tenant_info -from src import db, schema -from src.config import settings -from src.health_check_easy_ocr import health_check_preprocessing -from src.send_preprocess_results import send_preprocess_result -from src.tasks import GetLanguagesTask, PreprocessingTask -from src.text_merge import merge_words_to_paragraph -from src.utils.logger import get_logger -from src.utils.minio_utils import convert_bucket_name_if_s3prefix -from src.utils.utils import map_finish_status_for_assets +from processing import db, schema +from processing.config import settings +from processing.health_check_easy_ocr import health_check_preprocessing +from processing.send_preprocess_results import send_preprocess_result +from processing.tasks import GetLanguagesTask, PreprocessingTask +from processing.text_merge import merge_words_to_paragraph +from processing.utils.logger import get_logger +from processing.utils.minio_utils import convert_bucket_name_if_s3prefix +from processing.utils.utils import map_finish_status_for_assets logger = get_logger(__name__) app = FastAPI( diff --git a/processing/src/schema.py b/processing/processing/schema.py similarity index 100% rename from processing/src/schema.py rename to processing/processing/schema.py diff --git a/processing/src/send_preprocess_results.py b/processing/processing/send_preprocess_results.py similarity index 95% rename from processing/src/send_preprocess_results.py rename to processing/processing/send_preprocess_results.py index a43a291f7..955807047 100644 --- a/processing/src/send_preprocess_results.py +++ b/processing/processing/send_preprocess_results.py @@ -5,8 +5,8 @@ from fastapi import HTTPException from minio.error import MinioException -from src.utils.logger import get_logger -from src.utils.minio_utils import MinioCommunicator +from processing.utils.logger import get_logger +from processing.utils.minio_utils import MinioCommunicator logger = get_logger(__name__) diff --git a/processing/src/tasks.py b/processing/processing/tasks.py similarity index 93% rename from processing/src/tasks.py rename to processing/processing/tasks.py index 2f60e1319..db20fd90b 100644 --- a/processing/src/tasks.py +++ b/processing/processing/tasks.py @@ -10,16 +10,12 @@ from fastapi import HTTPException, status from sqlalchemy.orm import Session -from src.config import settings -from src.schema import PreprocessingStatus, Status -from src.utils.aiohttp_utils import send_request -from src.utils.logger import get_log_exception_msg, get_logger -from src.utils.utils import ( - execute_pipeline, - get_files_data, - get_model_url, - split_iterable, -) +from processing.config import settings +from processing.schema import PreprocessingStatus, Status +from processing.utils.aiohttp_utils import send_request +from processing.utils.logger import get_log_exception_msg, get_logger +from processing.utils.utils import (execute_pipeline, get_files_data, + get_model_url, split_iterable) logger = get_logger(__name__) diff --git a/processing/src/text_merge.py b/processing/processing/text_merge.py similarity index 95% rename from processing/src/text_merge.py rename to processing/processing/text_merge.py index eaced1055..2858d9fb5 100644 --- a/processing/src/text_merge.py +++ b/processing/processing/text_merge.py @@ -12,11 +12,11 @@ from fastapi import HTTPException from minio.error import MinioException -from src import schema -from src.schema import AnnotationData, MatchedPage, Page, ParagraphBbox -from src.third_party_code.box_util import stitch_boxes_into_lines -from src.third_party_code.table import BorderBox -from src.utils.minio_utils import MinioCommunicator +from processing import schema +from processing.schema import AnnotationData, MatchedPage, Page, ParagraphBbox +from processing.third_party_code.box_util import stitch_boxes_into_lines +from processing.third_party_code.table import BorderBox +from processing.utils.minio_utils import MinioCommunicator logger = logging.getLogger(__name__) diff --git a/processing/src/third_party_code/LICENSE-2.0_box_utils.txt b/processing/processing/third_party_code/LICENSE-2.0_box_utils.txt similarity index 100% rename from processing/src/third_party_code/LICENSE-2.0_box_utils.txt rename to processing/processing/third_party_code/LICENSE-2.0_box_utils.txt diff --git a/taxonomy/app/__init__.py b/processing/processing/third_party_code/__init__.py similarity index 100% rename from taxonomy/app/__init__.py rename to processing/processing/third_party_code/__init__.py diff --git a/processing/src/third_party_code/box_util.py b/processing/processing/third_party_code/box_util.py similarity index 100% rename from processing/src/third_party_code/box_util.py rename to processing/processing/third_party_code/box_util.py diff --git a/processing/src/third_party_code/table.py b/processing/processing/third_party_code/table.py similarity index 100% rename from processing/src/third_party_code/table.py rename to processing/processing/third_party_code/table.py diff --git a/taxonomy/app/microservice_communication/__init__.py b/processing/processing/utils/__init__.py similarity index 100% rename from taxonomy/app/microservice_communication/__init__.py rename to processing/processing/utils/__init__.py diff --git a/processing/src/utils/aiohttp_utils.py b/processing/processing/utils/aiohttp_utils.py similarity index 94% rename from processing/src/utils/aiohttp_utils.py rename to processing/processing/utils/aiohttp_utils.py index e9fe86a49..9f7abec55 100644 --- a/processing/src/utils/aiohttp_utils.py +++ b/processing/processing/utils/aiohttp_utils.py @@ -5,8 +5,8 @@ from aiohttp import ContentTypeError from fastapi import HTTPException -from src.config import settings -from src.utils.logger import get_logger +from processing.config import settings +from processing.utils.logger import get_logger logger = get_logger(__name__) diff --git a/processing/src/utils/logger.py b/processing/processing/utils/logger.py similarity index 97% rename from processing/src/utils/logger.py rename to processing/processing/utils/logger.py index 9ad39fe59..de0dcebb7 100644 --- a/processing/src/utils/logger.py +++ b/processing/processing/utils/logger.py @@ -2,7 +2,7 @@ import traceback from logging.config import dictConfig -from src.config import settings +from processing.config import settings log_config = { "version": 1, diff --git a/processing/src/utils/minio_utils.py b/processing/processing/utils/minio_utils.py similarity index 96% rename from processing/src/utils/minio_utils.py rename to processing/processing/utils/minio_utils.py index 2568688b6..166fcf5e1 100644 --- a/processing/src/utils/minio_utils.py +++ b/processing/processing/utils/minio_utils.py @@ -1,8 +1,8 @@ from minio import Minio from minio.credentials import AWSConfigProvider, EnvAWSProvider, IamAwsProvider -from src.config import settings -from src.utils.logger import get_logger +from processing.config import settings +from processing.utils.logger import get_logger logger = get_logger(__name__) diff --git a/processing/src/utils/utils.py b/processing/processing/utils/utils.py similarity index 95% rename from processing/src/utils/utils.py rename to processing/processing/utils/utils.py index 829087826..b9753cd50 100644 --- a/processing/src/utils/utils.py +++ b/processing/processing/utils/utils.py @@ -4,10 +4,10 @@ from cache import AsyncTTL from sqlalchemy.orm import Session -from src import db, schema -from src.config import settings -from src.utils.aiohttp_utils import send_request -from src.utils.logger import get_log_exception_msg, get_logger +from processing import db, schema +from processing.config import settings +from processing.utils.aiohttp_utils import send_request +from processing.utils.logger import get_log_exception_msg, get_logger logger = get_logger(__name__) T = TypeVar("T") diff --git a/processing/src/db/__init__.py b/processing/src/db/__init__.py deleted file mode 100644 index 037de0603..000000000 --- a/processing/src/db/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -import src.db.models -import src.db.service # noqa: F401 diff --git a/processing/tests/conftest.py b/processing/tests/conftest.py index 7b00f2178..138182aed 100644 --- a/processing/tests/conftest.py +++ b/processing/tests/conftest.py @@ -3,15 +3,15 @@ from unittest.mock import patch import pytest +from alembic import command +from alembic.config import Config from sqlalchemy import create_engine from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.orm import sessionmaker from sqlalchemy_utils import create_database, database_exists, drop_database -from alembic import command -from alembic.config import Config -from src.config import settings -from src.db.service import get_test_db_url +from processing.config import settings +from processing.db.service import get_test_db_url pytest_plugins = ["docker_compose"] diff --git a/processing/tests/integration/test_integration.py b/processing/tests/integration/test_integration.py index b6bcbe7a3..9cd6ae1ec 100644 --- a/processing/tests/integration/test_integration.py +++ b/processing/tests/integration/test_integration.py @@ -10,7 +10,6 @@ from requests.adapters import HTTPAdapter from urllib3.util.retry import Retry - pytestmark = pytest.mark.integration load_dotenv("./.env") @@ -97,7 +96,9 @@ def file_id(minio_client): return file_id -@pytest.mark.skip("Fails with ValueError: Unable to find `/processing/docker-compose.yml` for integration tests.") +@pytest.mark.skip( + "Fails with ValueError: Unable to find `/processing/docker-compose.yml` for integration tests." +) def test_minio_ok(minio_url, minio_client, file_id): objs = minio_client.list_objects( BUCKET, f"files/{file_id}", recursive=True @@ -110,14 +111,18 @@ def test_minio_ok(minio_url, minio_client, file_id): } -@pytest.mark.skip("Fails with ValueError: Unable to find `/processing/docker-compose.yml` for integration tests.") +@pytest.mark.skip( + "Fails with ValueError: Unable to find `/processing/docker-compose.yml` for integration tests." +) def test_url(minio_url, processing_url, preprocessing_url): assert "0.0.0.0:9000" in str(minio_url) assert "0.0.0.0:8080" in str(processing_url) assert "0.0.0.0:65432" in str(preprocessing_url) -@pytest.mark.skip("Fails with ValueError: Unable to find `/processing/docker-compose.yml` for integration tests.") +@pytest.mark.skip( + "Fails with ValueError: Unable to find `/processing/docker-compose.yml` for integration tests." +) def test_get_preprocessing_results_all_pages(processing_url, file_id): response = requests.get( url=processing_url.rstrip("/") + f"/tokens/{file_id}", @@ -127,7 +132,9 @@ def test_get_preprocessing_results_all_pages(processing_url, file_id): assert response.json() == [json.load(file1), json.load(file2)] -@pytest.mark.skip("Fails with ValueError: Unable to find `/processing/docker-compose.yml` for integration tests.") +@pytest.mark.skip( + "Fails with ValueError: Unable to find `/processing/docker-compose.yml` for integration tests." +) def test_get_preprocessing_results_some_pages(processing_url, file_id): response = requests.get( url=processing_url.rstrip("/") + f"/tokens/{file_id}", @@ -140,7 +147,9 @@ def test_get_preprocessing_results_some_pages(processing_url, file_id): assert response.json() == [file2_json] -@pytest.mark.skip("Fails with ValueError: Unable to find `/processing/docker-compose.yml` for integration tests.") +@pytest.mark.skip( + "Fails with ValueError: Unable to find `/processing/docker-compose.yml` for integration tests." +) def test_send_request_to_preprocessing( preprocessing_url, processing_url, minio_client, monkeypatch ): diff --git a/processing/tests/test_assets_status.py b/processing/tests/test_assets_status.py index 6e4fe04f7..5d7e5a981 100644 --- a/processing/tests/test_assets_status.py +++ b/processing/tests/test_assets_status.py @@ -2,8 +2,8 @@ import pytest -from src.config import settings -from src.tasks import PreprocessingTask +from processing.config import settings +from processing.tasks import PreprocessingTask @pytest.mark.skip @@ -24,12 +24,15 @@ def mock_preprocessing_task(): @pytest.mark.asyncio @pytest.mark.parametrize("status", ["failed", "in_progress", "preprocessed"]) async def test_send_status_to_assets(mock_preprocessing_task, status): - with patch("src.tasks.send_request") as mock: + with patch("processing.tasks.send_request") as mock: await mock_preprocessing_task.send_status_to_assets(status) mock.assert_awaited_once_with( method="PUT", url=settings.assets_url, - json={"file": int(mock_preprocessing_task.file_id), "status": status}, + json={ + "file": int(mock_preprocessing_task.file_id), + "status": status, + }, headers={ "X-Current-Tenant": mock_preprocessing_task.tenant, "Authorization": f"Bearer {mock_preprocessing_task.token}", diff --git a/processing/tests/test_text_merge.py b/processing/tests/test_text_merge.py index 0722150e1..9a1e63cbf 100644 --- a/processing/tests/test_text_merge.py +++ b/processing/tests/test_text_merge.py @@ -1,19 +1,9 @@ from unittest.mock import patch -from src.schema import ( - AnnotationData, - MatchedPage, - Page, - ParagraphBbox, - PageSize, - Input, -) -from src.text_merge import ( - convert_points_to_pixels, - match_page, - download_files, - stitch_boxes, -) +from processing.schema import (AnnotationData, Input, MatchedPage, Page, + PageSize, ParagraphBbox) +from processing.text_merge import (convert_points_to_pixels, download_files, + match_page, stitch_boxes) class ClientObj: @@ -223,7 +213,7 @@ def test_stitch_boxes(self): ), ] - @patch("src.text_merge.MinioCommunicator", return_value=MC()) + @patch("processing.text_merge.MinioCommunicator", return_value=MC()) def test_download(self, _1, tmp_path): request_data = AnnotationData( file="some_path/some_file.pdf", diff --git a/processing/tests/test_utils/test_utils.py b/processing/tests/test_utils/test_utils.py index 6ff15dfc7..4f8422c58 100644 --- a/processing/tests/test_utils/test_utils.py +++ b/processing/tests/test_utils/test_utils.py @@ -5,8 +5,8 @@ import responses from fastapi import HTTPException -from src.config import settings -from src.utils import utils +from processing.config import settings +from processing.utils import utils class MockResponse: diff --git a/scheduler/alembic.ini b/scheduler/alembic.ini index cc418411f..2c518b7e0 100644 --- a/scheduler/alembic.ini +++ b/scheduler/alembic.ini @@ -7,6 +7,10 @@ script_location = alembic # template used to generate migration files # file_template = %%(rev)s_%%(slug)s +# sys.path path, will be prepended to sys.path if present. +# defaults to the current working directory. +prepend_sys_path = . + # timezone to use when rendering the date # within the migration file as well as the filename. # string value is passed to dateutil.tz.gettz() diff --git a/scheduler/alembic/env.py b/scheduler/alembic/env.py index f09d345e8..6f106ff76 100644 --- a/scheduler/alembic/env.py +++ b/scheduler/alembic/env.py @@ -1,5 +1,6 @@ -from logging import config as logging_config import os +from logging import config as logging_config + import sqlalchemy from alembic import context from scheduler.db import models diff --git a/scheduler/scheduler/app.py b/scheduler/scheduler/app.py index 99c8b402f..ee1f93ba3 100644 --- a/scheduler/scheduler/app.py +++ b/scheduler/scheduler/app.py @@ -3,9 +3,9 @@ import tenant_dependency from fastapi import Depends, FastAPI, Header, HTTPException, status +from scheduler.db import service from scheduler import config, heartbeat, kafka_utils, log, runner, schemas -from scheduler.db import service logger = log.get_logger(__name__) diff --git a/scheduler/scheduler/db/service.py b/scheduler/scheduler/db/service.py index 51b83ceb4..a51684b6b 100644 --- a/scheduler/scheduler/db/service.py +++ b/scheduler/scheduler/db/service.py @@ -2,10 +2,10 @@ from typing import Any, Dict, List, Union import sqlalchemy +from scheduler.db import models from sqlalchemy import orm from scheduler import config, unit -from scheduler.db import models engine = sqlalchemy.create_engine( config.DB_URL, pool_size=int(config.POOL_SIZE) diff --git a/scheduler/scheduler/heartbeat.py b/scheduler/scheduler/heartbeat.py index 1fbf94e3e..a96374b1c 100644 --- a/scheduler/scheduler/heartbeat.py +++ b/scheduler/scheduler/heartbeat.py @@ -3,10 +3,10 @@ import random from aiokafka import AIOKafkaProducer +from scheduler.db import models, service from sqlalchemy import orm from scheduler import config, log, runner -from scheduler.db import models, service logger = log.get_logger(__name__) diff --git a/scheduler/scheduler/runner.py b/scheduler/scheduler/runner.py index 6d8c39791..c7a33bacd 100644 --- a/scheduler/scheduler/runner.py +++ b/scheduler/scheduler/runner.py @@ -2,9 +2,9 @@ import uuid import aiokafka +from scheduler.db import models from scheduler import exceptions, log, unit -from scheduler.db import models logger = log.get_logger(__name__) runner_id: str = str(uuid.uuid4()) diff --git a/scheduler/scheduler/unit.py b/scheduler/scheduler/unit.py index f3c021323..6e6424761 100644 --- a/scheduler/scheduler/unit.py +++ b/scheduler/scheduler/unit.py @@ -6,10 +6,10 @@ import aiohttp import aiokafka +from scheduler.db import models, service from sqlalchemy import exc from scheduler import config, exceptions, log -from scheduler.db import models, service logger = log.get_logger(__name__) diff --git a/scheduler/tests/test_heartbeat.py b/scheduler/tests/test_heartbeat.py index b035c7f06..f63a62614 100644 --- a/scheduler/tests/test_heartbeat.py +++ b/scheduler/tests/test_heartbeat.py @@ -5,9 +5,9 @@ import pytest from freezegun import freeze_time +from scheduler.db import models from scheduler import heartbeat, unit -from scheduler.db import models @freeze_time("2020-01-01") diff --git a/scheduler/tests/test_service.py b/scheduler/tests/test_service.py index 31a32cf22..07837cbd3 100644 --- a/scheduler/tests/test_service.py +++ b/scheduler/tests/test_service.py @@ -1,6 +1,7 @@ -from scheduler.db import models, service from unittest import mock +from scheduler.db import models, service + def test_add_into_db(testing_session, testing_unit_instance): service.add_into_db(testing_session, testing_unit_instance) diff --git a/scheduler/tests/test_unit.py b/scheduler/tests/test_unit.py index d14efdfc6..cf2b9b18c 100644 --- a/scheduler/tests/test_unit.py +++ b/scheduler/tests/test_unit.py @@ -1,6 +1,6 @@ +import pytest from tests import testing_data -import pytest from scheduler import exceptions, unit diff --git a/search/search/harvester.py b/search/search/harvester.py index d29ff3898..df95b1291 100644 --- a/search/search/harvester.py +++ b/search/search/harvester.py @@ -41,7 +41,8 @@ def create_boto3_config(): "- s3_credentials_provider is not set" ) logger.info( - f"S3_Credentials provider - {settings.s3_credentials_provider}") + f"S3_Credentials provider - {settings.s3_credentials_provider}" + ) return boto3_config diff --git a/search/search/schemas/pieces.py b/search/search/schemas/pieces.py index d2931b2d5..638257f07 100644 --- a/search/search/schemas/pieces.py +++ b/search/search/schemas/pieces.py @@ -32,12 +32,8 @@ def pieces_condition(properties: Dict[str, Any]) -> List[str]: class GeomObject(pydantic.BaseModel): category: str = pydantic.Field(..., example="Header") content: str = pydantic.Field(..., example="ElasticSearch") - document_id: pydantic.conint(ge=1) = pydantic.Field( - ..., example=1 - ) # type: ignore - page_number: pydantic.conint(ge=1) = pydantic.Field( - ..., example=1 - ) # type: ignore + document_id: pydantic.conint(ge=1) = pydantic.Field(..., example=1) # type: ignore + page_number: pydantic.conint(ge=1) = pydantic.Field(..., example=1) # type: ignore bbox: Optional[ pydantic.conlist(float, min_items=4, max_items=4) ] = pydantic.Field( @@ -46,9 +42,7 @@ class GeomObject(pydantic.BaseModel): tokens: Optional[List[str]] = pydantic.Field( None, example=["token1", "token2", "token3"] ) - job_id: pydantic.conint(ge=1) = pydantic.Field( - ..., example=1 - ) # type: ignore + job_id: pydantic.conint(ge=1) = pydantic.Field(..., example=1) # type: ignore class SearchResultSchema(pydantic.BaseModel): diff --git a/search/tests/conftest.py b/search/tests/conftest.py index 7a1d2436c..e5c88003b 100644 --- a/search/tests/conftest.py +++ b/search/tests/conftest.py @@ -6,17 +6,12 @@ from elasticsearch import AsyncElasticsearch from kafka.errors import TopicAlreadyExistsError from moto import mock_s3 +from tests.test_get import CHILD_CATEGORIES_DATA, TEST_DATA +from tests.test_harvester import (DOCS_IN_ES, INDEX_NAME, MANIFESTS, + S3_FAIL_PAGES, S3_PAGES) from search.config import settings from search.es import INDEX_SETTINGS -from tests.test_get import CHILD_CATEGORIES_DATA, TEST_DATA -from tests.test_harvester import ( - DOCS_IN_ES, - INDEX_NAME, - MANIFESTS, - S3_FAIL_PAGES, - S3_PAGES, -) BUCKET_NAME = INDEX_NAME diff --git a/search/tests/test_harvester.py b/search/tests/test_harvester.py index 029d486c9..44395b7aa 100644 --- a/search/tests/test_harvester.py +++ b/search/tests/test_harvester.py @@ -2,6 +2,7 @@ from unittest.mock import Mock import pytest + from search.harvester import parse_json, start_harvester from .override_app_dependency import TEST_TENANT diff --git a/taxonomy/Dockerfile b/taxonomy/Dockerfile index b9cd3ba01..b708b2d3d 100644 --- a/taxonomy/Dockerfile +++ b/taxonomy/Dockerfile @@ -14,7 +14,7 @@ RUN curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/inst # Copy using poetry.lock in case it doesn't exist yet COPY pyproject.toml poetry.lock .env alembic.ini version.txt ./ COPY ./alembic ./alembic -COPY ./app ./app +COPY taxonomy ./app FROM base as build diff --git a/taxonomy/alembic/env.py b/taxonomy/alembic/env.py index ad3916fdd..4cef73004 100644 --- a/taxonomy/alembic/env.py +++ b/taxonomy/alembic/env.py @@ -1,10 +1,10 @@ import os from logging.config import fileConfig +from alembic import context # type: ignore from sqlalchemy import engine_from_config, pool -from alembic import context # type: ignore -from app.database import SQLALCHEMY_DATABASE_URL, get_test_db_url +from taxonomy.database import SQLALCHEMY_DATABASE_URL, get_test_db_url # this is the Alembic Config object, which provides # access to the values within the .ini file in use. @@ -14,14 +14,16 @@ # This line sets up loggers basically. fileConfig(config.config_file_name) -from app.models import Base # noqa E402 +from taxonomy.models import Base # noqa E402 target_metadata = Base.metadata if not os.getenv("USE_TEST_DB"): config.set_main_option("sqlalchemy.url", SQLALCHEMY_DATABASE_URL) else: - config.set_main_option("sqlalchemy.url", get_test_db_url(SQLALCHEMY_DATABASE_URL)) + config.set_main_option( + "sqlalchemy.url", get_test_db_url(SQLALCHEMY_DATABASE_URL) + ) def run_migrations_offline(): diff --git a/taxonomy/alembic/versions/48dc50decbed_add_association_taxonomy_category.py b/taxonomy/alembic/versions/48dc50decbed_add_association_taxonomy_category.py index 0a5a8062f..25bc3778c 100644 --- a/taxonomy/alembic/versions/48dc50decbed_add_association_taxonomy_category.py +++ b/taxonomy/alembic/versions/48dc50decbed_add_association_taxonomy_category.py @@ -6,7 +6,6 @@ """ import sqlalchemy as sa - from alembic import op # revision identifiers, used by Alembic. diff --git a/taxonomy/alembic/versions/bdea8a93cafe_first_revision.py b/taxonomy/alembic/versions/bdea8a93cafe_first_revision.py index 3adec516a..5ece0c493 100644 --- a/taxonomy/alembic/versions/bdea8a93cafe_first_revision.py +++ b/taxonomy/alembic/versions/bdea8a93cafe_first_revision.py @@ -7,7 +7,6 @@ """ import sqlalchemy as sa import sqlalchemy_utils - from alembic import op # revision identifiers, used by Alembic. diff --git a/taxonomy/alembic/versions/d3ba69ca9d97_change_category_linking.py b/taxonomy/alembic/versions/d3ba69ca9d97_change_category_linking.py index 1dafe6073..8e37244bf 100644 --- a/taxonomy/alembic/versions/d3ba69ca9d97_change_category_linking.py +++ b/taxonomy/alembic/versions/d3ba69ca9d97_change_category_linking.py @@ -6,7 +6,6 @@ """ import sqlalchemy as sa - from alembic import op # revision identifiers, used by Alembic. diff --git a/taxonomy/app/schemas/__init__.py b/taxonomy/app/schemas/__init__.py deleted file mode 100644 index 0e5abc6aa..000000000 --- a/taxonomy/app/schemas/__init__.py +++ /dev/null @@ -1,33 +0,0 @@ -from app.schemas.errors import ( - BadRequestErrorSchema, - ConnectionErrorSchema, - NotFoundErrorSchema, -) -from app.schemas.taxon import ( - ParentsConcatenateResponseSchema, - TaxonBaseSchema, - TaxonInputSchema, - TaxonResponseSchema, -) -from app.schemas.taxonomy import ( - CategoryLinkSchema, - JobTaxonomySchema, - TaxonomyBaseSchema, - TaxonomyInputSchema, - TaxonomyResponseSchema, -) - -__all__ = [ - BadRequestErrorSchema, - ConnectionErrorSchema, - NotFoundErrorSchema, - TaxonBaseSchema, - TaxonInputSchema, - TaxonResponseSchema, - CategoryLinkSchema, - ParentsConcatenateResponseSchema, - TaxonomyBaseSchema, - TaxonomyInputSchema, - TaxonomyResponseSchema, - JobTaxonomySchema, -] diff --git a/taxonomy/documentation/update_docs.py b/taxonomy/documentation/update_docs.py index d49ee5071..8e10cf8ab 100644 --- a/taxonomy/documentation/update_docs.py +++ b/taxonomy/documentation/update_docs.py @@ -1,6 +1,6 @@ import yaml -from app.main import app +from taxonomy.main import app def str_presenter(dumper, data): diff --git a/taxonomy/app/taxon/__init__.py b/taxonomy/taxonomy/__init__.py similarity index 100% rename from taxonomy/app/taxon/__init__.py rename to taxonomy/taxonomy/__init__.py diff --git a/taxonomy/app/database.py b/taxonomy/taxonomy/database.py similarity index 100% rename from taxonomy/app/database.py rename to taxonomy/taxonomy/database.py diff --git a/taxonomy/app/errors.py b/taxonomy/taxonomy/errors.py similarity index 100% rename from taxonomy/app/errors.py rename to taxonomy/taxonomy/errors.py diff --git a/taxonomy/app/filters.py b/taxonomy/taxonomy/filters.py similarity index 79% rename from taxonomy/app/filters.py rename to taxonomy/taxonomy/filters.py index e415e80f0..1b2e981e2 100644 --- a/taxonomy/app/filters.py +++ b/taxonomy/taxonomy/filters.py @@ -1,6 +1,6 @@ from filter_lib import create_filter_model -from app.models import Taxon, Taxonomy +from taxonomy.models import Taxon, Taxonomy TaxonFilter = create_filter_model(Taxon, exclude=["tenant"]) TaxonomyFilter = create_filter_model(Taxonomy, exclude=["tenant"]) diff --git a/taxonomy/app/logging_setup.py b/taxonomy/taxonomy/logging_setup.py similarity index 100% rename from taxonomy/app/logging_setup.py rename to taxonomy/taxonomy/logging_setup.py diff --git a/taxonomy/app/main.py b/taxonomy/taxonomy/main.py similarity index 64% rename from taxonomy/app/main.py rename to taxonomy/taxonomy/main.py index c341ee67a..af43d8406 100644 --- a/taxonomy/app/main.py +++ b/taxonomy/taxonomy/main.py @@ -5,26 +5,18 @@ from fastapi import Depends, FastAPI from sqlalchemy.exc import DBAPIError, SQLAlchemyError -from app.errors import ( - CheckFieldError, - FieldConstraintError, - ForeignKeyError, - NoTaxonError, - NoTaxonomyError, - SelfParentError, - check_field_error_handler, - db_dbapi_error_handler, - db_sa_error_handler, - field_constraint_error_handler, - foreign_key_error_handler, - no_taxon_error_handler, - no_taxonomy_error_handler, - taxon_parent_child_error_handler, -) -from app.tags import TAGS -from app.taxon import resources as taxon_resources -from app.taxonomy import resources as taxonomy_resources -from app.token_dependency import TOKEN +from taxonomy.errors import (CheckFieldError, FieldConstraintError, + ForeignKeyError, NoTaxonError, NoTaxonomyError, + SelfParentError, check_field_error_handler, + db_dbapi_error_handler, db_sa_error_handler, + field_constraint_error_handler, + foreign_key_error_handler, no_taxon_error_handler, + no_taxonomy_error_handler, + taxon_parent_child_error_handler) +from taxonomy.tags import TAGS +from taxonomy.taxon import resources as taxon_resources +from taxonomy.taxonomy import resources as taxonomy_resources +from taxonomy.token_dependency import TOKEN load_dotenv(find_dotenv()) diff --git a/taxonomy/app/taxonomy/__init__.py b/taxonomy/taxonomy/microservice_communication/__init__.py similarity index 100% rename from taxonomy/app/taxonomy/__init__.py rename to taxonomy/taxonomy/microservice_communication/__init__.py diff --git a/taxonomy/app/microservice_communication/search.py b/taxonomy/taxonomy/microservice_communication/search.py similarity index 100% rename from taxonomy/app/microservice_communication/search.py rename to taxonomy/taxonomy/microservice_communication/search.py diff --git a/taxonomy/app/models.py b/taxonomy/taxonomy/models.py similarity index 93% rename from taxonomy/app/models.py rename to taxonomy/taxonomy/models.py index 975174316..8e6072959 100644 --- a/taxonomy/app/models.py +++ b/taxonomy/taxonomy/models.py @@ -1,21 +1,13 @@ from typing import Callable from uuid import uuid4 -from sqlalchemy import ( - VARCHAR, - Boolean, - CheckConstraint, - Column, - ForeignKey, - ForeignKeyConstraint, - Index, - Integer, -) +from sqlalchemy import (VARCHAR, Boolean, CheckConstraint, Column, ForeignKey, + ForeignKeyConstraint, Index, Integer) from sqlalchemy.orm import relationship, validates from sqlalchemy_utils import Ltree, LtreeType -from app.database import Base -from app.errors import CheckFieldError +from taxonomy.database import Base +from taxonomy.errors import CheckFieldError def default_tree(column_name: str) -> Callable: diff --git a/taxonomy/taxonomy/schemas/__init__.py b/taxonomy/taxonomy/schemas/__init__.py new file mode 100644 index 000000000..0a7fd55ca --- /dev/null +++ b/taxonomy/taxonomy/schemas/__init__.py @@ -0,0 +1,24 @@ +from taxonomy.schemas.errors import (BadRequestErrorSchema, + ConnectionErrorSchema, + NotFoundErrorSchema) +from taxonomy.schemas.taxon import (ParentsConcatenateResponseSchema, + TaxonBaseSchema, TaxonInputSchema, + TaxonResponseSchema) +from taxonomy.schemas.taxonomy import (CategoryLinkSchema, JobTaxonomySchema, + TaxonomyBaseSchema, TaxonomyInputSchema, + TaxonomyResponseSchema) + +__all__ = [ + BadRequestErrorSchema, + ConnectionErrorSchema, + NotFoundErrorSchema, + TaxonBaseSchema, + TaxonInputSchema, + TaxonResponseSchema, + CategoryLinkSchema, + ParentsConcatenateResponseSchema, + TaxonomyBaseSchema, + TaxonomyInputSchema, + TaxonomyResponseSchema, + JobTaxonomySchema, +] diff --git a/taxonomy/app/schemas/errors.py b/taxonomy/taxonomy/schemas/errors.py similarity index 100% rename from taxonomy/app/schemas/errors.py rename to taxonomy/taxonomy/schemas/errors.py diff --git a/taxonomy/app/schemas/taxon.py b/taxonomy/taxonomy/schemas/taxon.py similarity index 97% rename from taxonomy/app/schemas/taxon.py rename to taxonomy/taxonomy/schemas/taxon.py index 030225b53..b5d352e09 100644 --- a/taxonomy/app/schemas/taxon.py +++ b/taxonomy/taxonomy/schemas/taxon.py @@ -2,7 +2,7 @@ from pydantic import BaseModel, Field, validator -from app.errors import CheckFieldError +from taxonomy.errors import CheckFieldError class TaxonBaseSchema(BaseModel): diff --git a/taxonomy/app/schemas/taxonomy.py b/taxonomy/taxonomy/schemas/taxonomy.py similarity index 100% rename from taxonomy/app/schemas/taxonomy.py rename to taxonomy/taxonomy/schemas/taxonomy.py diff --git a/taxonomy/app/tags.py b/taxonomy/taxonomy/tags.py similarity index 100% rename from taxonomy/app/tags.py rename to taxonomy/taxonomy/tags.py diff --git a/users/src/__init__.py b/taxonomy/taxonomy/taxon/__init__.py similarity index 100% rename from users/src/__init__.py rename to taxonomy/taxonomy/taxon/__init__.py diff --git a/taxonomy/app/taxon/resources.py b/taxonomy/taxonomy/taxon/resources.py similarity index 82% rename from taxonomy/app/taxon/resources.py rename to taxonomy/taxonomy/taxon/resources.py index 5eeff7da3..2b6c45ec0 100644 --- a/taxonomy/app/taxon/resources.py +++ b/taxonomy/taxonomy/taxon/resources.py @@ -5,30 +5,20 @@ from sqlalchemy.orm import Session from sqlalchemy_filters.exceptions import BadFilterFormat -from app.database import get_db -from app.errors import NoTaxonError -from app.filters import TaxonFilter -from app.microservice_communication.search import X_CURRENT_TENANT_HEADER -from app.schemas import ( - BadRequestErrorSchema, - ConnectionErrorSchema, - NotFoundErrorSchema, - ParentsConcatenateResponseSchema, - TaxonBaseSchema, - TaxonInputSchema, - TaxonResponseSchema, -) -from app.tags import TAXON_TAG -from app.taxon.services import ( - add_taxon_db, - concatenated_parents_list, - delete_taxon_db, - fetch_bunch_taxons_db, - fetch_taxon_db, - filter_taxons, - insert_taxon_tree, - update_taxon_db, -) +from taxonomy.database import get_db +from taxonomy.errors import NoTaxonError +from taxonomy.filters import TaxonFilter +from taxonomy.microservice_communication.search import X_CURRENT_TENANT_HEADER +from taxonomy.schemas import (BadRequestErrorSchema, ConnectionErrorSchema, + NotFoundErrorSchema, + ParentsConcatenateResponseSchema, + TaxonBaseSchema, TaxonInputSchema, + TaxonResponseSchema) +from taxonomy.tags import TAXON_TAG +from taxonomy.taxon.services import (add_taxon_db, concatenated_parents_list, + delete_taxon_db, fetch_bunch_taxons_db, + fetch_taxon_db, filter_taxons, + insert_taxon_tree, update_taxon_db) router = APIRouter( prefix="/taxons", diff --git a/taxonomy/app/taxon/services.py b/taxonomy/taxonomy/taxon/services.py similarity index 96% rename from taxonomy/app/taxon/services.py rename to taxonomy/taxonomy/taxon/services.py index ab0a8bc13..8fd0b5159 100644 --- a/taxonomy/app/taxon/services.py +++ b/taxonomy/taxonomy/taxon/services.py @@ -7,15 +7,12 @@ from sqlalchemy.orm.query import Query from sqlalchemy_utils import Ltree -from app.errors import CheckFieldError, NoTaxonError, SelfParentError -from app.filters import TaxonFilter -from app.models import Taxon -from app.schemas import ( - ParentsConcatenateResponseSchema, - TaxonInputSchema, - TaxonResponseSchema, -) -from app.taxonomy.services import get_latest_taxonomy, get_taxonomy +from taxonomy.errors import CheckFieldError, NoTaxonError, SelfParentError +from taxonomy.filters import TaxonFilter +from taxonomy.models import Taxon +from taxonomy.schemas import (ParentsConcatenateResponseSchema, + TaxonInputSchema, TaxonResponseSchema) +from taxonomy.taxonomy.services import get_latest_taxonomy, get_taxonomy TaxonIdT = str TaxonPathT = str diff --git a/users/src/keycloak/__init__.py b/taxonomy/taxonomy/taxonomy/__init__.py similarity index 100% rename from users/src/keycloak/__init__.py rename to taxonomy/taxonomy/taxonomy/__init__.py diff --git a/taxonomy/app/taxonomy/resources.py b/taxonomy/taxonomy/taxonomy/resources.py similarity index 90% rename from taxonomy/app/taxonomy/resources.py rename to taxonomy/taxonomy/taxonomy/resources.py index 48f03e761..decab25e5 100644 --- a/taxonomy/app/taxonomy/resources.py +++ b/taxonomy/taxonomy/taxonomy/resources.py @@ -5,36 +5,26 @@ from sqlalchemy.orm import Session from sqlalchemy_filters.exceptions import BadFilterFormat -from app.database import get_db -from app.filters import TaxonomyFilter -from app.logging_setup import LOGGER -from app.microservice_communication.search import X_CURRENT_TENANT_HEADER -from app.schemas import ( - BadRequestErrorSchema, - CategoryLinkSchema, - ConnectionErrorSchema, - JobTaxonomySchema, - NotFoundErrorSchema, - TaxonomyBaseSchema, - TaxonomyInputSchema, - TaxonomyResponseSchema, -) -from app.tags import TAXONOMY_TAG -from app.taxonomy.services import ( - batch_latest_taxonomies, - batch_versioned_taxonomies, - bulk_create_relations_with_categories, - bulk_delete_category_association, - create_taxonomy_instance, - delete_taxonomy_instance, - filter_taxonomies, - get_latest_taxonomy, - get_linked_taxonomies, - get_second_latest_taxonomy, - get_taxonomies_by_job_id, - get_taxonomy, - update_taxonomy_instance, -) +from taxonomy.database import get_db +from taxonomy.filters import TaxonomyFilter +from taxonomy.logging_setup import LOGGER +from taxonomy.microservice_communication.search import X_CURRENT_TENANT_HEADER +from taxonomy.schemas import (BadRequestErrorSchema, CategoryLinkSchema, + ConnectionErrorSchema, JobTaxonomySchema, + NotFoundErrorSchema, TaxonomyBaseSchema, + TaxonomyInputSchema, TaxonomyResponseSchema) +from taxonomy.tags import TAXONOMY_TAG +from taxonomy.taxonomy.services import (batch_latest_taxonomies, + batch_versioned_taxonomies, + bulk_create_relations_with_categories, + bulk_delete_category_association, + create_taxonomy_instance, + delete_taxonomy_instance, + filter_taxonomies, get_latest_taxonomy, + get_linked_taxonomies, + get_second_latest_taxonomy, + get_taxonomies_by_job_id, get_taxonomy, + update_taxonomy_instance) router = APIRouter( prefix="/taxonomy", diff --git a/taxonomy/app/taxonomy/services.py b/taxonomy/taxonomy/taxonomy/services.py similarity index 95% rename from taxonomy/app/taxonomy/services.py rename to taxonomy/taxonomy/taxonomy/services.py index 68a36c83f..6cb6e9a66 100644 --- a/taxonomy/app/taxonomy/services.py +++ b/taxonomy/taxonomy/taxonomy/services.py @@ -4,16 +4,12 @@ from sqlalchemy import and_, desc, null, or_ from sqlalchemy.orm import Query, Session -from app.errors import CheckFieldError -from app.filters import TaxonomyFilter -from app.models import AssociationTaxonomyCategory, Taxonomy -from app.schemas import ( - CategoryLinkSchema, - JobTaxonomySchema, - TaxonomyBaseSchema, - TaxonomyInputSchema, - TaxonomyResponseSchema, -) +from taxonomy.errors import CheckFieldError +from taxonomy.filters import TaxonomyFilter +from taxonomy.models import AssociationTaxonomyCategory, Taxonomy +from taxonomy.schemas import (CategoryLinkSchema, JobTaxonomySchema, + TaxonomyBaseSchema, TaxonomyInputSchema, + TaxonomyResponseSchema) def create_taxonomy_instance( diff --git a/taxonomy/app/token_dependency.py b/taxonomy/taxonomy/token_dependency.py similarity index 100% rename from taxonomy/app/token_dependency.py rename to taxonomy/taxonomy/token_dependency.py diff --git a/taxonomy/tests/conftest.py b/taxonomy/tests/conftest.py index 8f582d43d..3d6b13f68 100644 --- a/taxonomy/tests/conftest.py +++ b/taxonomy/tests/conftest.py @@ -7,23 +7,25 @@ import pytest import sqlalchemy +from alembic import command +from alembic.config import Config from fastapi.testclient import TestClient from sqlalchemy import create_engine from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.orm import Session, sessionmaker from sqlalchemy_utils import create_database, database_exists, drop_database - -from alembic import command -from alembic.config import Config -from app.database import SQLALCHEMY_DATABASE_URL, Base, get_db, get_test_db_url -from app.main import app -from app.models import Taxon, Taxonomy -from app.schemas import CategoryLinkSchema, TaxonInputSchema, TaxonomyInputSchema -from app.taxon import services as taxon_services -from app.taxonomy import services as taxonomy_services -from app.token_dependency import TOKEN from tests.override_app_dependency import TEST_TENANTS, override +from taxonomy.database import (SQLALCHEMY_DATABASE_URL, Base, get_db, + get_test_db_url) +from taxonomy.main import app +from taxonomy.models import Taxon, Taxonomy +from taxonomy.schemas import (CategoryLinkSchema, TaxonInputSchema, + TaxonomyInputSchema) +from taxonomy.taxon import services as taxon_services +from taxonomy.taxonomy import services as taxonomy_services +from taxonomy.token_dependency import TOKEN + @pytest.fixture def client() -> TestClient: diff --git a/taxonomy/tests/test_taxon_crud.py b/taxonomy/tests/test_taxon_crud.py index 482704386..13cb107dd 100644 --- a/taxonomy/tests/test_taxon_crud.py +++ b/taxonomy/tests/test_taxon_crud.py @@ -3,10 +3,10 @@ from typing import Any, List, Optional import pytest - -from app.models import Taxon from tests.override_app_dependency import TEST_HEADER +from taxonomy.models import Taxon + TAXON_PATH = "/taxons" diff --git a/taxonomy/tests/test_taxonomy_router.py b/taxonomy/tests/test_taxonomy_router.py index 2efdcc8ef..6db0551d9 100644 --- a/taxonomy/tests/test_taxonomy_router.py +++ b/taxonomy/tests/test_taxonomy_router.py @@ -1,12 +1,12 @@ from typing import Tuple import pytest - -from app.models import Taxonomy -from app.schemas import CategoryLinkSchema -from app.taxonomy import services from tests.override_app_dependency import TEST_HEADER, TEST_TENANTS +from taxonomy.models import Taxonomy +from taxonomy.schemas import CategoryLinkSchema +from taxonomy.taxonomy import services + @pytest.mark.integration def test_create_taxonomy_should_work(overrided_token_client, db_session): diff --git a/users/Dockerfile b/users/Dockerfile index 332f5db44..898991d7e 100644 --- a/users/Dockerfile +++ b/users/Dockerfile @@ -3,7 +3,7 @@ FROM ${base_image} as build WORKDIR /opt/users_filter -COPY src /opt/users_filter/src +COPY users /opt/users_filter/src COPY requirements.txt /opt/users_filter COPY .env /opt/users_filter @@ -22,7 +22,7 @@ CMD pytest -vvv FROM sonarsource/sonar-scanner-cli:4.6 AS sonar -COPY src /sonar/src +COPY users /sonar/src COPY tests /sonar/tests COPY sonar-project.properties /sonar/sonar-project.properties diff --git a/users/tests/keycloak/test_query.py b/users/tests/keycloak/test_query.py index cb1ead1a7..13c127229 100644 --- a/users/tests/keycloak/test_query.py +++ b/users/tests/keycloak/test_query.py @@ -1,16 +1,15 @@ -"""Testing src/keycloak/query.py.""" +"""Testing users/keycloak/query.py.""" import json -from unittest.mock import patch, create_autospec +from unittest.mock import create_autospec, patch import pytest - -import src.keycloak.query as query -import src.keycloak.schemas as schemas +import users.keycloak.query as query +import users.keycloak.schemas as schemas @pytest.fixture def request_mock(): - with patch("src.keycloak.query.aiohttp.request") as mock: + with patch("users.keycloak.query.aiohttp.request") as mock: yield mock diff --git a/users/tests/keycloak/test_schemas.py b/users/tests/keycloak/test_schemas.py index 47a513dfe..b1d542415 100644 --- a/users/tests/keycloak/test_schemas.py +++ b/users/tests/keycloak/test_schemas.py @@ -1,6 +1,6 @@ -import src.keycloak.utils as kc_utils -import src.keycloak.schemas as kc_schemas import pytest +import users.keycloak.schemas as kc_schemas +import users.keycloak.utils as kc_utils user_1 = kc_schemas.User(username="user", id="1") user_2 = kc_schemas.User(username="u__r", id="2") diff --git a/users/tests/keycloak/test_utils.py b/users/tests/keycloak/test_utils.py index 8e88add38..070e271e1 100644 --- a/users/tests/keycloak/test_utils.py +++ b/users/tests/keycloak/test_utils.py @@ -1,6 +1,6 @@ -import src.keycloak.utils as kc_utils -from src.schemas import Users import pytest +import users.keycloak.utils as kc_utils +from users.schemas import Users @pytest.fixture @@ -28,4 +28,8 @@ def test_create_filters_with_empty_request_body(request_body): def test_create_filters(request_body): users = Users(filters=request_body) filters = kc_utils.create_filters(users) - assert filters == {"name": "h", "id": ["user_id"], "role": "role-annotator"} + assert filters == { + "name": "h", + "id": ["user_id"], + "role": "role-annotator", + } diff --git a/users/tests/test_main.py b/users/tests/test_main.py index e34462a31..48ef902df 100644 --- a/users/tests/test_main.py +++ b/users/tests/test_main.py @@ -2,12 +2,11 @@ from unittest.mock import patch import pytest +import users.keycloak.schemas as kc_schemas from fastapi import HTTPException from fastapi.testclient import TestClient from tenant_dependency import TenantData - -import src.keycloak.schemas as kc_schemas -from src.main import app, check_authorization, tenant +from users.main import app, check_authorization, tenant client = TestClient(app) @@ -140,7 +139,7 @@ def test_check_authorization_role_is_right(mock_tenant_data): check_authorization(token=mock_tenant_data, role="role") -@patch("src.keycloak.query.get_token_v2", return_value=token_schema) +@patch("users.keycloak.query.get_token_v2", return_value=token_schema) def test_login_body(token_schema): response = client.post( "/token", @@ -153,7 +152,7 @@ def test_login_body(token_schema): assert response.json() == token_representation -@patch("src.keycloak.query.get_token_v2", return_value=token_schema) +@patch("users.keycloak.query.get_token_v2", return_value=token_schema) @pytest.mark.parametrize( ("request_body", "status_code"), [ @@ -208,7 +207,7 @@ def test_login_status_code(token_schema, request_body, status_code): assert response.status_code == status_code -@patch("src.keycloak.query.get_user", return_value=user_1) +@patch("users.keycloak.query.get_user", return_value=user_1) class TestGetUserGWT: def test_get_user_jwt_body(self, mock_user, user_representation): response = client.get("/users/current") @@ -221,7 +220,7 @@ def test_get_user_jwt_status_code(self, mock_user): assert response.status_code == 200 -@patch("src.keycloak.query.get_user", return_value=user_1) +@patch("users.keycloak.query.get_user", return_value=user_1) class TestGetUser: def test_get_user_body(self, mock_user, user_representation): response = client.get("/users/user-id") @@ -238,7 +237,8 @@ def test_get_user_info_from_token_introspection( mocked_token1, mocked_token1_data ): with patch( - "src.keycloak.query.introspect_token", return_value=mocked_token1_data + "users.keycloak.query.introspect_token", + return_value=mocked_token1_data, ): response = client.get( "/users/current_v2", @@ -252,7 +252,7 @@ def test_get_user_info_from_token_introspection( mock_all_groups = [group_1, group_2] -@patch("src.keycloak.query.get_groups", return_value=mock_all_groups) +@patch("users.keycloak.query.get_groups", return_value=mock_all_groups) class TestGetTenants: def test_get_tenants_body(self, mock_groups): response = client.get("/tenants") @@ -263,8 +263,8 @@ def test_get_tenants_status_code(self, mock_groups): assert response.status_code == 200 -@patch("src.keycloak.query.create_group", return_value=None) -@patch("src.s3.create_bucket", return_value=None) +@patch("users.keycloak.query.create_group", return_value=None) +@patch("users.s3.create_bucket", return_value=None) class TestCreateTenant: def test_create_tenant_body(self, mock_group, mock_bucket): response = client.post("/tenants?tenant=tenant") @@ -291,10 +291,10 @@ def test_create_tenant_status_code( assert response.status_code == response_status_code -@patch("src.keycloak.query.get_groups", return_value=mock_all_groups) -@patch("src.keycloak.query.get_user", return_value=user_1) -@patch("src.keycloak.schemas.User.add_tenant", return_value=None) -@patch("src.keycloak.query.update_user", return_value=None) +@patch("users.keycloak.query.get_groups", return_value=mock_all_groups) +@patch("users.keycloak.query.get_user", return_value=user_1) +@patch("users.keycloak.schemas.User.add_tenant", return_value=None) +@patch("users.keycloak.query.update_user", return_value=None) class TestAddUserToTenant: @pytest.mark.parametrize( ("tenant", "expected_result"), @@ -335,8 +335,8 @@ def test_add_user_to_tenant2( assert response.status_code == expected_result -@patch("src.keycloak.query.get_user", return_value=user_1) -@patch("src.keycloak.query.update_user", return_value=None) +@patch("users.keycloak.query.get_user", return_value=user_1) +@patch("users.keycloak.query.update_user", return_value=None) @pytest.mark.parametrize( ("tenant", "expected_result"), [ @@ -350,8 +350,8 @@ def test_remove_user_from_tenant_body( assert response.json() == expected_result -@patch("src.keycloak.query.get_user", return_value=user_1) -@patch("src.keycloak.query.update_user", return_value=None) +@patch("users.keycloak.query.get_user", return_value=user_1) +@patch("users.keycloak.query.update_user", return_value=None) @pytest.mark.parametrize( ("tenant", "expected_result"), [ @@ -365,9 +365,9 @@ def test_remove_user_from_tenant_status_code( assert response.status_code == 200 -@patch("src.keycloak.query.get_users_v2", return_value=mock_all_users) +@patch("users.keycloak.query.get_users_v2", return_value=mock_all_users) @patch( - "src.keycloak.query.get_users_by_role", return_value=mock_users_with_role + "users.keycloak.query.get_users_by_role", return_value=mock_users_with_role ) class TestUsersSearch: @pytest.mark.parametrize("request_body", [{}, {"filters": []}]) @@ -769,9 +769,9 @@ def test_filter_users_by_all_filters_when_user_does_not_exist_body2( mock_users = [user] -@patch("src.keycloak.query.create_user", return_value=None) -@patch("src.keycloak.query.get_users_v2", return_value=mock_all_users) -@patch("src.keycloak.query.execute_action_email", return_value=None) +@patch("users.keycloak.query.create_user", return_value=None) +@patch("users.keycloak.query.get_users_v2", return_value=mock_all_users) +@patch("users.keycloak.query.execute_action_email", return_value=None) class TestUserRegistration: def test_user_registration_body(self, user, mock_all_users, action_email): response = client.post("/users/registration?email=mail@mail.ru") @@ -802,10 +802,10 @@ def test_get_idp_names_and_SSOauth_links( mocked_admin_auth_data, mocked_identity_providers_data ): with patch( - "src.keycloak.query.get_master_realm_auth_data", + "users.keycloak.query.get_master_realm_auth_data", return_value=mocked_admin_auth_data, ), patch( - "src.keycloak.query.get_identity_providers_data", + "users.keycloak.query.get_identity_providers_data", return_value=mocked_identity_providers_data, ): response = client.get("/identity_providers_data") diff --git a/users/tests/test_schemas.py b/users/tests/test_schemas.py index 8dccb2e15..c3fb39be1 100644 --- a/users/tests/test_schemas.py +++ b/users/tests/test_schemas.py @@ -1,4 +1,4 @@ -from src.schemas import Users +from users.schemas import Users def test_users_schemas(): diff --git a/users/tests/test_utils.py b/users/tests/test_utils.py index 04b445465..c3426d602 100644 --- a/users/tests/test_utils.py +++ b/users/tests/test_utils.py @@ -1,7 +1,8 @@ from unittest.mock import patch import pytest -from src import utils + +from users import utils def test_extract_idp_data_needed(): @@ -53,5 +54,5 @@ def test_extract_idp_data_needed(): ("prefix", "expected"), (("", "tenant"), ("prefix", "prefix-tenant")) ) def test_bucket_dependency(prefix: str, expected: str) -> None: - with patch("src.config.S3_PREFIX", prefix): + with patch("users.config.S3_PREFIX", prefix): assert utils.get_bucket_name("tenant") == expected diff --git a/users/users/__init__.py b/users/users/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/users/src/config.py b/users/users/config.py similarity index 100% rename from users/src/config.py rename to users/users/config.py diff --git a/users/users/keycloak/__init__.py b/users/users/keycloak/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/users/src/keycloak/query.py b/users/users/keycloak/query.py similarity index 94% rename from users/src/keycloak/query.py rename to users/users/keycloak/query.py index a5f753f28..2bc920b6f 100644 --- a/users/src/keycloak/query.py +++ b/users/users/keycloak/query.py @@ -1,11 +1,12 @@ from typing import Any, Dict, List, TypedDict, Union import aiohttp -import src.config as config -import src.keycloak.resources as resources -import src.keycloak.schemas as schemas +import users.config as config +import users.keycloak.resources as resources +import users.keycloak.schemas as schemas from fastapi import HTTPException, status -from src import logger + +from users import logger class AuthData(TypedDict): @@ -75,7 +76,8 @@ async def get_users_by_role( async def get_token_v2( - realm: str, request_form: Union[schemas.TokenRequest, schemas.RefreshTokenRequest] + realm: str, + request_form: Union[schemas.TokenRequest, schemas.RefreshTokenRequest], ) -> schemas.TokenResponse: """Get access token. @@ -171,7 +173,9 @@ async def introspect_token(token: str) -> Token_Data: ) return data_to_return except aiohttp.ClientConnectionError as e: - logger.Logger.error("Exception while sending request to Keycloak: %s", e) + logger.Logger.error( + "Exception while sending request to Keycloak: %s", e + ) raise HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=f"Exception while sending request to Keycloak: {e}", @@ -277,7 +281,8 @@ async def get_master_realm_auth_data() -> AuthData: } url = resources.token_uri.substitute(realm="master") logger.Logger.info( - "Sending request to Keycloak url: %s to get admin auth data, " "payload: %s", + "Sending request to Keycloak url: %s to get admin auth data, " + "payload: %s", url, payload, ) @@ -295,7 +300,9 @@ async def get_master_realm_auth_data() -> AuthData: return data_to_return except aiohttp.ClientConnectionError as e: - logger.Logger.error("Exception while sending request to Keycloak: %s", e) + logger.Logger.error( + "Exception while sending request to Keycloak: %s", e + ) raise HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=f"Exception while sending request to Keycloak: {e}", @@ -322,7 +329,9 @@ async def get_identity_providers_data( return await resp.json() except aiohttp.ClientConnectionError as e: - logger.Logger.error("Exception while sending request to Keycloak: %s", e) + logger.Logger.error( + "Exception while sending request to Keycloak: %s", e + ) raise HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=f"Exception while sending request to Keycloak: {e}", diff --git a/users/src/keycloak/resources.py b/users/users/keycloak/resources.py similarity index 96% rename from users/src/keycloak/resources.py rename to users/users/keycloak/resources.py index a6b334893..bf07f4c94 100644 --- a/users/src/keycloak/resources.py +++ b/users/users/keycloak/resources.py @@ -4,7 +4,7 @@ from string import Template from urllib.parse import urljoin -from src.config import KEYCLOAK_ENDPOINT +from users.config import KEYCLOAK_ENDPOINT def join_paths(*args: str) -> str: diff --git a/users/src/keycloak/schemas.py b/users/users/keycloak/schemas.py similarity index 99% rename from users/src/keycloak/schemas.py rename to users/users/keycloak/schemas.py index a46d9d5c4..bf791e8d9 100644 --- a/users/src/keycloak/schemas.py +++ b/users/users/keycloak/schemas.py @@ -283,6 +283,7 @@ class Config: class OAuthRequest(BaseModel): """Base class for authorization requests""" + client_id: Optional[str] grant_type: str client_secret: Optional[str] @@ -317,6 +318,7 @@ def from_fastapi_form( class RefreshTokenRequest(OAuthRequest): """Represents Keycloak token refreshment request""" + client_id: str = "admin-cli" grant_type: str = "refresh_token" refresh_token: str diff --git a/users/src/keycloak/utils.py b/users/users/keycloak/utils.py similarity index 78% rename from users/src/keycloak/utils.py rename to users/users/keycloak/utils.py index 46e85bf91..b2b77b835 100644 --- a/users/src/keycloak/utils.py +++ b/users/users/keycloak/utils.py @@ -1,7 +1,7 @@ from typing import Any, Dict, List, Optional, Union -import src.keycloak.schemas as schemas -from src.schemas import Users +import users.keycloak.schemas as schemas +from users.schemas import Users def create_filters(users: Users) -> Dict[str, Union[str, None]]: diff --git a/users/src/logger.py b/users/users/logger.py similarity index 100% rename from users/src/logger.py rename to users/users/logger.py diff --git a/users/src/main.py b/users/users/main.py similarity index 96% rename from users/src/main.py rename to users/users/main.py index 34be85d1f..6b3aaa468 100644 --- a/users/src/main.py +++ b/users/users/main.py @@ -2,26 +2,24 @@ import aiohttp import pydantic -import src.config as conf -import src.keycloak.query as kc_query -import src.keycloak.schemas as kc_schemas -import src.keycloak.utils as kc_utils +import users.config as conf +import users.keycloak.query as kc_query +import users.keycloak.schemas as kc_schemas +import users.keycloak.utils as kc_utils from aiohttp.web_exceptions import HTTPException as AIOHTTPException from apscheduler.schedulers.background import BackgroundScheduler from email_validator import EmailNotValidError, validate_email from fastapi import Depends, FastAPI, Header, HTTPException, Query, Request from fastapi.responses import JSONResponse from fastapi.security import OAuth2PasswordRequestForm -from src import s3, utils -from src.config import ( - KEYCLOAK_ROLE_ADMIN, - KEYCLOAK_USERS_PUBLIC_KEY, - ROOT_PATH, -) -from src.logger import Logger -from src.schemas import Users from tenant_dependency import TenantData, get_tenant_info from urllib3.exceptions import MaxRetryError +from users.config import (KEYCLOAK_ROLE_ADMIN, KEYCLOAK_USERS_PUBLIC_KEY, + ROOT_PATH) +from users.logger import Logger +from users.schemas import Users + +from users import s3, utils app = FastAPI(title="users", root_path=ROOT_PATH, version="0.1.2") realm = conf.KEYCLOAK_REALM diff --git a/users/src/s3.py b/users/users/s3.py similarity index 98% rename from users/src/s3.py rename to users/users/s3.py index 3f138b541..f6faab06f 100644 --- a/users/src/s3.py +++ b/users/users/s3.py @@ -2,7 +2,8 @@ from typing import Any, Dict, Optional from minio import Minio, credentials -from src import config, logger + +from users import config, logger class S3Providers(str, enum.Enum): diff --git a/users/src/schemas.py b/users/users/schemas.py similarity index 89% rename from users/src/schemas.py rename to users/users/schemas.py index bbe6831dc..ac11db41f 100644 --- a/users/src/schemas.py +++ b/users/users/schemas.py @@ -51,4 +51,6 @@ class FilterUserUserName(BaseModel): class Users(BaseModel): - filters: Optional[List[Union[FilterUserUserName, FilterUserUserID, FilterRole]]] + filters: Optional[ + List[Union[FilterUserUserName, FilterUserUserID, FilterRole]] + ] diff --git a/users/src/utils.py b/users/users/utils.py similarity index 98% rename from users/src/utils.py rename to users/users/utils.py index d7b980f84..d39e2849f 100644 --- a/users/src/utils.py +++ b/users/users/utils.py @@ -2,7 +2,8 @@ from typing import Any, Dict, List, Optional from minio import Minio -from src import config + +from users import config def extract_idp_data_needed( diff --git a/web/local.env b/web/local.env new file mode 100644 index 000000000..0124d4af7 --- /dev/null +++ b/web/local.env @@ -0,0 +1,41 @@ +REACT_APP_FE_API_NAMESPACES_FILEMANAGEMENT=/docs + +BASE_URL=http://localhost + +ANNOTATION_PORT=8000 +ASSETS_PORT=8001 +CONVERT_PORT=8002 +JOBS_PORT=8003 +MODELS_PORT=8004 +PIPELINES_PORT=8005 +PROCESSING_PORT=8006 +SCHEDULER_PORT=8007 +SEARCH_PORT=8008 +TAXONOMY_PORT=8009 +USERS_PORT=8010 + +REACT_APP_FILEMANAGEMENT_API=${BASE_URL}:${ASSETS_PORT} +REACT_APP_JOB_API=${BASE_URL}:${JOBS_PORT} +REACT_APP_PIPELINES_API=${BASE_URL}:${PIPELINES_PORT} +REACT_APP_CATEGORIES_API=${BASE_URL}:${ANNOTATION_PORT} +REACT_APP_TOKENS_API=${BASE_URL}:${PROCESSING_PORT} +REACT_APP_AUTH_API=${BASE_URL}:8080 +REACT_APP_USERS_API=${BASE_URL}:${USERS_PORT} +REACT_APP_MODELS_API=${BASE_URL}:${MODELS_PORT} +REACT_APP_SEARCH_API=${BASE_URL}:${SEARCH_PORT} +REACT_APP_TAXONOMIES_API=${BASE_URL}:${TAXONOMY_PORT} + +REACT_APP_FILEMANAGEMENT_API_NAMESPACE=/ +REACT_APP_JOBMANAGER_API_NAMESPACE=/ +REACT_APP_PIPELINES_API_NAMESPACE=/ +REACT_APP_CATEGORIES_API_NAMESPACE=/ +REACT_APP_TOKENS_API_NAMESPACE=/ +REACT_APP_AUTH_API_NAMESPACE="/auth" +REACT_APP_USERS_API_NAMESPACE=/ +REACT_APP_MODELS_API_NAMESPACE=/ +REACT_APP_SEARCH_API_NAMESPACE=/ +REACT_APP_TAXONOMIES_API_NAMESPACE=/ + +REACT_APP_AUTH_CLIENT_ID=admin-cli + +REACT_APP_ALLOW_MOCKS=false