diff --git a/.github/workflows/annotation.yml b/.github/workflows/annotation.yml index 42395a351..6b842a048 100644 --- a/.github/workflows/annotation.yml +++ b/.github/workflows/annotation.yml @@ -63,6 +63,7 @@ jobs: poetry install --no-root poetry add ../lib/filter_lib poetry add ../lib/tenants + poetry add ../lib/badgerdoc_cli poetry run alembic upgrade head poetry run pytest env: diff --git a/.github/workflows/assets.yml b/.github/workflows/assets.yml index a855cc1ba..2664d92d9 100644 --- a/.github/workflows/assets.yml +++ b/.github/workflows/assets.yml @@ -74,6 +74,7 @@ jobs: poetry install --no-root --no-interaction poetry add ../lib/filter_lib poetry add ../lib/tenants + poetry add ../lib/badgerdoc_cli - name: Test with pytest run: | cd assets diff --git a/.github/workflows/badgerdoc-cli.yml b/.github/workflows/badgerdoc-cli.yml new file mode 100644 index 000000000..1c880d4bc --- /dev/null +++ b/.github/workflows/badgerdoc-cli.yml @@ -0,0 +1,36 @@ +name: badgerdoc_cli linters and tests +on: + push: + paths: + - lib/badgerdoc_cli/** + - .github/worlflows/badgerdoc-cli.yml + pull_request: + paths: + - lib/badgerdoc_cli/** + - .github/worlflows/badgerdoc-cli.yml +jobs: + badgerdoc-cli-test: + strategy: + matrix: + python-version: [ "3.8.15" ] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + - name: Install dependencies + working-directory: ./lib/badgerdoc_cli + run: | + python -m pip install --upgrade pip + pip install ".[dev]" + - name: Run linters and checkers [isort -> black -> mypy -> pylint] + working-directory: ./lib/badgerdoc_cli + run: | + git ls-files -- . | xargs pre-commit run --files + - name: Run tests + working-directory: ./lib/badgerdoc_cli + run: | + pytest tests/ diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml new file mode 100644 index 000000000..68ad3033d --- /dev/null +++ b/.github/workflows/documentation.yml @@ -0,0 +1,377 @@ +name: Documentation build and push +jobs: + annotation: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [ "3.8.15" ] + env: + SPEC_PATH: ./docs/openapi/annotation.json + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + - name: Install dependencies + working-directory: ./annotation + run: | + python -m pip install --upgrade pip + pip install poetry + poetry install --no-root --without dev + poetry add ../lib/filter_lib + poetry add ../lib/tenants + poetry add ../lib/badgerdoc_cli + - name: Generate documentation + working-directory: ./annotation + run: poetry run badgerdoc openapi .$SPEC_PATH + env: + PYTHONPATH: . + - name: Save documentation + uses: actions/upload-artifact@v3 + with: + path: ${{ env.SPEC_PATH }} + assets: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [ "3.8.15" ] + env: + SPEC_PATH: ./docs/openapi/assets.json + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + - name: Install dependencies + working-directory: ./assets + run: | + sudo apt-get update && sudo apt-get -y install poppler-utils + python -m pip install --upgrade pip + pip install poetry + poetry install --no-root --without dev + poetry add ../lib/filter_lib + poetry add ../lib/tenants + poetry add ../lib/badgerdoc_cli + - name: Generate documentation + working-directory: ./assets + run: poetry run badgerdoc openapi .$SPEC_PATH + env: + PYTHONPATH: . + - name: Save documentation + uses: actions/upload-artifact@v3 + with: + path: ${{ env.SPEC_PATH }} + convert: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [ "3.8.15" ] + env: + SPEC_PATH: ./docs/openapi/convert.json + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + - name: Install dependencies + working-directory: ./convert + run: | + python -m pip install --upgrade pip + pip install poetry + poetry install --no-root --without dev + poetry add ../lib/tenants + poetry add ../lib/badgerdoc_cli + - name: Generate documentation + working-directory: ./convert + run: poetry run badgerdoc openapi .$SPEC_PATH + env: + PYTHONPATH: . + - name: Save documentation + uses: actions/upload-artifact@v3 + with: + path: ${{ env.SPEC_PATH }} + jobs: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [ "3.8.15" ] + env: + SPEC_PATH: ./docs/openapi/jobs.json + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + - name: Install dependencies + working-directory: ./jobs + run: | + python -m pip install --upgrade pip + pip install poetry + poetry install --no-root --without dev + poetry add ../lib/filter_lib + poetry add ../lib/tenants + poetry add ../lib/badgerdoc_cli + - name: Generate documentation + working-directory: ./jobs + run: poetry run badgerdoc openapi .$SPEC_PATH + env: + PYTHONPATH: . + - name: Save documentation + uses: actions/upload-artifact@v3 + with: + path: ${{ env.SPEC_PATH }} + models: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [ "3.8.15" ] + env: + SPEC_PATH: ./docs/openapi/models.json + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + - name: Install dependencies + working-directory: ./models + run: | + python -m pip install --upgrade pip + pip install poetry + poetry install --no-root --without dev + poetry add ../lib/filter_lib + poetry add ../lib/tenants + poetry add ../lib/badgerdoc_cli + - name: Generate documentation + working-directory: ./models + run: poetry run badgerdoc openapi .$SPEC_PATH + env: + PYTHONPATH: . + - name: Save documentation + uses: actions/upload-artifact@v3 + with: + path: ${{ env.SPEC_PATH }} + pipelines: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [ "3.8.15" ] + env: + SPEC_PATH: ./docs/openapi/pipelines.json + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + - name: Install dependencies + working-directory: ./pipelines + run: | + python -m pip install --upgrade pip + pip install poetry + poetry install --no-root --without dev + poetry add ../lib/filter_lib + poetry add ../lib/tenants + poetry add ../lib/badgerdoc_cli + - name: Generate documentation + working-directory: ./pipelines + run: poetry run badgerdoc openapi .$SPEC_PATH + env: + PYTHONPATH: . + - name: Save documentation + uses: actions/upload-artifact@v3 + with: + path: ${{ env.SPEC_PATH }} + processing: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [ "3.8.15" ] + env: + SPEC_PATH: ./docs/openapi/processing.json + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + - name: Install dependencies + working-directory: ./processing + run: | + python -m pip install --upgrade pip + pip install poetry + poetry install --no-root --without dev + poetry add ../lib/tenants + poetry add ../lib/badgerdoc_cli + - name: Generate documentation + working-directory: ./processing + run: poetry run badgerdoc openapi .$SPEC_PATH + env: + PYTHONPATH: . + - name: Save documentation + uses: actions/upload-artifact@v3 + with: + path: ${{ env.SPEC_PATH }} + scheduler: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [ "3.8.15" ] + env: + SPEC_PATH: ./docs/openapi/scheduler.json + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + - name: Install dependencies + working-directory: ./scheduler + run: | + python -m pip install --upgrade pip + pip install poetry + poetry install --no-root --without dev + poetry add ../lib/tenants + poetry add ../lib/badgerdoc_cli + - name: Generate documentation + working-directory: ./scheduler + run: poetry run badgerdoc openapi .$SPEC_PATH + env: + PYTHONPATH: . + - name: Save documentation + uses: actions/upload-artifact@v3 + with: + path: ${{ env.SPEC_PATH }} + search: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [ "3.8.15" ] + env: + SPEC_PATH: ./docs/openapi/search.json + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + - name: Install dependencies + working-directory: ./search + run: | + python -m pip install --upgrade pip + pip install poetry + poetry install --no-root --without dev + poetry add ../lib/filter_lib + poetry add ../lib/tenants + poetry add ../lib/badgerdoc_cli + - name: Generate documentation + working-directory: ./search + run: poetry run badgerdoc openapi .$SPEC_PATH + env: + PYTHONPATH: . + - name: Save documentation + uses: actions/upload-artifact@v3 + with: + path: ${{ env.SPEC_PATH }} + taxonomy: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [ "3.8.15" ] + env: + SPEC_PATH: ./docs/openapi/taxonomy.json + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + - name: Install dependencies + working-directory: ./taxonomy + run: | + python -m pip install --upgrade pip + pip install poetry + poetry install --no-root --without dev + poetry add ../lib/filter_lib + poetry add ../lib/tenants + poetry add ../lib/badgerdoc_cli + - name: Generate documentation + working-directory: ./taxonomy + run: poetry run badgerdoc openapi .$SPEC_PATH + env: + PYTHONPATH: . + - name: Save documentation + uses: actions/upload-artifact@v3 + with: + path: ${{ env.SPEC_PATH }} + users: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [ "3.8.15" ] + env: + SPEC_PATH: ./docs/openapi/users.json + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + - name: Install dependencies + working-directory: ./users + run: | + python -m pip install --upgrade pip + pip install poetry + poetry install --no-root --without dev + poetry add ../lib/tenants + poetry add ../lib/badgerdoc_cli + - name: Generate documentation + working-directory: ./users + run: poetry run badgerdoc openapi .$SPEC_PATH + env: + PYTHONPATH: . + - name: Save documentation + uses: actions/upload-artifact@v3 + with: + path: ${{ env.SPEC_PATH }} + commit-specs: + runs-on: ubuntu-latest + needs: [annotation, assets, convert, jobs, models, pipelines, processing, scheduler, search, taxonomy, users] + env: + DOCS_PATH: ./docs/openapi + steps: + - uses: actions/checkout@v3 + - name: Download generated specs + id: download + uses: actions/download-artifact@v3 + with: + path: /tmp + - name: Extract downloaded specs + run: mv ${{ steps.download.outputs.download-path }}/artifact/*.json ${{ env.DOCS_PATH }} + - name: Push documentation + working-directory: ${{ env.DOCS_PATH }} + run: | + git add . + if git diff --quiet HEAD ./; then + echo No changes in documentation, exiting + exit 0 + fi + echo '# TODO: Do nothing right now, replace with push to S3 later' +# git -c user.name='github-actions[bot]' -c user.email='github-actions[bot]@users.noreply.github.com' \ +# commit --message "docs: update openapi specs" +# git push origin HEAD diff --git a/.gitignore b/.gitignore index c8401d2fb..2388fb972 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,46 @@ build_dir/ -.env \ No newline at end of file +.env +# Editors +.vscode/ +.idea/ + +# Mac/OSX +.DS_Store + +# Windows +Thumbs.db + +# Source for the following rules: https://raw.githubusercontent.com/github/gitignore/master/Python.gitignore +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# Distribution / packaging +build/ +dist/ +eggs/ +.eggs/ +#lib/ +lib64/ +sdist/ +wheels/ +*.egg-info/ +*.egg + +# Unit test / coverage reports +htmlcov/ +.coverage +.coverage.* +.cache +coverage.xml +*.cover +.pytest_cache/ + +# Environments +.venv +env/ +venv/ + +# mypy +.mypy_cache/ diff --git a/README.md b/README.md index feb89e5af..e54b53e0b 100644 --- a/README.md +++ b/README.md @@ -112,7 +112,7 @@ cd assets && pipenv install --dev ``` 2. Install dependencies from "lib" folder: ``` -pipenv shell && pip install -e ../lib/filter_lib ../lib/tenants +pipenv shell && pip install -e ../lib/filter_lib ../lib/tenants ../lib/badgerdoc_cli ``` # Contributors diff --git a/annotation/alembic/versions/66cd6054c2d0_add_categories_tree.py b/annotation/alembic/versions/66cd6054c2d0_add_categories_tree.py index 58c611a5f..c2aa87e8e 100644 --- a/annotation/alembic/versions/66cd6054c2d0_add_categories_tree.py +++ b/annotation/alembic/versions/66cd6054c2d0_add_categories_tree.py @@ -9,6 +9,7 @@ import sqlalchemy_utils from alembic import op +from annotation import database # revision identifiers, used by Alembic. revision = "66cd6054c2d0" @@ -19,6 +20,7 @@ def upgrade(): # ### commands auto generated by Alembic - please adjust! ### + database.install_ltree_extension() op.add_column( "categories", sa.Column( diff --git a/annotation/annotation/annotations/resources.py b/annotation/annotation/annotations/resources.py index 5071acd66..4255073b8 100644 --- a/annotation/annotation/annotations/resources.py +++ b/annotation/annotation/annotations/resources.py @@ -64,7 +64,6 @@ 500: {"model": ConnectionErrorSchema}, }, summary="Save annotation by user.", - tags=[ANNOTATION_TAG], ) def post_annotation_by_user( doc: DocForSaveSchema, @@ -232,7 +231,6 @@ def post_annotation_by_user( 500: {"model": ConnectionErrorSchema}, }, summary="Save annotation by pipeline.", - tags=[ANNOTATION_TAG], ) def post_annotation_by_pipeline( doc: DocForSaveSchema, @@ -365,7 +363,7 @@ def get_jobs_by_file_id( }, summary="Get latest revision made by particular " "user (or by pipeline) for particular pages.", - tags=[REVISION_TAG, ANNOTATION_TAG], + tags=[REVISION_TAG], ) def get_latest_revision_by_user( job_id: int = Path(..., example=3), @@ -407,7 +405,7 @@ def get_latest_revision_by_user( summary="Get annotation for given revision." "Info will be accumulated from first revision up to" "given.", - tags=[REVISION_TAG, ANNOTATION_TAG], + tags=[REVISION_TAG], ) def get_annotations_up_to_given_revision( job_id: int = Path(..., example=1), @@ -500,7 +498,7 @@ def get_annotations_up_to_given_revision( 500: {"model": ConnectionErrorSchema}, }, summary="Get annotation for latest or particular revision.", - tags=[REVISION_TAG, ANNOTATION_TAG], + tags=[REVISION_TAG], ) def get_annotation_for_given_revision( job_id: int = Path(..., example=1), @@ -544,7 +542,7 @@ def get_annotation_for_given_revision( }, summary="Get all users revisions (or pipeline revision) " "for particular pages.", - tags=[REVISION_TAG, ANNOTATION_TAG], + tags=[REVISION_TAG], ) def get_all_revisions( job_id: int, diff --git a/annotation/annotation/database.py b/annotation/annotation/database.py index 7ba749db9..d77472733 100644 --- a/annotation/annotation/database.py +++ b/annotation/annotation/database.py @@ -18,14 +18,16 @@ engine = create_engine(SQLALCHEMY_DATABASE_URL) -# Ensure LTREE extensions is installed -with engine.connect() as conn: - try: - conn.execute(sqlalchemy.sql.text("CREATE EXTENSION LTREE")) - except sqlalchemy.exc.ProgrammingError as err_: - # Exctension installed, just skip error - if "DuplicateObject" not in str(err_): - raise err_ + +def install_ltree_extension() -> None: + # Ensure LTREE extensions is installed + with engine.connect() as conn: + try: + conn.execute(sqlalchemy.sql.text("CREATE EXTENSION LTREE")) + except sqlalchemy.exc.ProgrammingError as err_: + # Exctension installed, just skip error + if "DuplicateObject" not in str(err_): + raise err_ def todict(obj): diff --git a/annotation/annotation/jobs/resources.py b/annotation/annotation/jobs/resources.py index 379fd28b1..7dbb9e804 100644 --- a/annotation/annotation/jobs/resources.py +++ b/annotation/annotation/jobs/resources.py @@ -50,7 +50,7 @@ UnassignedFilesInfoSchema, ValidationSchema, ) -from annotation.tags import FILES_TAG, JOBS_TAG +from annotation.tags import JOBS_TAG from annotation.token_dependency import TOKEN from ..models import ( @@ -82,13 +82,13 @@ router = APIRouter( prefix="/jobs", responses={500: {"model": ConnectionErrorSchema}}, + tags=[JOBS_TAG], ) @router.post( "/{job_id}", status_code=status.HTTP_201_CREATED, - tags=[JOBS_TAG], responses={ 400: {"model": BadRequestErrorSchema}, }, @@ -207,7 +207,6 @@ def post_job( @router.patch( "/{job_id}", status_code=status.HTTP_204_NO_CONTENT, - tags=[JOBS_TAG], responses={ 400: {"model": BadRequestErrorSchema}, 404: {"model": NotFoundErrorSchema}, @@ -289,7 +288,6 @@ def update_job( "/{job_id}/files", status_code=status.HTTP_200_OK, response_model=JobFilesInfoSchema, - tags=[FILES_TAG], responses={ 404: {"model": NotFoundErrorSchema}, }, @@ -337,7 +335,6 @@ def get_job_files( "/{job_id}/files/unassigned", status_code=status.HTTP_200_OK, response_model=UnassignedFilesInfoSchema, - tags=[FILES_TAG], summary="Get list of unassigned files by job id.", ) def get_unassigned_files( @@ -408,7 +405,6 @@ def get_unassigned_files( @router.post( "/{job_id}/start", - tags=[JOBS_TAG], status_code=status.HTTP_200_OK, response_model=List[ManualAnnotationTaskSchema], responses={ @@ -482,7 +478,6 @@ def start_job( "/{job_id}/users", status_code=status.HTTP_200_OK, response_model=List[Dict[str, Union[UUID, int]]], - tags=[JOBS_TAG], responses={ 404: {"model": NotFoundErrorSchema}, }, @@ -510,7 +505,6 @@ def get_users_for_job( @router.get( "/{job_id}/categories", status_code=status.HTTP_200_OK, - tags=[JOBS_TAG], response_model=Page[Union[CategoryResponseSchema, str, dict]], summary="Get list of categories for provided job_id", responses={ @@ -542,7 +536,6 @@ def fetch_job_categories( @router.post( "/{job_id}/categories/search", status_code=status.HTTP_200_OK, - tags=[JOBS_TAG], response_model=Page[Union[CategoryResponseSchema, str, dict]], summary="Search categories for provided job_id", responses={ @@ -579,7 +572,6 @@ def search_job_categories( @router.get( "", status_code=status.HTTP_200_OK, - tags=[JOBS_TAG], summary="Get info about jobs, in which provided file ids participate", ) def get_jobs_info_by_files( @@ -602,7 +594,6 @@ def get_jobs_info_by_files( "/{job_id}/progress", status_code=status.HTTP_200_OK, response_model=JobProgressSchema, - tags=[JOBS_TAG], responses={ 404: {"model": NotFoundErrorSchema}, }, diff --git a/annotation/annotation/main.py b/annotation/annotation/main.py index 45f4c283b..8937641e4 100644 --- a/annotation/annotation/main.py +++ b/annotation/annotation/main.py @@ -1,12 +1,7 @@ import os import pathlib -from botocore.exceptions import BotoCoreError, ClientError -from dotenv import find_dotenv, load_dotenv -from fastapi import Depends, FastAPI -from sqlalchemy.exc import DBAPIError, SQLAlchemyError -from starlette.requests import Request - +from annotation import database from annotation import logger as app_logger from annotation.annotations import resources as annotations_resources from annotation.categories import resources as categories_resources @@ -42,6 +37,11 @@ from annotation.tags import TAGS from annotation.tasks import resources as task_resources from annotation.token_dependency import TOKEN +from botocore.exceptions import BotoCoreError, ClientError +from dotenv import find_dotenv, load_dotenv +from fastapi import Depends, FastAPI +from sqlalchemy.exc import DBAPIError, SQLAlchemyError +from starlette.requests import Request load_dotenv(find_dotenv()) @@ -70,6 +70,11 @@ def get_version() -> str: logger = app_logger.Logger +@app.on_event("startup") +def setup_db(): + database.install_ltree_extension() + + async def catch_exceptions_middleware(request: Request, call_next): try: return await call_next(request) @@ -105,3 +110,10 @@ async def catch_exceptions_middleware(request: Request, call_next): app.add_exception_handler(DBAPIError, db_dbapi_error_handler) app.add_exception_handler(SelfParentError, category_parent_child_error_handler) app.add_exception_handler(Exception, debug_exception_handler) + + +def cli_handler() -> None: + from badgerdoc_cli import cli_handler, init_cli_app + + init_cli_app(app) + cli_handler() diff --git a/annotation/annotation/metadata/resources.py b/annotation/annotation/metadata/resources.py index f64c2aab1..9faa8d520 100644 --- a/annotation/annotation/metadata/resources.py +++ b/annotation/annotation/metadata/resources.py @@ -4,11 +4,11 @@ X_CURRENT_TENANT_HEADER, ) from annotation.schemas import EntitiesStatusesSchema -from annotation.tags import METADATA_TAG, TASKS_TAG +from annotation.tags import METADATA_TAG router = APIRouter( prefix="/metadata", - tags=[TASKS_TAG, METADATA_TAG], + tags=[METADATA_TAG], ) diff --git a/annotation/annotation/revisions/resources.py b/annotation/annotation/revisions/resources.py index 2bc837d27..702f51bbb 100644 --- a/annotation/annotation/revisions/resources.py +++ b/annotation/annotation/revisions/resources.py @@ -10,11 +10,11 @@ ) from annotation.models import AnnotatedDoc from annotation.schemas import AnnotatedDocSchema, ConnectionErrorSchema -from annotation.tags import ANNOTATION_TAG, REVISION_TAG +from annotation.tags import REVISION_TAG router = APIRouter( prefix="/revisions", - tags=[REVISION_TAG, ANNOTATION_TAG], + tags=[REVISION_TAG], responses={500: {"model": ConnectionErrorSchema}}, ) diff --git a/annotation/annotation/tasks/resources.py b/annotation/annotation/tasks/resources.py index c11087c20..33e4d7c27 100644 --- a/annotation/annotation/tasks/resources.py +++ b/annotation/annotation/tasks/resources.py @@ -72,7 +72,7 @@ ValidationEndSchema, ValidationSchema, ) -from annotation.tags import REVISION_TAG, TASKS_TAG +from annotation.tags import TASKS_TAG from annotation.tasks.validation import ( create_annotation_tasks, create_validation_tasks, @@ -716,7 +716,6 @@ def delete_batch_tasks( "/{task_id}/pages_summary", status_code=status.HTTP_200_OK, response_model=PagesInfoSchema, - tags=[REVISION_TAG], responses={ 404: {"model": NotFoundErrorSchema}, }, diff --git a/annotation/pyproject.toml b/annotation/pyproject.toml index b0dfa9a9f..752d7393f 100644 --- a/annotation/pyproject.toml +++ b/annotation/pyproject.toml @@ -20,7 +20,7 @@ cachetools = "^5.0.0" kafka-python = "^2.0.2" sqlalchemy-utils = "^0.38.3" -[tool.poetry.dev-dependencies] +[tool.poetry.group.dev.dependencies] pre-commit = "^2.14.0" commitizen = "^2.18.0" black = "^22.3.0" @@ -32,6 +32,9 @@ moto = {extras = ["s3"], version = "^2.2.8"} pytest-cov = "^3.0.0" click = "^8.1.3" +[tool.poetry.scripts] +badgerdoc = "annotation.main:cli_handler" + [build-system] requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" diff --git a/assets/assets/main.py b/assets/assets/main.py index 0f850aa04..23b979a05 100644 --- a/assets/assets/main.py +++ b/assets/assets/main.py @@ -1,8 +1,9 @@ -from assets import routers -from assets.config import settings from fastapi import Depends, FastAPI from tenant_dependency import get_tenant_info +from assets import routers +from assets.config import settings + tenant = get_tenant_info(url=settings.keycloak_uri, algorithm="RS256") @@ -18,3 +19,10 @@ app.include_router(routers.datasets_router.router) app.include_router(routers.bonds_router.router) app.include_router(routers.s3_router.router) + + +def cli_handler() -> None: + from badgerdoc_cli import cli_handler, init_cli_app + + init_cli_app(app) + cli_handler() diff --git a/assets/pyproject.toml b/assets/pyproject.toml index f725095ec..ccb83e2be 100644 --- a/assets/pyproject.toml +++ b/assets/pyproject.toml @@ -21,7 +21,7 @@ importlib-resources = "*" boto3 = "*" requests = "*" -[tool.poetry.dev-dependencies] +[tool.poetry.group.dev.dependencies] pre-commit = "*" pytest = "*" pytest-cov = "*" @@ -32,6 +32,9 @@ pylint = "*" types-requests = "*" sqlalchemy-utils = "*" +[tool.poetry.scripts] +badgerdoc = "assets.main:cli_handler" + [build-system] requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" diff --git a/common/minio_service/pyproject.toml b/common/minio_service/pyproject.toml index 77998b773..b51eab8f6 100644 --- a/common/minio_service/pyproject.toml +++ b/common/minio_service/pyproject.toml @@ -14,7 +14,7 @@ mypy-extensions = "^0.4.3" add-logging = "commands:add_logger" get-setup = "commands:get_setup" -[tool.poetry.dev-dependencies] +[tool.poetry.group.dev.dependencies] pytest = "^5.2" diff --git a/common/model_api/pyproject.toml b/common/model_api/pyproject.toml index b8408b04b..a644567a8 100644 --- a/common/model_api/pyproject.toml +++ b/common/model_api/pyproject.toml @@ -13,7 +13,7 @@ pdfplumber = "0.5.28" fastapi = "^0.70.0" uvicorn = "^0.15.0" -[tool.poetry.dev-dependencies] +[tool.poetry.group.dev.dependencies] mypy = "0.910" pytest = "^6.2.5" isort = "5.10.1" diff --git a/common/page_rendering/pyproject.toml b/common/page_rendering/pyproject.toml index 8debc7305..2134f0a71 100644 --- a/common/page_rendering/pyproject.toml +++ b/common/page_rendering/pyproject.toml @@ -14,7 +14,7 @@ pdfplumber = "^0.5.28" add-logging = "commands:add_logger" get-setup = "commands:get_setup" -[tool.poetry.dev-dependencies] +[tool.poetry.group.dev.dependencies] pytest = "^5.2" [build-system] diff --git a/convert/convert/config.py b/convert/convert/config.py index f5fcc4a8d..10080bacf 100644 --- a/convert/convert/config.py +++ b/convert/convert/config.py @@ -4,7 +4,6 @@ import boto3 from botocore.client import BaseClient -from convert import logger from dotenv import load_dotenv from mypy_extensions import KwArg, VarArg from pydantic import BaseSettings, Field @@ -12,6 +11,8 @@ from requests.adapters import HTTPAdapter from requests.packages.urllib3.util.retry import Retry +from convert import logger + load_dotenv() @@ -45,7 +46,7 @@ class Settings(BaseSettings): job_service_url: Optional[str] = os.getenv("JOB_SERVICE_URL") annotation_service_url: Optional[str] = os.getenv("ANNOTATION_SERVICE_URL") taxonomy_service_url: Optional[str] = os.getenv("TAXONOMY_SERVICE_URL") - keycloak_url: Optional[str] = os.getenv("KEYCLOAK_URL") + keycloak_url: str = os.getenv("KEYCLOAK_URL", "") def get_version() -> str: diff --git a/convert/convert/main.py b/convert/convert/main.py index b2d4f77c2..90839a158 100644 --- a/convert/convert/main.py +++ b/convert/convert/main.py @@ -16,3 +16,10 @@ app.include_router(labelstudio.router) app.include_router(text.router) app.include_router(pdf.router) + + +def cli_handler() -> None: + from badgerdoc_cli import cli_handler, init_cli_app + + init_cli_app(app) + cli_handler() diff --git a/convert/pyproject.toml b/convert/pyproject.toml index 05a293cde..bafa18b2f 100644 --- a/convert/pyproject.toml +++ b/convert/pyproject.toml @@ -20,7 +20,7 @@ requests = "^2.28.1" python-dotenv = "^0.21.0" pymupdf-fonts = "^1.0.5" -[tool.poetry.dev-dependencies] +[tool.poetry.group.dev.dependencies] pytest = "^6.2.5" pytest-cov = "^3.0.0" pytest-testinfra = "^6.4.0" @@ -31,6 +31,9 @@ black = "^22.3" datamodel-code-generator = "^0.17.1" responses = "^0.22.0" +[tool.poetry.scripts] +badgerdoc = "convert.main:cli_handler" + [tool.black] line_length = "79" diff --git a/docs/dist/favicon-16x16.png b/docs/dist/favicon-16x16.png new file mode 100644 index 000000000..8b194e617 Binary files /dev/null and b/docs/dist/favicon-16x16.png differ diff --git a/docs/dist/favicon-32x32.png b/docs/dist/favicon-32x32.png new file mode 100644 index 000000000..249737fe4 Binary files /dev/null and b/docs/dist/favicon-32x32.png differ diff --git a/docs/dist/index.css b/docs/dist/index.css new file mode 100644 index 000000000..f2376fdaa --- /dev/null +++ b/docs/dist/index.css @@ -0,0 +1,16 @@ +html { + box-sizing: border-box; + overflow: -moz-scrollbars-vertical; + overflow-y: scroll; +} + +*, +*:before, +*:after { + box-sizing: inherit; +} + +body { + margin: 0; + background: #fafafa; +} diff --git a/docs/dist/oauth2-redirect.html b/docs/dist/oauth2-redirect.html new file mode 100644 index 000000000..564091718 --- /dev/null +++ b/docs/dist/oauth2-redirect.html @@ -0,0 +1,79 @@ + + +
+>16&255,u[c++]=t>>8&255,u[c++]=255&t;2===s&&(t=n[e.charCodeAt(r)]<<2|n[e.charCodeAt(r+1)]>>4,u[c++]=255&t);1===s&&(t=n[e.charCodeAt(r)]<<10|n[e.charCodeAt(r+1)]<<4|n[e.charCodeAt(r+2)]>>2,u[c++]=t>>8&255,u[c++]=255&t);return u},t.fromByteArray=function(e){for(var t,n=e.length,o=n%3,a=[],i=16383,s=0,l=n-o;s0)throw new Error("Invalid string. Length must be a multiple of 4");var r=e.indexOf("=");return-1===r&&(r=t),[r,r===t?0:4-r%4]}function u(e,t,n){for(var o,a,i=[],s=t;ss&&(r=s-l),a=r;a>=0;a--){let r=!0;for(let n=0;n>>=0,isFinite(r)?(r>>>=0,void 0===n&&(n="utf8")):(n=r,r=void 0)}const o=this.length-t;if((void 0===r||r>o)&&(r=o),e.length>0&&(r<0||t<0)||t>this.length)throw new RangeError("Attempt to write outside buffer bounds");n||(n="utf8");let a=!1;for(;;)switch(n){case"hex":return w(this,e,t,r);case"utf8":case"utf-8":return E(this,e,t,r);case"ascii":case"latin1":case"binary":return x(this,e,t,r);case"base64":return _(this,e,t,r);case"ucs2":case"ucs-2":case"utf16le":case"utf-16le":return S(this,e,t,r);default:if(a)throw new TypeError("Unknown encoding: "+n);n=(""+n).toLowerCase(),a=!0}},l.prototype.toJSON=function(){return{type:"Buffer",data:Array.prototype.slice.call(this._arr||this,0)}};const C=4096;function O(e,t,r){let n="";r=Math.min(e.length,r);for(let o=t;o