fix: fix error handling in files_data_to_pipeline_arg #963

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open

ervandagadzhanyan wants to merge 54 commits into main from fix-err-handling

.env.example

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -1,7 +1,6 @@
  
    # !!!! Important !!!!

    # This .env file is for running local installation only

    ################################################

    # General configuration

    ################################################

    LOG_LEVEL=DEBUG

    @@ -13,47 +12,43 @@ POSTGRES_USER=postgres
  
    POSTGRES_PASSWORD=postgres

    POSTGRES_DB=badgerdoc

    # S3 configuration

    # You should repeat aws creds in both of sections

    # because minio lib doesn't use env vars

    S3_PROVIDER=minio

    ################################################

    # Storage provider configuration

    ################################################

    # Boto configuration

    # Can be one of: minio, azure, s3

    STORAGE_PROVIDER=minio

    AWS_ACCESS_KEY_ID=

    AWS_SECRET_ACCESS_KEY=

    AWS_DEFAULT_REGION=

    # Minio configuration

    # This MINIO_PUBLIC_HOST must be accessible from your browser

    MINIO_PUBLIC_HOST=http://badgerdoc-minio:9000

    # Minio dev configuration

    # Azure Blob Storage Configuration

    AZURE_BLOB_STORAGE_CONNECTION_STRING=

    S3_SECURE=false

    AWS_REGION=

    # S3 configuration

    # TODO: We need to unify this configuration, boto3 requires with http, Minio without

    # TODO: DEPRECATED S3_ENDPOINT_URL

    S3_ENDPOINT_URL=http://badgerdoc-minio:9000

    S3_ENDPOINT=badgerdoc-minio:9000

    S3_ACCESS_KEY=minioadmin

    S3_SECRET_KEY=minioadmin

    S3_PREFIX=

    S3_REGION=

    S3_SECURE=false

    MINIO_DEFAULT_BUCKETS=local

    # KeyCloak dev configuration

    # KeyCloak

    KEYCLOAK_HOST=http://badgerdoc-keycloak:8080

    KEYCLOAK_SYSTEM_USER_CLIENT=badgerdoc-internal

    KEYCLOAK_SYSTEM_USER_SECRET=

    KEYCLOAK_USER=admin

    KEYCLOAK_PASSWORD=admin

    # Gottenberg dev configuration

    GOTENBERG_HOST=gotenberg

    ################################################

    # Services

    ################################################

    ASSETS_SERVICE_SCHEME=http

    ASSETS_SERVICE_HOST=badgerdoc-assets

    @@ -62,13 +57,14 @@ ASSETS_SERVICE_PORT=8080
  
    ANNOTATION_SERVICE_SCHEME=http

    ANNOTATION_SERVICE_HOST=badgerdoc-annotation

    ANNOTATION_SERVICE_PORT=8080

    AGREEMENT_SCORE_SERVICE_HOST=localhost:5000 # TODO: add prefix ANNOTATION_

    JOBS_SERVICE_SCHEME=http

    JOBS_SERVICE_HOST=badgerdoc-jobs

    JOBS_SERVICE_PORT=8080

    JOBS_RUN_PIPELINES_WITH_SIGNED_URL=False

    # pre signed urls expire in 48 hours

    S3_PRE_SIGNED_EXPIRES_HOURS=48

    JOBS_SIGNED_URL_ENABLED=False

    JOBS_SIGNED_URL_TTL=12

    JOBS_SIGNED_URL_KEY_NAME=

    TAXONOMY_SERVICE_SCHEME=http

    TAXONOMY_SERVICE_HOST=badgerdoc-taxonomy

    @@ -91,11 +87,12 @@ USERS_SERVICE_PORT=8080
  
    # Web configuration

    WEB_CORS=*

    KAFKA_BOOTSTRAP_SERVER=badgerdoc-kafka:9092 # TODO: remove port

    KAFKA_SEARCH_TOPIC=search

    AGREEMENT_SCORE_SERVICE_HOST=localhost:5000 # TODO: remove port

    MAX_REQ_SIZE=100M

    ################################################

    # Pipelines

    ################################################

    # Airflow configuration

    AIRFLOW_ENABLED=True

    @@ -106,7 +103,6 @@ AIRFLOW_SERVICE_HOST=airflow-webserver
  
    AIRFLOW_SERVICE_PORT=8080

    AIRFLOW_SERVICE_PATH_PREFIX=/api/v1

    # Databricks configuration

    DATABRICKS_ENABLED=False

.github/workflows/annotation.yml

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -1,38 +1,38 @@
  
    name: annotation precommit and test

    run-name: annotation precommit and test

    name: annotation

    run-name: annotation

    on:

      push:

        paths:

          - 'annotation/**'

          - "annotation/**"

      pull_request:

        paths:

          - 'annotation/**'

          - "annotation/**"

    jobs:

      annotation-pre-commit-actions:

      linters:

        strategy:

          matrix:

            python-version: [ "3.8.15" ]

            python-version: ["3.8.15"]

        runs-on: ubuntu-latest

        steps:

          - uses: actions/checkout@v3

          - uses: actions/setup-python@v4

            with:

              python-version: ${{ matrix.python-version }}

              cache: 'pip'

              cache: "pip"

          - run: pip install flake8

          - run: flake8 --extend-ignore=E203 annotation

      annotation-build:

      tests:

        runs-on: ubuntu-latest

        strategy:

          matrix:

            python-version: [ "3.8.15" ]

            python-version: ["3.8.15"]

        steps:

          - uses: actions/checkout@v3

          - name: Set up Python ${{ matrix.python-version }}

            uses: actions/setup-python@v4

            with:

              python-version: ${{ matrix.python-version }}

              cache: 'pip'

              cache: "pip"

          - name: Install dependencies and run tests

            run: |

              python -m pip install --upgrade pip

    @@ -41,6 +41,7 @@ jobs:
  
              poetry install --no-root

              poetry add ../lib/filter_lib

              poetry add ../lib/tenants

              poetry add ../lib/badgerdoc_storage

              poetry run pytest

            env:

              POSTGRES_HOST: 127.0.0.1

.github/workflows/assets.yml

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -1,40 +1,40 @@
  
    name: assets precommit and test

    run-name: assets precommit and test

    name: assets

    run-name: assets

    on:

      push:

        paths:

          - 'assets/**'

          - '.github/workflows/assets.yml'

          - "assets/**"

          - ".github/workflows/assets.yml"

      pull_request:

        paths:

          - 'assets/**'

          - '.github/workflows/assets.yml'

          - "assets/**"

          - ".github/workflows/assets.yml"

    jobs:

      assets-pre-commit-actions:

      linters:

        strategy:

          matrix:

            python-version: [ "3.8.15" ]

            python-version: ["3.8.15"]

        runs-on: ubuntu-latest

        steps:

          - uses: actions/checkout@v3

          - uses: actions/setup-python@v4

            with:

              python-version: ${{ matrix.python-version }}

              cache: 'pip'

              cache: "pip"

          - run: pip install flake8

          - run: flake8 --extend-ignore=E203 assets

      assets-build:

      tests:

        runs-on: ubuntu-latest

        strategy:

          matrix:

            python-version: [ "3.8.15" ]

            python-version: ["3.8.15"]

        steps:

          - uses: actions/checkout@v3

          - name: Set up Python ${{ matrix.python-version }}

            uses: actions/setup-python@v4

            with:

              python-version: ${{ matrix.python-version }}

              cache: 'pip'

              cache: "pip"

          - name: Install dependencies

            run: |

              python -m pip install --upgrade pip

    @@ -44,6 +44,7 @@ jobs:
  
              poetry install --no-root --no-interaction

              poetry add ../lib/filter_lib

              poetry add ../lib/tenants

              poetry add ../lib/badgerdoc_storage

          - name: Test with pytest

            run: |

              cd assets

.github/workflows/badgerdoc-build.yml

This file was deleted.

.github/workflows/badgerdoc-storage.yml

-Original file line number
+Diff line change
@@ -0,0 +1,32 @@
+    name: badgerdoc_storage
+    run-name: badgerdoc_storage
+    on:
+      push:
+        paths:
+          - lib/badgerdoc_storage/**
+          - .github/worlflows/badgerdoc_storage.yml
+      pull_request:
+        paths:
+          - lib/badgerdoc_storage/**
+          - .github/worlflows/badgerdoc_storage.yml
+    jobs:
+      linters:
+        strategy:
+          matrix:
+            python-version: ["3.8.15", "3.12"]
+        runs-on: ubuntu-latest
+        defaults:
+          run:
+            working-directory: ./lib/badgerdoc_storage
+        steps:
+          - uses: actions/checkout@v3
+          - uses: actions/setup-python@v4
+            with:
+              python-version: ${{ matrix.python-version }}
+              cache: "pip"
+          - run: pip install poetry==1.7.1
+          - run: poetry install
+          - run: poetry run pylint src/
+          - run: poetry run mypy src/
+          - run: poetry run black --check src/
+          - run: poetry run isort --check src/

.github/workflows/badgerdoc.yml

-Original file line number
+Diff line change
@@ -0,0 +1,53 @@
+    name: badgerdoc
+    on:
+      pull_request:
+        branches:
+          - "**"
+      push:
+        branches:
+          - "**"
+      pull_request_target:
+        types:
+          - opened
+          - edited
+          - synchronize
+    env:
+      PYTHON_VERSION: 3.8
+    jobs:
+      linters:
+        runs-on: ubuntu-latest
+        steps:
+          - name: Checkout
+            uses: actions/checkout@v4
+          - name: Setup python ${{ env.PYTHON_VERSION }}
+            uses: actions/setup-python@v5
+            with:
+              python-version: ${{ env.PYTHON_VERSION }}
+          - name: Install python dependencies
+            run: |
+              python -m pip install --upgrade pip
+              pip install poetry
+              poetry install --only dev
+          - name: Run Isort
+            run: |
+              poetry run isort --diff --check-only .
+          - name: Run Black
+            run: |
+              poetry run black --config pyproject.toml --diff --check .
+      build:
+        if: github.event_name == 'pull_request' && github.base_ref == 'main'
+        runs-on: ubuntu-latest
+        steps:
+          - uses: actions/checkout@v4
+          - name: Build services
+            run: make build_badgerdoc
+      semantic_pr:
+        if: github.event_name == 'pull_request_target'
+        runs-on: ubuntu-latest
+        steps:
+          - uses: amannn/action-semantic-pull-request@v5
+            env:
+              GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/convert.yml

This file was deleted.

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

fix: fix error handling in files_data_to_pipeline_arg #963

Uh oh!

Diff view

Diff view

There are no files selected for viewing

Uh oh!

Uh oh!

Uh oh!

fix: fix error handling in files_data_to_pipeline_arg #963

Are you sure you want to change the base?

Uh oh!

fix: fix error handling in files_data_to_pipeline_arg #963

Uh oh!

Uh oh!

Diff view

Diff view

There are no files selected for viewing

Uh oh!

Uh oh!

Uh oh!