diff --git a/.github/workflows/CDA-testing.yml b/.github/workflows/CDA-testing.yml index eb17ad52..8d5960d7 100644 --- a/.github/workflows/CDA-testing.yml +++ b/.github/workflows/CDA-testing.yml @@ -2,7 +2,7 @@ name: CI on: push: - branches: [main, githubAction-testing] + branches: [main] pull_request: branches: [main] workflow_dispatch: @@ -14,40 +14,42 @@ jobs: steps: - uses: actions/checkout@v5 - - name: set up backend - run: docker compose up --build -d + - name: Set up backend + run: | + docker compose pull + docker compose up -d - name: Set Up Python uses: actions/setup-python@v6 with: python-version: '3.9.X' - # Unlike the code-check workflow, this job requires the dev dependencies to be - # installed to make sure we have the necessary, tools, stub files, etc. - - name: Install Poetry + # Use actions-poetry to handle installation + - name: Install Poetry and Dependencies uses: abatilo/actions-poetry@v4 - - name: Add Poetry to PATH (for act) - if: env.ACT - run: echo "/root/.local/bin" >> $GITHUB_PATH + # Set Poetry to use an in-project virtual environment + - name: Configure Poetry for in-project venv + run: poetry config virtualenvs.in-project true - - name: Cache Virtual Environment + # Poetry will handle installation and caching + - name: Cache Python dependencies uses: actions/cache@v4 + id: cache-poetry-venv with: - path: ./.venv - key: venv-${{ hashFiles('poetry.lock') }} + path: .venv + key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }} - - name: Install Dependencies - run: poetry install + # Install dependencies only if cache is missed + - name: Install dependencies + if: steps.cache-poetry-venv.outputs.cache-hit != 'true' + run: poetry install --no-root # Run pytest and generate coverage report data. - - name: Run Tests - run: poetry run pytest tests/cda/ --doctest-modules --cov --cov-report=xml:out/coverage.xml - - # Run mypy with strict mode enabled. Only the main source code is type checked (test - # and example code is excluded). - - name: Check Types - run: poetry run mypy --strict cwms/ + - name: Run Tests and Check Types + run: | + poetry run pytest tests/cda/ --doctest-modules --cov --cov-report=xml:out/coverage.xml + poetry run mypy --strict cwms/ - name: Generate Coverage Report uses: irongut/CodeCoverageSummary@v1.3.0 diff --git a/docker-compose.yml b/docker-compose.yml index 47e1ec10..acba714a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,10 +10,9 @@ services: - CWMS_PASSWORD=simplecwmspasswD1 - OFFICE_ID=HQ - OFFICE_EROC=s0 - ports: - - "1526:1521" + ports: ["1526:1521"] healthcheck: - test: ["CMD","tnsping", "FREEPDB1"] + test: ["CMD", "tnsping", "FREEPDB1"] interval: 30s timeout: 50s retries: 50 @@ -32,9 +31,9 @@ services: - INSTALLONCE=1 - QUIET=1 command: > - sh -xc "sqlplus CWMS_20/$$CWMS_PASSWORD@$$DB_HOST_PORT$$DB_NAME @/setup_sql/users $$OFFICE_EROC" - volumes: - - ./compose_files/sql:/setup_sql:ro + sh -xc "sqlplus CWMS_20/$$CWMS_PASSWORD@$$DB_HOST_PORT$$DB_NAME @/setup_sql/users + $$OFFICE_EROC" + volumes: [./compose_files/sql:/setup_sql:ro] depends_on: db: condition: service_healthy @@ -43,7 +42,6 @@ services: traefik: condition: service_healthy - data-api: depends_on: auth: @@ -75,10 +73,9 @@ services: - cwms.dataapi.access.openid.altAuthUrl=http://localhost:${APP_PORT:-8082} - cwms.dataapi.access.openid.useAltWellKnown=true - cwms.dataapi.access.openid.issuer=http://localhost:${APP_PORT:-8082}/auth/realms/cwms - expose: - - 7000 + expose: [7000] healthcheck: - test: ["CMD","/usr/bin/curl", "-I","localhost:7000/cwms-data/offices/HEC"] + test: ["CMD", "/usr/bin/curl", "-I", "localhost:7000/cwms-data/offices/HEC"] interval: 5s timeout: 1s retries: 100 @@ -90,9 +87,12 @@ services: auth: image: quay.io/keycloak/keycloak:19.0.1 - command: ["start-dev", "--features-disabled=admin2","--import-realm"] + command: ["start-dev", "--import-realm"] healthcheck: - test: "/usr/bin/curl -If localhost:${APP_PORT:-8082}/auth/health/ready || exit 1" + test: + - "CMD-SHELL" + - "/usr/bin/curl -If localhost:${APP_PORT:-8082}/auth/health/ready || exit\ + \ 1" interval: 5s timeout: 1s retries: 100 @@ -108,6 +108,8 @@ services: - KC_PROXY=none - KC_HTTP_ENABLED=true - KC_HTTP_RELATIVE_PATH=/auth + - KC_HOSTNAME=localhost + - KC_DB=dev-file volumes: - ./compose_files/keycloak/realm.json:/opt/keycloak/data/import/realm.json:ro labels: @@ -119,17 +121,12 @@ services: traefik: condition: service_healthy - - # Proxy for HTTPS for OpenID traefik: image: traefik:v3.3.3 - ports: - - "${APP_PORT:-8082}:80" - expose: - - "8080" - volumes: - - "/var/run/docker.sock:/var/run/docker.sock:ro" + ports: ["${APP_PORT:-8082}:80"] + expose: ["8080"] + volumes: ["/var/run/docker.sock:/var/run/docker.sock:ro"] healthcheck: test: traefik healthcheck --ping command: @@ -142,4 +139,4 @@ services: labels: - "traefik.enable=true" - "traefik.http.routers.traefik.rule=PathPrefix(`/traefik`)" - - "traefik.http.routers.traefik.service=api@internal" \ No newline at end of file + - "traefik.http.routers.traefik.service=api@internal" diff --git a/tests/cda/blobs/blob_CDA_test.py b/tests/cda/blobs/blob_CDA_test.py index 7535921c..62f8fa6f 100644 --- a/tests/cda/blobs/blob_CDA_test.py +++ b/tests/cda/blobs/blob_CDA_test.py @@ -1,13 +1,16 @@ # tests/test_blob.py from __future__ import annotations +import base64 +import mimetypes from datetime import datetime, timezone +from pathlib import Path from typing import Optional import pandas as pd import pytest -import cwms +import cwms.catalog.blobs as blobs TEST_OFFICE = "MVP" TEST_BLOB_ID = "PYTEST_BLOB_ALPHA" @@ -25,12 +28,12 @@ def ensure_clean_slate(): """Delete the test blob (if it exists) before/after running this module.""" try: - cwms.delete_blob(office_id=TEST_OFFICE, blob_id=TEST_BLOB_ID) + blobs.delete_blob(office_id=TEST_OFFICE, blob_id=TEST_BLOB_ID) except Exception: pass yield try: - cwms.delete_blob(office_id=TEST_OFFICE, blob_id=TEST_BLOB_ID) + blobs.delete_blob(office_id=TEST_OFFICE, blob_id=TEST_BLOB_ID) except Exception: pass @@ -44,7 +47,7 @@ def _find_blob_row(office: str, blob_id: str) -> Optional[pd.Series]: """ Helper: return the row for blob_id from cwms.get_blobs(...).df if present. """ - res = cwms.get_blobs(office_id=office, blob_id_like=blob_id) + res = blobs.get_blobs(office_id=office, blob_id_like=blob_id) df = res if isinstance(res, pd.DataFrame) else getattr(res, "df", None) if df is None or df.empty: return None @@ -55,6 +58,28 @@ def _find_blob_row(office: str, blob_id: str) -> Optional[pd.Series]: return match.iloc[0] if not match.empty else None +def test_store_blob_excel(): + excel_file_path = Path(__file__).parent.parent / "resources" / "blob_test.xlsx" + with open(excel_file_path, "rb") as f: + file_data = f.read() + mime_type, _ = mimetypes.guess_type(excel_file_path) + excel_blob_id = "TEST_BLOB_EXCEL" + payload = { + "office-id": TEST_OFFICE, + "id": excel_blob_id, + "description": "testing excel file", + "media-type-id": mime_type, + "value": base64.b64encode(file_data).decode("utf-8"), + } + blobs.store_blobs(data=payload) + try: + row = _find_blob_row(TEST_OFFICE, excel_blob_id) + assert row is not None, "Stored blob not found in listing" + finally: + # Cleanup excel + blobs.delete_blob(blob_id=excel_blob_id, office_id=TEST_OFFICE) + + def test_store_blob(): # Build request JSON for store_blobs payload = { @@ -64,7 +89,7 @@ def test_store_blob(): "media-type-id": TEST_MEDIA_TYPE, "value": TEST_TEXT, } - cwms.store_blobs(payload, fail_if_exists=True) + blobs.store_blobs(payload, fail_if_exists=True) # Verify via listing metadata row = _find_blob_row(TEST_OFFICE, TEST_BLOB_ID) @@ -76,14 +101,14 @@ def test_store_blob(): assert TEST_DESC in str(row["description"]) # Verify content by downloading - content = cwms.get_blob(office_id=TEST_OFFICE, blob_id=TEST_BLOB_ID) + content = blobs.get_blob(office_id=TEST_OFFICE, blob_id=TEST_BLOB_ID) assert isinstance(content, str) and content, "Empty blob content" assert TEST_TEXT in content def test_get_blob(): # Do a simple read of the blob created in test_store_blob - content = cwms.get_blob(office_id=TEST_OFFICE, blob_id=TEST_BLOB_ID) + content = blobs.get_blob(office_id=TEST_OFFICE, blob_id=TEST_BLOB_ID) assert TEST_TEXT in content assert len(content) >= len(TEST_TEXT) @@ -97,7 +122,7 @@ def test_update_blob(): "media-type-id": TEST_MEDIA_TYPE, "value": TEST_TEXT_UPDATED, } - cwms.update_blob(update, fail_if_not_exists=True) + blobs.update_blob(update, fail_if_not_exists=True) # Confirm updated metadata row = _find_blob_row(TEST_OFFICE, TEST_BLOB_UPDATED_ID) @@ -106,5 +131,5 @@ def test_update_blob(): assert TEST_DESC_UPDATED in str(row["description"]) # Verify new content - content = cwms.get_blob(office_id=TEST_OFFICE, blob_id=TEST_BLOB_UPDATED_ID) + content = blobs.get_blob(office_id=TEST_OFFICE, blob_id=TEST_BLOB_UPDATED_ID) assert TEST_TEXT_UPDATED in content diff --git a/tests/cda/resources/blob_test.xlsx b/tests/cda/resources/blob_test.xlsx new file mode 100644 index 00000000..53c97965 Binary files /dev/null and b/tests/cda/resources/blob_test.xlsx differ