From 7e1579fd63a98ca8999cd99335089ac2ba6f9813 Mon Sep 17 00:00:00 2001 From: Ben Topping Date: Tue, 6 May 2025 13:46:09 +0100 Subject: [PATCH 1/2] feat(mongo): upgrades CI and docker containers to mongo 8.0 --- .github/workflows/automated_release.yml | 4 ++-- .github/workflows/ci.yml | 9 ++------- Dockerfile | 10 +++++++--- dependencies/docker-compose.yml | 11 +++++------ dependencies/scripts/mongo-init-replica-set.sh | 2 +- dependencies/up.sh | 2 +- 6 files changed, 18 insertions(+), 20 deletions(-) diff --git a/.github/workflows/automated_release.yml b/.github/workflows/automated_release.yml index 4834349e..fce88cca 100644 --- a/.github/workflows/automated_release.yml +++ b/.github/workflows/automated_release.yml @@ -41,9 +41,9 @@ jobs: --tag docker.pkg.github.com/${IMAGE_NAME}:${BRANCH_NAME} - name: Start MongoDB - uses: supercharge/mongodb-github-action@1.7.0 + uses: supercharge/mongodb-github-action@1.12.0 with: - mongodb-version: 4.2 + mongodb-version: 8.0 mongodb-replica-set: heron_rs - name: Create SQL Server testing database diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c8582968..ed83c329 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -135,17 +135,12 @@ jobs: run: | pipenv sync --dev --system - name: Start MongoDB - uses: supercharge/mongodb-github-action@1.7.0 + uses: supercharge/mongodb-github-action@1.12.0 with: - mongodb-version: 4.2 + mongodb-version: 8.0 mongodb-replica-set: heron_rs - name: Setup the test MLWH and Events databases run: | - curl https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add - - curl https://packages.microsoft.com/config/ubuntu/$(lsb_release -rs)/prod.list | sudo tee /etc/apt/sources.list.d/mssql-release.list - sudo apt-get update - sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18 - sudo apt-get install -y unixodbc-dev python setup_test_db.py - name: Create SQL Server testing database run: | diff --git a/Dockerfile b/Dockerfile index 58c0e417..07ec810d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,11 +20,15 @@ RUN apt-get update && \ unixodbc-dev # Install the Microsoft ODBC driver for SQL Server -RUN curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - && \ - curl https://packages.microsoft.com/config/debian/10/prod.list > /etc/apt/sources.list.d/mssql-release.list +# https://learn.microsoft.com/en-us/sql/connect/odbc/linux-mac/installing-the-microsoft-odbc-driver-for-sql-server?view=sql-server-ver16&tabs=debian18-install%2Calpine17-install%2Cdebian8-install%2Credhat7-13-install%2Crhel7-offline#18 +# Download the package to configure the Microsoft repo +RUN curl -sSL -O https://packages.microsoft.com/config/debian/$(grep VERSION_ID /etc/os-release | cut -d '"' -f 2 | cut -d '.' -f 1)/packages-microsoft-prod.deb +# Install the package +RUN dpkg -i packages-microsoft-prod.deb +# Delete the file +RUN rm packages-microsoft-prod.deb RUN apt-get update RUN ACCEPT_EULA=Y apt-get install -y msodbcsql18 -RUN apt-get install -y unixodbc-dev # Install the package manager - pipenv RUN pip install --upgrade pip && \ diff --git a/dependencies/docker-compose.yml b/dependencies/docker-compose.yml index 18d358af..609a9a64 100644 --- a/dependencies/docker-compose.yml +++ b/dependencies/docker-compose.yml @@ -1,4 +1,3 @@ -version: "3.8" services: # A MySQL service as close to production as currently possible # To run it as a standalone container: @@ -16,9 +15,9 @@ services: # A MongoDB service as close to production as currently possible # To run it as a standalone container: - # docker run --name mongo -d -p 27017:27017 --entrypoint="/usr/bin/mongod --bind_ip_all --replSet heron_rs" mongo:4.2 - mongo-4.2: - image: mongo:4.2 + # docker run --name mongo -d -p 27017:27017 --entrypoint="/usr/bin/mongod --bind_ip_all --replSet heron_rs" mongo:8.0 + mongo-8.0: + image: mongo:8.0 container_name: mongo ports: - "27017:27017" @@ -62,9 +61,9 @@ services: # docker network create redpanda-network # docker volume create redpanda-volume # docker run --name=schema_registry --net=redpanda-network -v "redpanda-volume:/var/lib/redpanda/data" -p 8081:8081 \ - # docker.vectorized.io/vectorized/redpanda start --overprovisioned --smp 1 --memory 250M + # redpandadata/redpanda start --overprovisioned --smp 1 --memory 250M schema_registry: - image: docker.vectorized.io/vectorized/redpanda + image: redpandadata/redpanda networks: - redpanda-network ports: diff --git a/dependencies/scripts/mongo-init-replica-set.sh b/dependencies/scripts/mongo-init-replica-set.sh index 45a2e454..916490d4 100755 --- a/dependencies/scripts/mongo-init-replica-set.sh +++ b/dependencies/scripts/mongo-init-replica-set.sh @@ -1,6 +1,6 @@ #!/bin/bash -mongo < Date: Tue, 6 May 2025 13:58:32 +0100 Subject: [PATCH 2/2] fix(ci): restores mssql-server connector in db setup, removes deprecated datetime utcnow references --- .github/workflows/ci.yml | 4 ++++ tests/fixtures/data/positive_samples_in_source_plate.py | 2 +- tests/fixtures/data/samples.py | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ed83c329..d8088fe7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -144,6 +144,10 @@ jobs: python setup_test_db.py - name: Create SQL Server testing database run: | + curl https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add - + curl https://packages.microsoft.com/config/ubuntu/$(lsb_release -rs)/prod.list | sudo tee /etc/apt/sources.list.d/mssql-release.list + sudo apt-get update + sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18 python setup_sqlserver_test_db.py - name: Test with pytest run: | diff --git a/tests/fixtures/data/positive_samples_in_source_plate.py b/tests/fixtures/data/positive_samples_in_source_plate.py index 47d6e1ad..df193cc6 100644 --- a/tests/fixtures/data/positive_samples_in_source_plate.py +++ b/tests/fixtures/data/positive_samples_in_source_plate.py @@ -16,7 +16,7 @@ ) from tests.fixtures.data.source_plates import SOURCE_PLATES -DATE_TESTED_NOW = datetime.utcnow() +DATE_TESTED_NOW = datetime.now() POSITIVE_SAMPLES = [ { diff --git a/tests/fixtures/data/samples.py b/tests/fixtures/data/samples.py index 715396e8..4d5a8130 100644 --- a/tests/fixtures/data/samples.py +++ b/tests/fixtures/data/samples.py @@ -17,7 +17,7 @@ ) from tests.fixtures.data.source_plates import SOURCE_PLATES -DATE_TESTED_NOW = datetime.utcnow() +DATE_TESTED_NOW = datetime.now() # NOTE: Remember that the samples of 'plate_123' are joined to the priority samples # There should be 7 fit to pick samples from all the plates below SAMPLES = [