From 5b99254d308be04fdd863753de958112d9c46ef2 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Mon, 27 Sep 2021 00:37:14 -0400
Subject: [PATCH] ref: Add Dockerfile and Github Actions pipeline (#2)

* ref: Add Dockerfile and Github Actions pipeline

* fix: do not use versioning

* fix: use the right stage at build time

* imp: use python3 official image

* fix: add lxml dependency

* fix: add g++ for building

* fix: add missing libffi

* fix: do not add extra users

* fix: use the same user as python

* ref: use best methods from @kowh-ai and @mbocevski

* fix: dockerfile

* fix: do not add www-data group as already exists

* fix: use alpine as base image instead of python

* ref: use three building methods

* fix: use the right build test segmentation

* fix: more improvements

* fix: Do not use healtcheck on dev

* fix: use the ckan variables
---
 .github/workflows/development.yml |  69 ++++++++++++++++++++
 .github/workflows/master.yml      | 100 ++++++++++++++++++++++++++++
 .github/workflows/release.yml     |  81 +++++++++++++++++++++++
 Dockerfile                        | 104 ++++++++++++++++++++++++++++++
 4 files changed, 354 insertions(+)
 create mode 100644 .github/workflows/development.yml
 create mode 100644 .github/workflows/master.yml
 create mode 100644 .github/workflows/release.yml
 create mode 100644 Dockerfile

diff --git a/.github/workflows/development.yml b/.github/workflows/development.yml
new file mode 100644
index 0000000..b2ec1a8
--- /dev/null
+++ b/.github/workflows/development.yml
@@ -0,0 +1,69 @@
+name: Development build
+
+on:
+  pull_request:
+    branches:
+      - 'master'
+    paths:
+      - "**/workflows/**"
+      - "**.py"
+      - "**.ini"
+      - "**.txt"
+      - "Dockerfile"
+
+env:
+  # Registry variables
+  HUB_BASE: ckan
+  GIT_BASE: ghcr.io/ckan
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Inject slug/short variables
+        uses: rlespinasse/github-slug-action@v3.x
+
+      - name: Download Repository
+        uses: actions/checkout@v2
+
+      # Build the Docker image and push it
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Cache Docker layers
+        uses: actions/cache@v2
+        with:
+          path: |
+            /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push
+        id: docker_build
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          target: main
+          tags: |
+            ${{ env.GITHUB_REPOSITORY_SLUG }}:${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}
+          load: true
+          cache-from: type=registry,ref=${{ env.GIT_BASE }}/${{ env.GITHUB_REPOSITORY_NAME_PART_SLUG }}:${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}
+          cache-to: type=inline
+
+      - name: Run ${{ env.GITHUB_REPOSITORY_NAME_PART_SLUG }} tests
+        run: |
+          docker run -d -p 127.0.0.1:8800:8800 --name datapusher ${{ env.GITHUB_REPOSITORY_SLUG }}:${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}
+          docker ps | grep -q datapusher
diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
new file mode 100644
index 0000000..07fb391
--- /dev/null
+++ b/.github/workflows/master.yml
@@ -0,0 +1,100 @@
+name: Master release
+
+on:
+  push:
+    branches:
+      - 'master'
+    paths:
+      - "**/workflows/**"
+      - "**.py"
+      - "**.ini"
+      - "**.txt"
+      - "Dockerfile"
+
+env:
+  # Registry variables
+  HUB_BASE: ckan
+  GIT_BASE: ghcr.io/ckan
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Inject slug/short variables
+        uses: rlespinasse/github-slug-action@v3.x
+
+      - name: Download Repository
+        uses: actions/checkout@v2
+
+      # Build the Docker image and push it
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Cache Docker layers
+        uses: actions/cache@v2
+        with:
+          path: |
+            /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push
+        id: docker_build
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          target: main
+          tags: |
+            ${{ env.GIT_BASE }}/${{ env.GITHUB_REPOSITORY_NAME_PART_SLUG }}:${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}
+            ${{ env.GIT_BASE }}/${{ env.GITHUB_REPOSITORY_NAME_PART_SLUG }}:dev
+            ${{ env.HUB_BASE }}/${{ env.GITHUB_REPOSITORY_NAME_PART_SLUG }}:${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}
+            ${{ env.HUB_BASE }}/${{ env.GITHUB_REPOSITORY_NAME_PART_SLUG }}:dev
+          push: true
+          cache-from: type=registry,ref=${{ env.GIT_BASE }}/${{ env.GITHUB_REPOSITORY_NAME_PART_SLUG }}:${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}
+          cache-to: type=inline
+
+      - name: Image digest
+        run: echo ${{ steps.docker_build.outputs.digest }}
+
+  test:
+    needs: ['build']
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Inject slug/short variables
+        uses: rlespinasse/github-slug-action@v3.x
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+        with:
+          driver-opts: network=host
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Run ${{ env.GITHUB_REPOSITORY_NAME_PART_SLUG }} tests
+        run: |
+          docker pull ${{ env.GIT_BASE }}/${{ env.GITHUB_REPOSITORY_NAME_PART_SLUG }}:${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}
+          docker run -d -p 127.0.0.1:8800:8800 --name datapusher ${{ env.GIT_BASE }}/${{ env.GITHUB_REPOSITORY_NAME_PART_SLUG }}:${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}
+          docker ps | grep -q datapusher
+
+      - name: Healthcheck ${{ env.GITHUB_REPOSITORY_NAME_PART_SLUG }}
+        run: |
+          curl --fail http://localhost:8800/job
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
new file mode 100644
index 0000000..09bbf53
--- /dev/null
+++ b/.github/workflows/release.yml
@@ -0,0 +1,81 @@
+name: Stable release
+
+on:
+  release:
+    types:
+      - published
+env:
+  # Registry variables
+  HUB_BASE: ckan
+  GIT_BASE: ghcr.io/ckan
+
+jobs:
+  versioning:
+    runs-on: ubuntu-latest
+
+    outputs:
+      version: ${{ steps.set.outputs.version }}
+    steps:
+      - name: Getting API Version
+        id: get
+        uses: actions/github-script@v4
+        with:
+          result-encoding: string
+          script: |
+            return context.payload.release.tag_name.substring(0,2)
+      - name: Setting API Version
+        id: set
+        run: echo "::set-output name=version::${{ steps.get.outputs.result }}"
+
+  build:
+    needs: ['versioning']
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Inject slug/short variables
+        uses: rlespinasse/github-slug-action@v3.x
+
+      - name: Download Repository
+        uses: actions/checkout@v2
+
+      # Build the Docker image and push it
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Cache Docker layers
+        uses: actions/cache@v2
+        with:
+          path: |
+            /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push
+        id: docker_build
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          target: main
+          tags: |
+            ${{ env.GIT_BASE }}/${{ env.GITHUB_REPOSITORY_NAME_PART_SLUG }}:${{ needs.versioning.outputs.version }}
+            ${{ env.GIT_BASE }}/${{ env.GITHUB_REPOSITORY_NAME_PART_SLUG }}:latest
+            ${{ env.HUB_BASE }}/${{ env.GITHUB_REPOSITORY_NAME_PART_SLUG }}:${{ needs.versioning.outputs.version }}
+            ${{ env.HUB_BASE }}/${{ env.GITHUB_REPOSITORY_NAME_PART_SLUG }}:latest
+          push: true
+          cache-from: type=registry,ref=${{ env.GIT_BASE }}/${{ env.GITHUB_REPOSITORY_NAME_PART_SLUG }}:latest
+          cache-to: type=inline
+
+      - name: Image digest
+        run: echo ${{ steps.docker_build.outputs.digest }}
diff --git a/Dockerfile b/Dockerfile
new file mode 100644
index 0000000..db8f9d7
--- /dev/null
+++ b/Dockerfile
@@ -0,0 +1,104 @@
+#############
+### Build ###
+#############
+FROM alpine:3.13 as build
+
+# Set src dirs
+ENV APP_DIR=/srv/app
+ENV PIP_SRC=${APP_DIR}
+ENV JOB_CONFIG ${APP_DIR}/datapusher_settings.py
+
+WORKDIR ${APP_DIR}
+
+# Packages to build datapusher
+RUN apk add --no-cache \
+    python3 \
+    # Temporary packages to build DataPusher requirements
+    && apk add --no-cache --virtual .build-deps \
+    py3-pip \
+    py3-wheel \
+    libffi-dev \
+    libressl-dev \
+    gcc \
+    git \
+    musl-dev \
+    python3-dev \
+    libxml2-dev \
+    libxslt-dev \
+    libmagic \
+    openssl-dev \
+    cargo
+
+# Create the src directory
+RUN mkdir -p ${APP_DIR}/src
+WORKDIR ${APP_DIR}/src
+
+# Copy datapusher to APP_DIR
+COPY ./README.md .
+COPY ./setup.py .
+COPY ./requirements.txt .
+COPY ./datapusher ./datapusher
+
+# Fetch and build datapusher and requirements
+RUN pip3 wheel --wheel-dir=/wheels .
+RUN pip3 wheel --wheel-dir=/wheels -r requirements.txt
+
+# Copy requirements.txt to /wheels
+RUN cp requirements.txt /wheels/requirements.txt
+
+RUN apk del .build-deps && \
+    rm -rf ${APP_DIR}/src
+
+############
+### MAIN ###
+############
+FROM alpine:3.13 as main
+
+ENV APP_DIR=/usr/lib/ckan/datapusher
+ENV WSGI_FILE ${APP_DIR}/src/datapusher/deployment/datapusher.wsgi
+ENV WSGI_CONFIG ${APP_DIR}/datapusher-uwsgi.ini
+
+RUN apk add --no-cache \
+    python3 \
+    py3-pip \
+    curl \
+    pcre \
+    libmagic \
+    libxslt \
+    libxml2 \
+    uwsgi \
+    uwsgi-http \
+    uwsgi-corerouter \
+    uwsgi-python
+
+WORKDIR ${APP_DIR}
+
+COPY ./deployment/datapusher.wsgi .
+COPY ./deployment/datapusher-uwsgi.ini .
+
+# Get artifacts from build stages
+COPY --from=build /wheels /wheels
+
+RUN pip3 install --no-index --find-links=/wheels datapusher && \
+    pip3 install --no-index --find-links=/wheels -r /wheels/requirements.txt && \
+    # Set timezone
+    echo "UTC" >  /etc/timezone && \
+    # Change uwsgi http worker to listen on all interfaces
+    sed 's/127.0.0.1/0.0.0.0/g' -i ${APP_DIR}/datapusher-uwsgi.ini && \
+    # Remove default values in ini file
+    sed -i '/http/d' ${APP_DIR}/datapusher-uwsgi.ini && \
+    sed -i '/wsgi-file/d' ${APP_DIR}/datapusher-uwsgi.ini && \
+    sed -i '/virtualenv/d' ${APP_DIR}/datapusher-uwsgi.ini && \
+    # Remove wheels
+    rm -rf /wheels
+
+# Create a local user and group to run the app
+RUN addgroup -g 92 -S www-data && \
+    adduser -u 92 -h /srv/app -H -D -S -G www-data www-data
+
+EXPOSE 8800
+
+USER www-data
+
+CMD ["sh", "-c", \
+    "uwsgi --plugins=http,python --http=0.0.0.0:8800 --socket=/tmp/uwsgi.sock --ini=`echo ${APP_DIR}`/datapusher-uwsgi.ini --wsgi-file=`echo ${APP_DIR}`/datapusher.wsgi"]