From ffe89b8600f88411fe5b1f27fea0f8d3a735fb1f Mon Sep 17 00:00:00 2001 From: Toby Petty Date: Fri, 7 Jan 2022 16:31:02 +0000 Subject: [PATCH 1/9] Initial commit of dockerfile --- .dockerignore | 3 +++ Dockerfile | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 .dockerignore create mode 100644 Dockerfile diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..7a39a8fb9 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,3 @@ +# Virtual environments +.direnv +venv/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..ae1a6e318 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,39 @@ +FROM python:3.8 as digital-land-python + +RUN set -xe && \ + apt-get update && \ + apt-get install -y \ + make \ + git \ + gdal-bin \ + libspatialite-dev \ + libsqlite3-mod-spatialite + +WORKDIR /src +COPY . /src + +# Necessary to make sure the `digital-land` easyinstall entry script is found when invoking +# `digital-land` via shell from python space +ENV VIRTUAL_ENV=/opt/venv +RUN pip install --upgrade pip +RUN pip install virtualenv +RUN cd /opt && virtualenv venv --always-copy +ENV PATH="$VIRTUAL_ENV/bin:$PATH" + +RUN make init + +FROM digital-land-python as digital-land-specification + +RUN mkdir -p /collection +WORKDIR /collection +RUN set -ex; \ + wget https://github.com/digital-land/makerules/archive/refs/heads/run-pipeline-commands-locally.zip -O makerules.zip; \ + unzip makerules.zip; \ + make specification -f makerules-run-pipeline-commands-locally/makerules.mk +# TODO switch branch when merged +# wget https://github.com/digital-land/makerules/archive/refs/heads/main.zip -O makerules.zip; \ +# make specification -f makerules-main/makerules.mk + +FROM digital-land-specification as digital-land-pipeline + +WORKDIR /pipeline From 9e4856642ae259f6ae7c3ed0e4f81adb4300cfb9 Mon Sep 17 00:00:00 2001 From: Toby Petty Date: Mon, 10 Jan 2022 09:17:51 +0000 Subject: [PATCH 2/9] Added ECR repo and command to build and push docker image on merge into main --- .github/workflows/docker-push.yml | 27 +++++++++++++++++++++++++++ .gitignore | 4 ++++ README.md | 6 ++++++ makerules/python.mk | 15 +++++++++++++++ 4 files changed, 52 insertions(+) create mode 100644 .github/workflows/docker-push.yml diff --git a/.github/workflows/docker-push.yml b/.github/workflows/docker-push.yml new file mode 100644 index 000000000..23124102b --- /dev/null +++ b/.github/workflows/docker-push.yml @@ -0,0 +1,27 @@ +name: Build and push digital-land-python to public ECR +on: + push: + branches: + - main + +jobs: + build: + + runs-on: ubuntu-20.04 + + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + python-version: 3.8 + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-1 + - name: Build & Push + run: | + make docker-build + make docker-push + diff --git a/.gitignore b/.gitignore index 90e7b15fc..576d3ec57 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,7 @@ collection specification ssl.pem var + +.envrc +.direnv +pyrightconfig.json diff --git a/README.md b/README.md index c12661b87..43a84a2a7 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,12 @@ Development requires Python 3.6.2 or later, we recommend using a [virtual enviro make python -m digital-land --help +There is also a docker image available on public ECR + +``` + docker run -it public.ecr.aws/l6z6v3j6/digital-land-python:latest digital-land +``` + ## Release procedure Update the tagged version number: diff --git a/makerules/python.mk b/makerules/python.mk index 942c91877..2835c639c 100644 --- a/makerules/python.mk +++ b/makerules/python.mk @@ -1,4 +1,5 @@ PIP_INSTALL_PACKAGE=[test] +ECR_REPO=public.ecr.aws/l6z6v3j6/ all:: lint test coverage @@ -41,3 +42,17 @@ upload:: dist makerules:: curl -qfsL '$(SOURCE_URL)/makerules/main/python.mk' > makerules/python.mk + +docker-build: docker-check + docker build . -t $(ECR_REPO)digital-land-python + +docker-push: docker-check login-docker + docker push $(ECR_REPO)digital-land-python + +docker-check: +ifeq (, $(shell which docker)) + $(error "No docker in $(PATH), consider doing apt-get install docker OR brew install --cask docker") +endif + +login-docker: + aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws From 5318d0f4ae0e9b716dc07e668f1bb5a9e3cab531 Mon Sep 17 00:00:00 2001 From: Toby Petty Date: Wed, 12 Jan 2022 10:10:20 +0000 Subject: [PATCH 3/9] Added entrypoint wrapper to allow installation of collection repository python dependencies as root at container start time --- Dockerfile | 7 +++++++ docker_entrypoint.sh | 22 ++++++++++++++++++++++ 2 files changed, 29 insertions(+) create mode 100755 docker_entrypoint.sh diff --git a/Dockerfile b/Dockerfile index ae1a6e318..fe8c6bf77 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,6 +3,7 @@ FROM python:3.8 as digital-land-python RUN set -xe && \ apt-get update && \ apt-get install -y \ + gosu \ make \ git \ gdal-bin \ @@ -36,4 +37,10 @@ RUN set -ex; \ FROM digital-land-specification as digital-land-pipeline +# Approach borrowed from https://denibertovic.com/posts/handling-permissions-with-docker-volumes/ +COPY docker_entrypoint.sh /usr/local/bin/docker_entrypoint.sh +RUN chmod +x /usr/local/bin/docker_entrypoint.sh + +ENTRYPOINT ["/usr/local/bin/docker_entrypoint.sh"] + WORKDIR /pipeline diff --git a/docker_entrypoint.sh b/docker_entrypoint.sh new file mode 100755 index 000000000..3c5741160 --- /dev/null +++ b/docker_entrypoint.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# Add local user +# Either use the LOCAL_USER_ID if passed in at runtime or +# fallback + +USER_ID=${LOCAL_USER_ID:-9001} + +# Install packages from the collection repository +[[ -f /pipeline/requirements.txt ]] && /opt/venv/bin/pip install --upgrade -r requirements.txt +[[ -f /pipeline/setup.py ]] && /opt/venv/bin/pip install -e ".${PIP_INSTALL_PACKAGE:-test}" +/opt/venv/bin/pip install csvkit +# make init -f /collection/makerules-main/makerules.mk +# # TODO switch branch when merged +# # make init -f /collection/makerules-run-pipeline-commands-locally/makerules.mk + +echo "Starting with UID : $USER_ID" +useradd --shell /bin/bash -u $USER_ID -o -c "" -m user +export HOME=/home/user + +exec gosu user "$@" + From faf78ca6934d4a5151aae99a6249038b603054b4 Mon Sep 17 00:00:00 2001 From: Toby Petty Date: Wed, 12 Jan 2022 11:49:27 +0000 Subject: [PATCH 4/9] Ensure that any directories created by docker as bind mounts are chowned to LOCAL_USER_ID Also, switch to a more sensible fallback value of UID running docker container on missing LOCAL_USER_ID --- docker_entrypoint.sh | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/docker_entrypoint.sh b/docker_entrypoint.sh index 3c5741160..eb9eacadb 100755 --- a/docker_entrypoint.sh +++ b/docker_entrypoint.sh @@ -2,9 +2,16 @@ # Add local user # Either use the LOCAL_USER_ID if passed in at runtime or -# fallback +# fallback to $UID -USER_ID=${LOCAL_USER_ID:-9001} +USER_ID=${LOCAL_USER_ID:-$UID} + +echo "Starting with UID : $USER_ID" +useradd --shell /bin/bash -u $USER_ID -o -c "" -m user +export HOME=/home/user + +# Docker will create any new local folders on host specified in bind mounts as being owned by root. Fix this +chown -R $USER_ID /pipeline # Install packages from the collection repository [[ -f /pipeline/requirements.txt ]] && /opt/venv/bin/pip install --upgrade -r requirements.txt @@ -14,9 +21,5 @@ USER_ID=${LOCAL_USER_ID:-9001} # # TODO switch branch when merged # # make init -f /collection/makerules-run-pipeline-commands-locally/makerules.mk -echo "Starting with UID : $USER_ID" -useradd --shell /bin/bash -u $USER_ID -o -c "" -m user -export HOME=/home/user - exec gosu user "$@" From 7c130adafc9ee741788ac727395043df50b0ccb9 Mon Sep 17 00:00:00 2001 From: Toby Petty Date: Wed, 12 Jan 2022 16:42:21 +0000 Subject: [PATCH 5/9] Add awscli to apt install --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index fe8c6bf77..13bb03eee 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,6 +3,7 @@ FROM python:3.8 as digital-land-python RUN set -xe && \ apt-get update && \ apt-get install -y \ + awscli \ gosu \ make \ git \ From d90b144088356d237c8293ef915863079460b7c1 Mon Sep 17 00:00:00 2001 From: Toby Petty Date: Thu, 13 Jan 2022 12:53:06 +0000 Subject: [PATCH 6/9] Switch makerules branch, remove docker_entrypoint, tag images as digital-land-python --- Dockerfile | 10 ++-------- docker_entrypoint.sh | 25 ------------------------- makerules/python.mk | 8 ++++---- 3 files changed, 6 insertions(+), 37 deletions(-) delete mode 100755 docker_entrypoint.sh diff --git a/Dockerfile b/Dockerfile index 13bb03eee..af625bb90 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,19 +29,13 @@ FROM digital-land-python as digital-land-specification RUN mkdir -p /collection WORKDIR /collection RUN set -ex; \ - wget https://github.com/digital-land/makerules/archive/refs/heads/run-pipeline-commands-locally.zip -O makerules.zip; \ + wget https://github.com/digital-land/makerules/archive/refs/heads/run-all-pipeline-commands-locally-take-2.zip -O makerules.zip; \ unzip makerules.zip; \ - make specification -f makerules-run-pipeline-commands-locally/makerules.mk + make specification -f makerules-run-all-pipeline-commands-locally-take-2/makerules.mk # TODO switch branch when merged # wget https://github.com/digital-land/makerules/archive/refs/heads/main.zip -O makerules.zip; \ # make specification -f makerules-main/makerules.mk FROM digital-land-specification as digital-land-pipeline -# Approach borrowed from https://denibertovic.com/posts/handling-permissions-with-docker-volumes/ -COPY docker_entrypoint.sh /usr/local/bin/docker_entrypoint.sh -RUN chmod +x /usr/local/bin/docker_entrypoint.sh - -ENTRYPOINT ["/usr/local/bin/docker_entrypoint.sh"] - WORKDIR /pipeline diff --git a/docker_entrypoint.sh b/docker_entrypoint.sh deleted file mode 100755 index eb9eacadb..000000000 --- a/docker_entrypoint.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -# Add local user -# Either use the LOCAL_USER_ID if passed in at runtime or -# fallback to $UID - -USER_ID=${LOCAL_USER_ID:-$UID} - -echo "Starting with UID : $USER_ID" -useradd --shell /bin/bash -u $USER_ID -o -c "" -m user -export HOME=/home/user - -# Docker will create any new local folders on host specified in bind mounts as being owned by root. Fix this -chown -R $USER_ID /pipeline - -# Install packages from the collection repository -[[ -f /pipeline/requirements.txt ]] && /opt/venv/bin/pip install --upgrade -r requirements.txt -[[ -f /pipeline/setup.py ]] && /opt/venv/bin/pip install -e ".${PIP_INSTALL_PACKAGE:-test}" -/opt/venv/bin/pip install csvkit -# make init -f /collection/makerules-main/makerules.mk -# # TODO switch branch when merged -# # make init -f /collection/makerules-run-pipeline-commands-locally/makerules.mk - -exec gosu user "$@" - diff --git a/makerules/python.mk b/makerules/python.mk index 2835c639c..d2f9455cc 100644 --- a/makerules/python.mk +++ b/makerules/python.mk @@ -44,15 +44,15 @@ makerules:: curl -qfsL '$(SOURCE_URL)/makerules/main/python.mk' > makerules/python.mk docker-build: docker-check - docker build . -t $(ECR_REPO)digital-land-python + docker build . -t $(ECR_REPO)digital-land-python:digital-land-python -docker-push: docker-check login-docker - docker push $(ECR_REPO)digital-land-python +docker-push: docker-check docker-ecr-login + docker push $(ECR_REPO)digital-land-python:digital-land-python docker-check: ifeq (, $(shell which docker)) $(error "No docker in $(PATH), consider doing apt-get install docker OR brew install --cask docker") endif -login-docker: +docker-ecr-login: aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws From 8d33a72d5a6eba9457bd2658e23737b065d20af5 Mon Sep 17 00:00:00 2001 From: Toby Petty Date: Fri, 14 Jan 2022 12:32:18 +0000 Subject: [PATCH 7/9] Add additional pipeline apt dependency --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index af625bb90..bf49c3e7d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,6 +4,7 @@ RUN set -xe && \ apt-get update && \ apt-get install -y \ awscli \ + time \ gosu \ make \ git \ From d366009f007557610d3a89229aa9c03b3f17ba41 Mon Sep 17 00:00:00 2001 From: Toby Petty Date: Fri, 14 Jan 2022 17:18:49 +0000 Subject: [PATCH 8/9] Add build target, refactor into env var to help document similar naming --- makerules/python.mk | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/makerules/python.mk b/makerules/python.mk index d2f9455cc..b05a298b4 100644 --- a/makerules/python.mk +++ b/makerules/python.mk @@ -1,6 +1,10 @@ PIP_INSTALL_PACKAGE=[test] ECR_REPO=public.ecr.aws/l6z6v3j6/ +LAYER_NAME=digital-land-python +REPO_NAME=digital-land-python +ECR_PATH=$(ECR_REPO)$(REPO_NAME):$(LAYER_NAME) + all:: lint test coverage lint:: black-check flake8 @@ -44,10 +48,16 @@ makerules:: curl -qfsL '$(SOURCE_URL)/makerules/main/python.mk' > makerules/python.mk docker-build: docker-check - docker build . -t $(ECR_REPO)digital-land-python:digital-land-python + docker build . -t $(ECR_PATH) --target $(LAYER_NAME) docker-push: docker-check docker-ecr-login - docker push $(ECR_REPO)digital-land-python:digital-land-python + docker push $(ECR_PATH) + +docker-pull: docker-check docker-ecr-login + docker pull $(ECR_PATH) + +docker-shell: docker-check docker-ecr-login + docker run -it $(ECR_PATH) bash docker-check: ifeq (, $(shell which docker)) From fcb20b6311edea427e84f8f20ad36c962a7dbbc8 Mon Sep 17 00:00:00 2001 From: Toby Petty Date: Mon, 18 Jul 2022 16:03:39 +0100 Subject: [PATCH 9/9] Enable building docker image from feature branch --- .github/workflows/docker-push.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker-push.yml b/.github/workflows/docker-push.yml index 23124102b..4cfaba053 100644 --- a/.github/workflows/docker-push.yml +++ b/.github/workflows/docker-push.yml @@ -2,7 +2,8 @@ name: Build and push digital-land-python to public ECR on: push: branches: - - main + # - main # TODO reinstate this before merge + - run-pipeline-commands-locally jobs: build: