From 8a2dc9819ce3917b64b395ecbbfe957f9a487c38 Mon Sep 17 00:00:00 2001 From: Toby Petty Date: Wed, 12 Jan 2022 16:05:07 +0000 Subject: [PATCH 01/10] Initial commit for new approach * Build image for component with python deps and /pipeline * Run makefile inside component --- Dockerfile | 12 ++++++++ docker.mk | 87 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+) create mode 100644 Dockerfile create mode 100644 docker.mk diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..f9a9c30 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,12 @@ +ARG REPO=public.ecr.aws/l6z6v3j6/ +FROM ${REPO}digital-land-python:latest + +COPY . /pipeline + +# TODO add labels? + +RUN set -xe; \ + [ -f /pipeline/requirements.txt ] && /opt/venv/bin/pip install --upgrade -r requirements.txt; \ + [ -f /pipeline/setup.py ] && /opt/venv/bin/pip install -e ".${PIP_INSTALL_PACKAGE:-test}" + +ENTRYPOINT ["make"] diff --git a/docker.mk b/docker.mk new file mode 100644 index 0000000..55bd985 --- /dev/null +++ b/docker.mk @@ -0,0 +1,87 @@ +ifneq ($(DOCKERISED),1) +DOCKERISED = 0 +DEVELOPMENT = 0 +else +# Run in development mode by default for now +ifeq ($(DEVELOPMENT),0) +DEVELOPMENT = 0 +else +DEVELOPMENT = 1 +endif +endif +$(info DOCKERISED is $(DOCKERISED)) +$(info DEVELOPMENT is $(DEVELOPMENT)) + +EXTRA_MOUNTS := +EXTRA_DL_ARGS := +ifeq ($(DEVELOPMENT),1) +EXTRA_MOUNTS += -v $(PWD)/local_collection/collection/log:/pipeline/collection/log +EXTRA_MOUNTS += -v $(PWD)/local_collection/collection/resource:/pipeline/collection/resource +ifneq (,$(wildcard ./fixed)) +EXTRA_MOUNTS += -v $(PWD)/local_collection/fixed:/pipeline/fixed +endif +ifneq (,$(wildcard ./harmonised)) +EXTRA_MOUNTS += -v $(PWD)/local_collection/harmonised:/pipeline/harmonised +endif +ifneq (,$(wildcard ./harmonised)) +EXTRA_MOUNTS += -v $(PWD)/local_collection/transformed:/pipeline/transformed +endif + +ifdef ($(LOCAL_SPECIFICATION_PATH),) +EXTRA_MOUNTS += -v $(LOCAL_SPECIFICATION_PATH)/specification:/collection/specification +else ifeq ($(LOCAL_SPECIFICATION),1) +EXTRA_MOUNTS += -v $(PWD)/../specification/specification:/collection/specification +endif + +ifdef ($(LOCAL_DL_PYTHON_PATH),) +EXTRA_MOUNTS += -v $(LOCAL_DL_PYTHON_PATH):/Src +else ifeq ($(LOCAL_DL_PYTHON),1) +EXTRA_MOUNTS += -v $(PWD)/../digital-land-python:/src +endif + +endif +$(info EXTRA_MOUNTS is $(EXTRA_MOUNTS)) + +# DOCKER_TAG=latest +ECR_URL=public.ecr.aws/l6z6v3j6/ +DOCKER_TAG=$(shell basename $(PWD)) +DOCKER_PATH=$(ECR_URL)digital-land-python:$(DOCKER_TAG) + +dockerised = docker run -t \ + -e LOCAL_USER_ID=$(shell id -u) \ + -e AWS_ACCESS_KEY_ID \ + -e AWS_DEFAULT_REGION \ + -e AWS_REGION \ + -e AWS_SECRET_ACCESS_KEY \ + -e AWS_SECURITY_TOKEN \ + -e AWS_SESSION_EXPIRATION \ + -e AWS_SESSION_TOKEN \ + -v $(PWD):/pipeline \ + $(EXTRA_MOUNTS) \ + $(DOCKER_PATH) + +shell_cmd = $(dockerised) bash + +digital-land-cli = $(dockerised) \ + digital-land \ + $(EXTRA_DL_ARGS) + +digital-land = $(dockerised) make + +ifeq ($(DOCKERISED),1) +init:: docker-pull +endif + +docker-build:: + docker build . -f makerules/Dockerfile -t $(DOCKER_PATH) + +docker-pull:: +ifndef ($(DISABLE_DOCKER_PULL),) + docker pull $(ECR_URL)digital-land-python:$(DOCKER_TAG) +endif + +debug_shell: + $(shell_cmd) + +digital-land-cli: + $(digital-land-cli) From 2dc4667029eb3de5e0569c40c12ddc2b5b94bf4c Mon Sep 17 00:00:00 2001 From: Toby Petty Date: Wed, 12 Jan 2022 16:29:05 +0000 Subject: [PATCH 02/10] Fixed calling through to the makefile in the docker container --- docker.mk | 41 ++++++++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/docker.mk b/docker.mk index 55bd985..097f0ff 100644 --- a/docker.mk +++ b/docker.mk @@ -12,42 +12,46 @@ endif $(info DOCKERISED is $(DOCKERISED)) $(info DEVELOPMENT is $(DEVELOPMENT)) -EXTRA_MOUNTS := +EXTRA_DOCKER_ARGS := EXTRA_DL_ARGS := ifeq ($(DEVELOPMENT),1) -EXTRA_MOUNTS += -v $(PWD)/local_collection/collection/log:/pipeline/collection/log -EXTRA_MOUNTS += -v $(PWD)/local_collection/collection/resource:/pipeline/collection/resource +EXTRA_DOCKER_ARGS += -v $(PWD)/local_collection/collection/log:/pipeline/collection/log +EXTRA_DOCKER_ARGS += -v $(PWD)/local_collection/collection/resource:/pipeline/collection/resource ifneq (,$(wildcard ./fixed)) -EXTRA_MOUNTS += -v $(PWD)/local_collection/fixed:/pipeline/fixed +EXTRA_DOCKER_ARGS += -v $(PWD)/local_collection/fixed:/pipeline/fixed endif ifneq (,$(wildcard ./harmonised)) -EXTRA_MOUNTS += -v $(PWD)/local_collection/harmonised:/pipeline/harmonised +EXTRA_DOCKER_ARGS += -v $(PWD)/local_collection/harmonised:/pipeline/harmonised endif ifneq (,$(wildcard ./harmonised)) -EXTRA_MOUNTS += -v $(PWD)/local_collection/transformed:/pipeline/transformed +EXTRA_DOCKER_ARGS += -v $(PWD)/local_collection/transformed:/pipeline/transformed endif ifdef ($(LOCAL_SPECIFICATION_PATH),) -EXTRA_MOUNTS += -v $(LOCAL_SPECIFICATION_PATH)/specification:/collection/specification +EXTRA_DOCKER_ARGS += -v $(LOCAL_SPECIFICATION_PATH)/specification:/collection/specification else ifeq ($(LOCAL_SPECIFICATION),1) -EXTRA_MOUNTS += -v $(PWD)/../specification/specification:/collection/specification +EXTRA_DOCKER_ARGS += -v $(PWD)/../specification/specification:/collection/specification endif ifdef ($(LOCAL_DL_PYTHON_PATH),) -EXTRA_MOUNTS += -v $(LOCAL_DL_PYTHON_PATH):/Src +EXTRA_DOCKER_ARGS += -v $(LOCAL_DL_PYTHON_PATH):/Src else ifeq ($(LOCAL_DL_PYTHON),1) -EXTRA_MOUNTS += -v $(PWD)/../digital-land-python:/src +EXTRA_DOCKER_ARGS += -v $(PWD)/../digital-land-python:/src endif +ifeq ($(word 2, $(MAKECMDGOALS)),shell_cmd) +EXTRA_DOCKER_ARGS += --entrypoint bash endif -$(info EXTRA_MOUNTS is $(EXTRA_MOUNTS)) + +endif +$(info EXTRA_DOCKER_ARGS is $(EXTRA_DOCKER_ARGS)) # DOCKER_TAG=latest ECR_URL=public.ecr.aws/l6z6v3j6/ DOCKER_TAG=$(shell basename $(PWD)) DOCKER_PATH=$(ECR_URL)digital-land-python:$(DOCKER_TAG) -dockerised = docker run -t \ +docker-prefix = docker run -t \ -e LOCAL_USER_ID=$(shell id -u) \ -e AWS_ACCESS_KEY_ID \ -e AWS_DEFAULT_REGION \ @@ -57,16 +61,23 @@ dockerised = docker run -t \ -e AWS_SESSION_EXPIRATION \ -e AWS_SESSION_TOKEN \ -v $(PWD):/pipeline \ - $(EXTRA_MOUNTS) \ + $(EXTRA_DOCKER_ARGS) + +dockerised = $(docker-prefix) \ $(DOCKER_PATH) -shell_cmd = $(dockerised) bash +shell_cmd:: + $(docker-prefix) \ + --entrypoint bash \ + $(DOCKER_PATH) digital-land-cli = $(dockerised) \ digital-land \ $(EXTRA_DL_ARGS) -digital-land = $(dockerised) make +dockerised:: + $(info MAKECMDGOALS is $(MAKECMDGOALS)) + $(dockerised) $(word 2, $(MAKECMDGOALS)) ifeq ($(DOCKERISED),1) init:: docker-pull From be99a51ae4ee30b9935b598055b096c4fc2a9363 Mon Sep 17 00:00:00 2001 From: Toby Petty Date: Wed, 12 Jan 2022 17:11:20 +0000 Subject: [PATCH 03/10] move back to running container as host user to simplify file permissioning and fix digital-land-cli entrypoint --- docker.mk | 28 ++++++++++------------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/docker.mk b/docker.mk index 097f0ff..038f2d4 100644 --- a/docker.mk +++ b/docker.mk @@ -39,9 +39,6 @@ else ifeq ($(LOCAL_DL_PYTHON),1) EXTRA_DOCKER_ARGS += -v $(PWD)/../digital-land-python:/src endif -ifeq ($(word 2, $(MAKECMDGOALS)),shell_cmd) -EXTRA_DOCKER_ARGS += --entrypoint bash -endif endif $(info EXTRA_DOCKER_ARGS is $(EXTRA_DOCKER_ARGS)) @@ -52,7 +49,7 @@ DOCKER_TAG=$(shell basename $(PWD)) DOCKER_PATH=$(ECR_URL)digital-land-python:$(DOCKER_TAG) docker-prefix = docker run -t \ - -e LOCAL_USER_ID=$(shell id -u) \ + -u $(shell id -u) \ -e AWS_ACCESS_KEY_ID \ -e AWS_DEFAULT_REGION \ -e AWS_REGION \ @@ -71,28 +68,23 @@ shell_cmd:: --entrypoint bash \ $(DOCKER_PATH) -digital-land-cli = $(dockerised) \ - digital-land \ - $(EXTRA_DL_ARGS) - -dockerised:: +dockerised:: docker-pull $(info MAKECMDGOALS is $(MAKECMDGOALS)) $(dockerised) $(word 2, $(MAKECMDGOALS)) -ifeq ($(DOCKERISED),1) -init:: docker-pull -endif - docker-build:: docker build . -f makerules/Dockerfile -t $(DOCKER_PATH) docker-pull:: ifndef ($(DISABLE_DOCKER_PULL),) - docker pull $(ECR_URL)digital-land-python:$(DOCKER_TAG) + docker pull $(DOCKER_PATH) endif -debug_shell: - $(shell_cmd) -digital-land-cli: - $(digital-land-cli) +digital-land-cli:: + $(docker-prefix) \ + --entrypoint digital-land \ + $(DOCKER_PATH) \ + $(wordlist 2, $(words $(MAKECMDGOALS)), $(MAKECMDGOALS)) + + From 24ac90d934e5b842c54adec50d8af5aca94d7562 Mon Sep 17 00:00:00 2001 From: Toby Petty Date: Wed, 12 Jan 2022 17:27:35 +0000 Subject: [PATCH 04/10] Run make init instead of using manually installing python dependencies in collection Dockerfile --- Dockerfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index f9a9c30..d595220 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,8 +5,6 @@ COPY . /pipeline # TODO add labels? -RUN set -xe; \ - [ -f /pipeline/requirements.txt ] && /opt/venv/bin/pip install --upgrade -r requirements.txt; \ - [ -f /pipeline/setup.py ] && /opt/venv/bin/pip install -e ".${PIP_INSTALL_PACKAGE:-test}" +RUN make init ENTRYPOINT ["make"] From 7d8f7d31cdcb1b6b75b46e378f8268b86921d044 Mon Sep 17 00:00:00 2001 From: Toby Petty Date: Thu, 13 Jan 2022 12:16:31 +0000 Subject: [PATCH 05/10] Align with make-specific practises and pass target for dockerisation as arg --- docker.mk | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/docker.mk b/docker.mk index 038f2d4..f29d284 100644 --- a/docker.mk +++ b/docker.mk @@ -9,8 +9,6 @@ else DEVELOPMENT = 1 endif endif -$(info DOCKERISED is $(DOCKERISED)) -$(info DEVELOPMENT is $(DEVELOPMENT)) EXTRA_DOCKER_ARGS := EXTRA_DL_ARGS := @@ -41,7 +39,6 @@ endif endif -$(info EXTRA_DOCKER_ARGS is $(EXTRA_DOCKER_ARGS)) # DOCKER_TAG=latest ECR_URL=public.ecr.aws/l6z6v3j6/ @@ -70,12 +67,13 @@ shell_cmd:: dockerised:: docker-pull $(info MAKECMDGOALS is $(MAKECMDGOALS)) - $(dockerised) $(word 2, $(MAKECMDGOALS)) + $(dockerised) \ + $(TARGET) -docker-build:: +docker-build:: docker-check docker build . -f makerules/Dockerfile -t $(DOCKER_PATH) -docker-pull:: +docker-pull:: docker-ecr-login ifndef ($(DISABLE_DOCKER_PULL),) docker pull $(DOCKER_PATH) endif @@ -85,6 +83,16 @@ digital-land-cli:: $(docker-prefix) \ --entrypoint digital-land \ $(DOCKER_PATH) \ - $(wordlist 2, $(words $(MAKECMDGOALS)), $(MAKECMDGOALS)) + $(TARGET) +docker-check: +ifeq (, $(shell which docker)) + $(error "No docker in $(PATH), consider doing apt-get install docker OR brew install --cask docker") +endif + +docker-ecr-login: docker-check + aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws + +docker-push: docker-ecr-login + docker push $(DOCKER_PATH) From 861f593f4b9bc22819b773103d08609b8ab4e9f1 Mon Sep 17 00:00:00 2001 From: Toby Petty Date: Fri, 14 Jan 2022 11:37:16 +0000 Subject: [PATCH 06/10] Ensure pre-requisite folders aren't created as root by docker bind mount --- docker.mk | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/docker.mk b/docker.mk index f29d284..49780de 100644 --- a/docker.mk +++ b/docker.mk @@ -13,6 +13,7 @@ endif EXTRA_DOCKER_ARGS := EXTRA_DL_ARGS := ifeq ($(DEVELOPMENT),1) +init:: mk-local-collection specification EXTRA_DOCKER_ARGS += -v $(PWD)/local_collection/collection/log:/pipeline/collection/log EXTRA_DOCKER_ARGS += -v $(PWD)/local_collection/collection/resource:/pipeline/collection/resource ifneq (,$(wildcard ./fixed)) @@ -38,6 +39,11 @@ EXTRA_DOCKER_ARGS += -v $(PWD)/../digital-land-python:/src endif +else +mk-collection-resource:: + mkdir -p collection/resource + +init:: mk-collection-resource specification endif # DOCKER_TAG=latest @@ -45,6 +51,10 @@ ECR_URL=public.ecr.aws/l6z6v3j6/ DOCKER_TAG=$(shell basename $(PWD)) DOCKER_PATH=$(ECR_URL)digital-land-python:$(DOCKER_TAG) +mk-local-collection:: + mkdir -p local_collection/collection/log + mkdir -p local_collection/collection/resource + docker-prefix = docker run -t \ -u $(shell id -u) \ -e AWS_ACCESS_KEY_ID \ @@ -74,7 +84,7 @@ docker-build:: docker-check docker build . -f makerules/Dockerfile -t $(DOCKER_PATH) docker-pull:: docker-ecr-login -ifndef ($(DISABLE_DOCKER_PULL),) +ifneq ($(DISABLE_DOCKER_PULL),1) docker pull $(DOCKER_PATH) endif From c44fe72dd9e9a31ff68ecc480935521507c0232a Mon Sep 17 00:00:00 2001 From: Toby Petty Date: Fri, 14 Jan 2022 11:51:02 +0000 Subject: [PATCH 07/10] Remove requirement for aws authentication if DISABLE_DOCKER_PULL=1 --- docker.mk | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docker.mk b/docker.mk index 49780de..51a7fbf 100644 --- a/docker.mk +++ b/docker.mk @@ -75,7 +75,7 @@ shell_cmd:: --entrypoint bash \ $(DOCKER_PATH) -dockerised:: docker-pull +dockerised:: $(info MAKECMDGOALS is $(MAKECMDGOALS)) $(dockerised) \ $(TARGET) @@ -83,12 +83,13 @@ dockerised:: docker-pull docker-build:: docker-check docker build . -f makerules/Dockerfile -t $(DOCKER_PATH) -docker-pull:: docker-ecr-login ifneq ($(DISABLE_DOCKER_PULL),1) +docker-pull:: docker-ecr-login docker pull $(DOCKER_PATH) +else +docker-pull:: endif - digital-land-cli:: $(docker-prefix) \ --entrypoint digital-land \ From 87a13608768cfc6dc56bf16b4872ed9cb069e6a0 Mon Sep 17 00:00:00 2001 From: Toby Petty Date: Fri, 14 Jan 2022 12:36:03 +0000 Subject: [PATCH 08/10] Full pipeline runs for example of article-4-collection Command used: `make TARGET="fetch-organisation collect collection dataset" DEVELOPMENT=1 DISABLE_DOCKER_PULL=1 docker-build mk-local-collection dockerised` Changes: * Always create harmonised and fixed directories before docker run * Create transformed issue and dataset directories before docker run * Work around weird issues with init inheritence not working Outstanding refactors: * Do we need all these directories * Why isn't init inheritence working? --- Dockerfile | 2 +- docker.mk | 38 ++++++++++++++++++++++++-------------- 2 files changed, 25 insertions(+), 15 deletions(-) diff --git a/Dockerfile b/Dockerfile index d595220..8df7595 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ ARG REPO=public.ecr.aws/l6z6v3j6/ -FROM ${REPO}digital-land-python:latest +FROM ${REPO}digital-land-python:digital-land-python COPY . /pipeline diff --git a/docker.mk b/docker.mk index 51a7fbf..63aa2a2 100644 --- a/docker.mk +++ b/docker.mk @@ -10,21 +10,20 @@ DEVELOPMENT = 1 endif endif +fetch-organisation:: + @mkdir -p $(CACHE_DIR) + curl -qfs "https://raw.githubusercontent.com/digital-land/organisation-dataset/main/collection/organisation.csv" > $(CACHE_DIR)organisation.csv + EXTRA_DOCKER_ARGS := EXTRA_DL_ARGS := ifeq ($(DEVELOPMENT),1) -init:: mk-local-collection specification EXTRA_DOCKER_ARGS += -v $(PWD)/local_collection/collection/log:/pipeline/collection/log EXTRA_DOCKER_ARGS += -v $(PWD)/local_collection/collection/resource:/pipeline/collection/resource -ifneq (,$(wildcard ./fixed)) +EXTRA_DOCKER_ARGS += -v $(PWD)/local_collection/transformed:/pipeline/transformed +EXTRA_DOCKER_ARGS += -v $(PWD)/local_collection/issue:/pipeline/issue +EXTRA_DOCKER_ARGS += -v $(PWD)/local_collection/dataset:/pipeline/dataset EXTRA_DOCKER_ARGS += -v $(PWD)/local_collection/fixed:/pipeline/fixed -endif -ifneq (,$(wildcard ./harmonised)) EXTRA_DOCKER_ARGS += -v $(PWD)/local_collection/harmonised:/pipeline/harmonised -endif -ifneq (,$(wildcard ./harmonised)) -EXTRA_DOCKER_ARGS += -v $(PWD)/local_collection/transformed:/pipeline/transformed -endif ifdef ($(LOCAL_SPECIFICATION_PATH),) EXTRA_DOCKER_ARGS += -v $(LOCAL_SPECIFICATION_PATH)/specification:/collection/specification @@ -38,12 +37,26 @@ else ifeq ($(LOCAL_DL_PYTHON),1) EXTRA_DOCKER_ARGS += -v $(PWD)/../digital-land-python:/src endif +mk-local-collection:: + mkdir -p local_collection/collection/log + mkdir -p local_collection/collection/resource + mkdir -p local_collection/transformed + mkdir -p local_collection/issue + mkdir -p local_collection/dataset + mkdir -p local_collection/fixed + mkdir -p local_collection/harmonised +init:: mk-local-collection specification fetch-organisation else -mk-collection-resource:: +mk-collection:: mkdir -p collection/resource + mkdir -p collection/transformed + mkdir -p issue + mkdir -p dataset + mkdir -p fixed + mkdir -p harmonised -init:: mk-collection-resource specification +init:: mk-collection specification fetch-organisation endif # DOCKER_TAG=latest @@ -51,10 +64,6 @@ ECR_URL=public.ecr.aws/l6z6v3j6/ DOCKER_TAG=$(shell basename $(PWD)) DOCKER_PATH=$(ECR_URL)digital-land-python:$(DOCKER_TAG) -mk-local-collection:: - mkdir -p local_collection/collection/log - mkdir -p local_collection/collection/resource - docker-prefix = docker run -t \ -u $(shell id -u) \ -e AWS_ACCESS_KEY_ID \ @@ -65,6 +74,7 @@ docker-prefix = docker run -t \ -e AWS_SESSION_EXPIRATION \ -e AWS_SESSION_TOKEN \ -v $(PWD):/pipeline \ + -v dl-pipeline-var-cache:/var/cache \ $(EXTRA_DOCKER_ARGS) dockerised = $(docker-prefix) \ From e016714f74d33344e325bd54959bb7aa470508d8 Mon Sep 17 00:00:00 2001 From: Toby Petty Date: Fri, 14 Jan 2022 13:20:17 +0000 Subject: [PATCH 09/10] Fix init chaining issues by importing docker.mk from within makerules.mk --- docker.mk | 11 ++--------- makerules.mk | 2 ++ 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/docker.mk b/docker.mk index 63aa2a2..e1f5a12 100644 --- a/docker.mk +++ b/docker.mk @@ -10,9 +10,6 @@ DEVELOPMENT = 1 endif endif -fetch-organisation:: - @mkdir -p $(CACHE_DIR) - curl -qfs "https://raw.githubusercontent.com/digital-land/organisation-dataset/main/collection/organisation.csv" > $(CACHE_DIR)organisation.csv EXTRA_DOCKER_ARGS := EXTRA_DL_ARGS := @@ -37,7 +34,7 @@ else ifeq ($(LOCAL_DL_PYTHON),1) EXTRA_DOCKER_ARGS += -v $(PWD)/../digital-land-python:/src endif -mk-local-collection:: +mk-collection:: mkdir -p local_collection/collection/log mkdir -p local_collection/collection/resource mkdir -p local_collection/transformed @@ -46,7 +43,6 @@ mk-local-collection:: mkdir -p local_collection/fixed mkdir -p local_collection/harmonised -init:: mk-local-collection specification fetch-organisation else mk-collection:: mkdir -p collection/resource @@ -56,10 +52,8 @@ mk-collection:: mkdir -p fixed mkdir -p harmonised -init:: mk-collection specification fetch-organisation endif -# DOCKER_TAG=latest ECR_URL=public.ecr.aws/l6z6v3j6/ DOCKER_TAG=$(shell basename $(PWD)) DOCKER_PATH=$(ECR_URL)digital-land-python:$(DOCKER_TAG) @@ -74,7 +68,6 @@ docker-prefix = docker run -t \ -e AWS_SESSION_EXPIRATION \ -e AWS_SESSION_TOKEN \ -v $(PWD):/pipeline \ - -v dl-pipeline-var-cache:/var/cache \ $(EXTRA_DOCKER_ARGS) dockerised = $(docker-prefix) \ @@ -85,7 +78,7 @@ shell_cmd:: --entrypoint bash \ $(DOCKER_PATH) -dockerised:: +dockerised:: mk-collection $(info MAKECMDGOALS is $(MAKECMDGOALS)) $(dockerised) \ $(TARGET) diff --git a/makerules.mk b/makerules.mk index 6f954eb..0124be7 100644 --- a/makerules.mk +++ b/makerules.mk @@ -48,6 +48,8 @@ SPATIALITE_EXTENSION="/usr/local/lib/mod_spatialite.dylib" endif endif +include makerules/docker.mk + all:: first-pass second-pass first-pass:: From 048cb37a92da80a328bd0790b58632b8e5cd372a Mon Sep 17 00:00:00 2001 From: Toby Petty Date: Fri, 14 Jan 2022 13:46:30 +0000 Subject: [PATCH 10/10] Remove dependency on DOCKERISED as an arg, and introspect targets instead --- docker.mk | 9 +++++---- pipeline.mk | 2 ++ 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/docker.mk b/docker.mk index e1f5a12..1439498 100644 --- a/docker.mk +++ b/docker.mk @@ -1,9 +1,10 @@ -ifneq ($(DOCKERISED),1) +ifneq ($(findstring dockerised,$(MAKECMDGOALS)),dockerised) DOCKERISED = 0 DEVELOPMENT = 0 else -# Run in development mode by default for now -ifeq ($(DEVELOPMENT),0) +DOCKERISED = 1 +# Don't run in development mode by default for now +ifndef ($(DEVELOPMENT),) DEVELOPMENT = 0 else DEVELOPMENT = 1 @@ -79,7 +80,7 @@ shell_cmd:: $(DOCKER_PATH) dockerised:: mk-collection - $(info MAKECMDGOALS is $(MAKECMDGOALS)) + $(info Running '$(TARGET)' targets in $(DOCKER_TAG) container) $(dockerised) \ $(TARGET) diff --git a/pipeline.mk b/pipeline.mk index f0d1d57..348d44e 100644 --- a/pipeline.mk +++ b/pipeline.mk @@ -63,7 +63,9 @@ define build-dataset = md5sum $@ $(basename $@).sqlite3 endef +ifneq ($(DOCKERISED),1) collection:: collection/pipeline.mk +endif -include collection/pipeline.mk