diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..8df7595 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,10 @@ +ARG REPO=public.ecr.aws/l6z6v3j6/ +FROM ${REPO}digital-land-python:digital-land-python + +COPY . /pipeline + +# TODO add labels? + +RUN make init + +ENTRYPOINT ["make"] diff --git a/docker.mk b/docker.mk new file mode 100644 index 0000000..1439498 --- /dev/null +++ b/docker.mk @@ -0,0 +1,113 @@ +ifneq ($(findstring dockerised,$(MAKECMDGOALS)),dockerised) +DOCKERISED = 0 +DEVELOPMENT = 0 +else +DOCKERISED = 1 +# Don't run in development mode by default for now +ifndef ($(DEVELOPMENT),) +DEVELOPMENT = 0 +else +DEVELOPMENT = 1 +endif +endif + + +EXTRA_DOCKER_ARGS := +EXTRA_DL_ARGS := +ifeq ($(DEVELOPMENT),1) +EXTRA_DOCKER_ARGS += -v $(PWD)/local_collection/collection/log:/pipeline/collection/log +EXTRA_DOCKER_ARGS += -v $(PWD)/local_collection/collection/resource:/pipeline/collection/resource +EXTRA_DOCKER_ARGS += -v $(PWD)/local_collection/transformed:/pipeline/transformed +EXTRA_DOCKER_ARGS += -v $(PWD)/local_collection/issue:/pipeline/issue +EXTRA_DOCKER_ARGS += -v $(PWD)/local_collection/dataset:/pipeline/dataset +EXTRA_DOCKER_ARGS += -v $(PWD)/local_collection/fixed:/pipeline/fixed +EXTRA_DOCKER_ARGS += -v $(PWD)/local_collection/harmonised:/pipeline/harmonised + +ifdef ($(LOCAL_SPECIFICATION_PATH),) +EXTRA_DOCKER_ARGS += -v $(LOCAL_SPECIFICATION_PATH)/specification:/collection/specification +else ifeq ($(LOCAL_SPECIFICATION),1) +EXTRA_DOCKER_ARGS += -v $(PWD)/../specification/specification:/collection/specification +endif + +ifdef ($(LOCAL_DL_PYTHON_PATH),) +EXTRA_DOCKER_ARGS += -v $(LOCAL_DL_PYTHON_PATH):/Src +else ifeq ($(LOCAL_DL_PYTHON),1) +EXTRA_DOCKER_ARGS += -v $(PWD)/../digital-land-python:/src +endif + +mk-collection:: + mkdir -p local_collection/collection/log + mkdir -p local_collection/collection/resource + mkdir -p local_collection/transformed + mkdir -p local_collection/issue + mkdir -p local_collection/dataset + mkdir -p local_collection/fixed + mkdir -p local_collection/harmonised + +else +mk-collection:: + mkdir -p collection/resource + mkdir -p collection/transformed + mkdir -p issue + mkdir -p dataset + mkdir -p fixed + mkdir -p harmonised + +endif + +ECR_URL=public.ecr.aws/l6z6v3j6/ +DOCKER_TAG=$(shell basename $(PWD)) +DOCKER_PATH=$(ECR_URL)digital-land-python:$(DOCKER_TAG) + +docker-prefix = docker run -t \ + -u $(shell id -u) \ + -e AWS_ACCESS_KEY_ID \ + -e AWS_DEFAULT_REGION \ + -e AWS_REGION \ + -e AWS_SECRET_ACCESS_KEY \ + -e AWS_SECURITY_TOKEN \ + -e AWS_SESSION_EXPIRATION \ + -e AWS_SESSION_TOKEN \ + -v $(PWD):/pipeline \ + $(EXTRA_DOCKER_ARGS) + +dockerised = $(docker-prefix) \ + $(DOCKER_PATH) + +shell_cmd:: + $(docker-prefix) \ + --entrypoint bash \ + $(DOCKER_PATH) + +dockerised:: mk-collection + $(info Running '$(TARGET)' targets in $(DOCKER_TAG) container) + $(dockerised) \ + $(TARGET) + +docker-build:: docker-check + docker build . -f makerules/Dockerfile -t $(DOCKER_PATH) + +ifneq ($(DISABLE_DOCKER_PULL),1) +docker-pull:: docker-ecr-login + docker pull $(DOCKER_PATH) +else +docker-pull:: +endif + +digital-land-cli:: + $(docker-prefix) \ + --entrypoint digital-land \ + $(DOCKER_PATH) \ + $(TARGET) + +docker-check: +ifeq (, $(shell which docker)) + $(error "No docker in $(PATH), consider doing apt-get install docker OR brew install --cask docker") +endif + +docker-ecr-login: docker-check + aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws + +docker-push: docker-ecr-login + docker push $(DOCKER_PATH) + diff --git a/makerules.mk b/makerules.mk index bc07d6c..54f7318 100644 --- a/makerules.mk +++ b/makerules.mk @@ -57,6 +57,8 @@ SPATIALITE_EXTENSION="/usr/local/lib/mod_spatialite.dylib" endif endif +include makerules/docker.mk + all:: first-pass second-pass first-pass:: diff --git a/pipeline.mk b/pipeline.mk index 2495116..6ada626 100644 --- a/pipeline.mk +++ b/pipeline.mk @@ -75,9 +75,12 @@ define build-dataset = csvstack $(wildcard $(ISSUE_DIR)/$(notdir $(basename $@))/*.csv) > $(basename $@)-issue.csv endef +ifneq ($(DOCKERISED),1) collection:: digital-land collection-pipeline-makerules > collection/pipeline.mk +endif + -include collection/pipeline.mk # restart the make process to pick-up collected resource files