From 04def5354436bf51cc8c4a4238cc84af6ed531ef Mon Sep 17 00:00:00 2001 From: Toby Petty Date: Fri, 7 Jan 2022 16:02:24 +0000 Subject: [PATCH 1/5] Define (but don't hook) specification if makerules/specification not present Initial commit of dockerised-fetch (WIP) --- collection.mk | 4 ++++ makerules.mk | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/collection.mk b/collection.mk index e4cf268..3112d70 100644 --- a/collection.mk +++ b/collection.mk @@ -59,3 +59,7 @@ load-resources:: collection/resource/%: @mkdir -p collection/resource/ curl -qfsL '$(DATASTORE_URL)$(REPOSITORY)/$(RESOURCE_DIR)$(notdir $@)' > $@ + +# dev +dockerised-fetch:: + docker run -v $(PWD):/pipeline --workdir /pipeline digital_land_python digital-land fetch $(ENDPOINT_URL) --specification-dir=/collection/specification/ diff --git a/makerules.mk b/makerules.mk index 6f954eb..20f2d1d 100644 --- a/makerules.mk +++ b/makerules.mk @@ -86,7 +86,6 @@ prune:: makerules:: curl -qfsL '$(SOURCE_URL)/makerules/main/makerules.mk' > makerules/makerules.mk -ifeq (,$(wildcard ./makerules/specification.mk)) # update local copies of specification files specification:: @mkdir -p specification/ @@ -101,6 +100,7 @@ specification:: curl -qfsL '$(SOURCE_URL)/specification/main/specification/pipeline.csv' > specification/pipeline.csv curl -qfsL '$(SOURCE_URL)/specification/main/specification/theme.csv' > specification/theme.csv +ifeq (,$(wildcard ./makerules/specification.mk)) init:: specification endif From 932d7e0c3f1c558b2427a46c14686f1cc65e59ed Mon Sep 17 00:00:00 2001 From: Toby Petty Date: Fri, 7 Jan 2022 17:21:49 +0000 Subject: [PATCH 2/5] Improved dockerised-fetch make command, allowed for use of local specification and digital-land-python repos --- collection.mk | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/collection.mk b/collection.mk index 3112d70..029b7e8 100644 --- a/collection.mk +++ b/collection.mk @@ -16,6 +16,19 @@ ifeq ($(DATASTORE_URL),) DATASTORE_URL=https://collection-dataset.s3.eu-west-2.amazonaws.com/ endif +EXTRA_MOUNTS := +ifdef ($(LOCAL_SPECIFICATION_PATH),) + EXTRA_MOUNTS += -v $(LOCAL_SPECIFICATION_PATH)/specification:/collection/specification +else ifeq ($(LOCAL_SPECIFICATION),1) + EXTRA_MOUNTS += -v $(PWD)/../specification/specificaiton:/collection/specification +endif + +ifdef ($(LOCAL_DL_PYTHON_PATH),) + EXTRA_MOUNTS += -v $(LOCAL_DL_PYTHON_PATH):/Src +else ifeq ($(LOCAL_DL_PYTHON),1) + EXTRA_MOUNTS += -v $(PWD)/../digital-land-python:/src" +endif + # data sources SOURCE_CSV=$(COLLECTION_DIR)source.csv @@ -62,4 +75,15 @@ collection/resource/%: # dev dockerised-fetch:: - docker run -v $(PWD):/pipeline --workdir /pipeline digital_land_python digital-land fetch $(ENDPOINT_URL) --specification-dir=/collection/specification/ + mkdir -p local_collection + docker run -t \ + -u $(shell id -u) \ + -v $(PWD):/pipeline \ + -v $(PWD)/local_collection:/data \ + $(EXTRA_MOUNTS) \ + --workdir /data \ + $(ECR_URL)digital_land_python:$(DOCKER_TAG) \ + digital-land \ + --specification-dir /collection/specification \ + fetch \ + '$(ENDPOINT_URL)' From e4da3e891db44b261380b95b8ae26ea2ea2edefc Mon Sep 17 00:00:00 2001 From: Toby Petty Date: Fri, 7 Jan 2022 17:42:31 +0000 Subject: [PATCH 3/5] Moved functionality out into development.mk till we can make a docker.mk --- collection.mk | 28 ---------------------------- development.mk | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 28 deletions(-) diff --git a/collection.mk b/collection.mk index 029b7e8..e4cf268 100644 --- a/collection.mk +++ b/collection.mk @@ -16,19 +16,6 @@ ifeq ($(DATASTORE_URL),) DATASTORE_URL=https://collection-dataset.s3.eu-west-2.amazonaws.com/ endif -EXTRA_MOUNTS := -ifdef ($(LOCAL_SPECIFICATION_PATH),) - EXTRA_MOUNTS += -v $(LOCAL_SPECIFICATION_PATH)/specification:/collection/specification -else ifeq ($(LOCAL_SPECIFICATION),1) - EXTRA_MOUNTS += -v $(PWD)/../specification/specificaiton:/collection/specification -endif - -ifdef ($(LOCAL_DL_PYTHON_PATH),) - EXTRA_MOUNTS += -v $(LOCAL_DL_PYTHON_PATH):/Src -else ifeq ($(LOCAL_DL_PYTHON),1) - EXTRA_MOUNTS += -v $(PWD)/../digital-land-python:/src" -endif - # data sources SOURCE_CSV=$(COLLECTION_DIR)source.csv @@ -72,18 +59,3 @@ load-resources:: collection/resource/%: @mkdir -p collection/resource/ curl -qfsL '$(DATASTORE_URL)$(REPOSITORY)/$(RESOURCE_DIR)$(notdir $@)' > $@ - -# dev -dockerised-fetch:: - mkdir -p local_collection - docker run -t \ - -u $(shell id -u) \ - -v $(PWD):/pipeline \ - -v $(PWD)/local_collection:/data \ - $(EXTRA_MOUNTS) \ - --workdir /data \ - $(ECR_URL)digital_land_python:$(DOCKER_TAG) \ - digital-land \ - --specification-dir /collection/specification \ - fetch \ - '$(ENDPOINT_URL)' diff --git a/development.mk b/development.mk index 7e81738..430c92f 100644 --- a/development.mk +++ b/development.mk @@ -3,6 +3,22 @@ workon\ dev +EXTRA_MOUNTS := +ifdef ($(LOCAL_SPECIFICATION_PATH),) + EXTRA_MOUNTS += -v $(LOCAL_SPECIFICATION_PATH)/specification:/collection/specification +else ifeq ($(LOCAL_SPECIFICATION),1) + EXTRA_MOUNTS += -v $(PWD)/../specification/specificaiton:/collection/specification +endif + +ifdef ($(LOCAL_DL_PYTHON_PATH),) + EXTRA_MOUNTS += -v $(LOCAL_DL_PYTHON_PATH):/Src +else ifeq ($(LOCAL_DL_PYTHON),1) + EXTRA_MOUNTS += -v $(PWD)/../digital-land-python:/src +endif + +DOCKER_TAG=latest +ECR_URL= + # useful when developing # export PIP_REQUIRE_VIRTUALENV=true @@ -25,3 +41,19 @@ prune:: makerules:: curl -qfsL '$(SOURCE_URL)/makerules/main/development.mk' > makerules/development.mk + +dockerised = docker run -t \ + -u $(shell id -u) \ + -v $(PWD):/pipeline \ + -v $(PWD)/local_collection:/data \ + $(EXTRA_MOUNTS) \ + --workdir /data \ + $(ECR_URL)digital_land_python:$(DOCKER_TAG) \ + digital-land \ + --specification-dir /collection/specification + +dockerised-fetch:: + mkdir -p local_collection + $(dockerised) \ + fetch \ + '$(ENDPOINT_URL)' From 8256996b6fc572c92840e04302b33d7e1932bf80 Mon Sep 17 00:00:00 2001 From: Toby Petty Date: Mon, 10 Jan 2022 11:40:19 +0000 Subject: [PATCH 4/5] Add ECR_REPO domain --- development.mk | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/development.mk b/development.mk index 430c92f..d704ae1 100644 --- a/development.mk +++ b/development.mk @@ -17,7 +17,7 @@ else ifeq ($(LOCAL_DL_PYTHON),1) endif DOCKER_TAG=latest -ECR_URL= +ECR_URL=public.ecr.aws/l6z6v3j6/ # useful when developing # export PIP_REQUIRE_VIRTUALENV=true @@ -48,11 +48,16 @@ dockerised = docker run -t \ -v $(PWD)/local_collection:/data \ $(EXTRA_MOUNTS) \ --workdir /data \ - $(ECR_URL)digital_land_python:$(DOCKER_TAG) \ + $(ECR_URL)digital-land-python:$(DOCKER_TAG) \ digital-land \ --specification-dir /collection/specification -dockerised-fetch:: +docker-pull:: +ifndef ($(DISABLE_DOCKER_PULL),) + docker pull $(ECR_URL)digital-land-python:$(DOCKER_TAG) +endif + +dockerised-fetch:: docker-pull mkdir -p local_collection $(dockerised) \ fetch \ From 1889d6eb336a7e624d5f8de72ff736d039ea3721 Mon Sep 17 00:00:00 2001 From: Toby Petty Date: Tue, 11 Jan 2022 17:13:41 +0000 Subject: [PATCH 5/5] Handle package installation from repository within entrypoint script --- development.mk | 2 +- makerules.mk | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/development.mk b/development.mk index d704ae1..688e70d 100644 --- a/development.mk +++ b/development.mk @@ -43,7 +43,7 @@ makerules:: curl -qfsL '$(SOURCE_URL)/makerules/main/development.mk' > makerules/development.mk dockerised = docker run -t \ - -u $(shell id -u) \ + -e LOCAL_USER_ID=$(shell id -u) \ -v $(PWD):/pipeline \ -v $(PWD)/local_collection:/data \ $(EXTRA_MOUNTS) \ diff --git a/makerules.mk b/makerules.mk index 20f2d1d..8b0a7cb 100644 --- a/makerules.mk +++ b/makerules.mk @@ -58,6 +58,7 @@ second-pass:: @: # initialise +ifeq (,$(wildcard /.dockerenv )) init:: pip install --upgrade pip ifneq (,$(wildcard requirements.txt)) @@ -66,6 +67,7 @@ endif ifneq (,$(wildcard setup.py)) pip install -e .$(PIP_INSTALL_PACKAGE) endif +endif submodules:: git submodule update --init --recursive --remote