Skip to content

Implement way to run pipeline commands locally in docker, starting with fetch #5

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
Staberinde wants to merge 5 commits into from
Closed

Implement way to run pipeline commands locally in docker, starting with fetch #5

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
04def53
Define (but don't hook) specification if makerules/specification not
Staberinde Jan 7, 2022
932d7e0
Improved dockerised-fetch make command, allowed for use of local spec…
Staberinde Jan 7, 2022
e4da3e8
Moved functionality out into development.mk till we can make a docker.mk
Staberinde Jan 7, 2022
8256996
Add ECR_REPO domain
Staberinde Jan 10, 2022
1889d6e
Handle package installation from repository within entrypoint script
Staberinde Jan 11, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 37 additions & 0 deletions development.mk
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,22 @@
workon\
dev

EXTRA_MOUNTS :=
ifdef ($(LOCAL_SPECIFICATION_PATH),)
EXTRA_MOUNTS += -v $(LOCAL_SPECIFICATION_PATH)/specification:/collection/specification
else ifeq ($(LOCAL_SPECIFICATION),1)
EXTRA_MOUNTS += -v $(PWD)/../specification/specificaiton:/collection/specification
endif

ifdef ($(LOCAL_DL_PYTHON_PATH),)
EXTRA_MOUNTS += -v $(LOCAL_DL_PYTHON_PATH):/Src
else ifeq ($(LOCAL_DL_PYTHON),1)
EXTRA_MOUNTS += -v $(PWD)/../digital-land-python:/src
endif

DOCKER_TAG=latest
ECR_URL=public.ecr.aws/l6z6v3j6/

# useful when developing
# export PIP_REQUIRE_VIRTUALENV=true

Expand All @@ -25,3 +41,24 @@ prune::

makerules::
curl -qfsL '$(SOURCE_URL)/makerules/main/development.mk' > makerules/development.mk

dockerised = docker run -t \
-e LOCAL_USER_ID=$(shell id -u) \
-v $(PWD):/pipeline \
-v $(PWD)/local_collection:/data \
$(EXTRA_MOUNTS) \
--workdir /data \
$(ECR_URL)digital-land-python:$(DOCKER_TAG) \
digital-land \
--specification-dir /collection/specification

docker-pull::
ifndef ($(DISABLE_DOCKER_PULL),)
docker pull $(ECR_URL)digital-land-python:$(DOCKER_TAG)
endif

dockerised-fetch:: docker-pull
mkdir -p local_collection
$(dockerised) \
fetch \
'$(ENDPOINT_URL)'
4 changes: 3 additions & 1 deletion makerules.mk
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ second-pass::
@:

# initialise
ifeq (,$(wildcard /.dockerenv ))
init::
pip install --upgrade pip
ifneq (,$(wildcard requirements.txt))
Expand All @@ -66,6 +67,7 @@ endif
ifneq (,$(wildcard setup.py))
pip install -e .$(PIP_INSTALL_PACKAGE)
endif
endif

submodules::
git submodule update --init --recursive --remote
Expand All @@ -86,7 +88,6 @@ prune::
makerules::
curl -qfsL '$(SOURCE_URL)/makerules/main/makerules.mk' > makerules/makerules.mk

ifeq (,$(wildcard ./makerules/specification.mk))
# update local copies of specification files
specification::
@mkdir -p specification/
Expand All @@ -101,6 +102,7 @@ specification::
curl -qfsL '$(SOURCE_URL)/specification/main/specification/pipeline.csv' > specification/pipeline.csv
curl -qfsL '$(SOURCE_URL)/specification/main/specification/theme.csv' > specification/theme.csv

ifeq (,$(wildcard ./makerules/specification.mk))
init:: specification
endif

Expand Down