From 63fe4fcc200384c625ff727126c39226d185401c Mon Sep 17 00:00:00 2001 From: Tomek Fraczek Date: Wed, 8 May 2019 15:41:29 -0700 Subject: [PATCH 01/22] updated requirements file --- requirements.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 2812921f..0c4278ce 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,6 @@ scipy matplotlib numexpr cython -django-celery traits pandas patsy @@ -17,7 +16,9 @@ sphinx numpydoc tornado tables +sklearn pyserial h5py pygame -ipdb \ No newline at end of file +ipdb +django-celery \ No newline at end of file From cb3fafa6887dfdc6145c8a691b59d13937d235a8 Mon Sep 17 00:00:00 2001 From: Tomek Fraczek Date: Thu, 9 May 2019 12:19:53 -0700 Subject: [PATCH 02/22] Move to docker based install Dockerfile to prepare for installation Clone of install script dedicated to docker installation - Remove sudo - Remove git pull of tasks and analysis - Update rabbitmq repo - Remove redundant apt-get calls - Install all pip dependencies through requirements file --- bmi3d.dockerfile | 17 ++++++ install/docker_install.sh | 119 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 136 insertions(+) create mode 100644 bmi3d.dockerfile create mode 100644 install/docker_install.sh diff --git a/bmi3d.dockerfile b/bmi3d.dockerfile new file mode 100644 index 00000000..c1bb1ab9 --- /dev/null +++ b/bmi3d.dockerfile @@ -0,0 +1,17 @@ +FROM python:3 + +RUN echo $HOME + +RUN apt-get install apt + +RUN mkdir $HOME/code/ +COPY . $HOME/code/bmi3d/ +WORKDIR $HOME/code/bmi3d/ + +RUN echo $HOME +RUN echo $PWD + +RUN sed -i 's/\r$//' install/docker_install.sh # Fix line endings because windows breaks bash +RUN ./install/docker_install.sh + +CMD [ "python", "./your-daemon-or-script.py" ] \ No newline at end of file diff --git a/install/docker_install.sh b/install/docker_install.sh new file mode 100644 index 00000000..1a138db0 --- /dev/null +++ b/install/docker_install.sh @@ -0,0 +1,119 @@ +#!/bin/bash +####### Declare environment variables +CODE=/code +BMI3D=$CODE/bmi3d ### Directory in which to install the bmi3d software +USER=root # We're in a docker container so root is safe + +####### Set up directories +mkdir -p $CODE +mkdir /backup +chown $USER /backup + +mkdir /storage +chown -R $USER /storage +mkdir /storage/plots +mkdir $CODE/src/ + +# make log directory +mkdir $BMI3D/log + +# Add the repository to get the rabbitmq server +curl -s https://packagecloud.io/install/repositories/rabbitmq/rabbitmq-server/script.deb.sh | bash +apt-get update + + +####### Install Ubuntu dependencies +# apt-get -y install python-pip libhdf5-serial-dev +# setup the CIFS +apt-get -y install smbclient cifs-utils +# pygame +# apt-get -y install mercurial python-dev python-numpy ffmpeg libsdl-image1.2-dev libsdl-mixer1.2-dev libsdl-ttf2.0-dev libsmpeg-dev libsdl1.2-dev libportmidi-dev libswscale-dev libavformat-dev libavcodec-dev +# install tools +apt-get -y install bison flex +# ssh +apt-get -y install openssh-server +# text editors +# apt-get -y install sublime-text vim-gnome +apt-get -y install rabbitmq-server +apt-get -y install libusb-dev +apt-get -y install ipython +# NIDAQ +apt-get -y install libcomedi-dev python-comedilib swig +# DHCP server +apt-get -y install isc-dhcp-server +apt-get -y install sqlite3 +# Arduino IDE +apt-get install arduino arduino-core +# Serial lib +apt-get install setserial + +####### Install Python dependencies +pip install -r requirements.txt + + +####### Download any src code +git clone https://github.com/sgowda/plot $HOME/code/plotutil +git clone https://github.com/sgowda/robotics_toolbox $HOME/code/robotics +# pygame +hg clone https://bitbucket.org/pygame/pygame $HOME/code/pygame +# Phidgets code +wget http://www.phidgets.com/downloads/libraries/libphidget.tar.gz +wget http://www.phidgets.com/downloads/libraries/PhidgetsPython.zip + + + + +####### Install source code, configure software +# plexread module +cd $BMI3D/riglib +python setup.py install + +# pygame +cd $HOME/code/pygame +python setup.py install + +# symlink for iPython +ln -s /usr/bin/ipython /usr/bin/ipy + +# NIDAQ software -- deprecated! +# $HOME/code/bmi3d/riglib/nidaq/build.sh + +# Phidgets libraries +cd $CODE/src/ +tar xzf libphidget.tar.gz +cd libphidget* +./configure +make +make install + +cd $CODE/src/ +unzip PhidgetsPython.zip +cd PhidgetsPython +python setup.py install + + + +####### Configure udev rules, permissions +# Phidgets +cp $CODE/src/libphidget*/udev/99-phidgets.rules /etc/udev/rules.d +chmod a+r /etc/udev/rules.d/99-phidgets.rules +# NIDAQ +cp $HOME/code/bmi3d/install/udev/comedi.rules /etc/udev/rules.d/ +chmod a+r /etc/udev/rules.d/comedi.rules +udevadm control --reload-rules +# Group permissions +usermod -a -G iocard $USER # NIDAQ card belongs to iocard group +usermod -a -G dialout $USER # Serial ports belong to 'dialout' group + + +####### Reconfigure .bashrc +sed -i '$a export PYTHONPATH=$PYTHONPATH:$HOME/code/robotics' $HOME/.bashrc +sed -i '$a export BMI3D=/home/lab/code/bmi3d' $HOME/.bashrc +sed -i '$a source $HOME/code/bmi3d/pathconfig.sh' $HOME/.bashrc +source $HOME/.bashrc + +chown -R $USER ~/.matplotlib + +cd $HOME/code/bmi3d/db +python manage.py syncdb +# Add superuser 'lab' with password 'lab' From 9b6a02e711d128bad2316a49f5816f03cbf44b6b Mon Sep 17 00:00:00 2001 From: Tomek Fraczek Date: Thu, 9 May 2019 14:24:27 -0700 Subject: [PATCH 03/22] Split docker install to facilitate caching, remove opengl-accelerate --- bmi3d.dockerfile | 27 ++++++--- install/docker/package_install.sh | 32 +++++++++++ .../src_code_install.sh} | 57 +------------------ requirements.txt | 1 - 4 files changed, 53 insertions(+), 64 deletions(-) create mode 100644 install/docker/package_install.sh rename install/{docker_install.sh => docker/src_code_install.sh} (53%) diff --git a/bmi3d.dockerfile b/bmi3d.dockerfile index c1bb1ab9..d8595d99 100644 --- a/bmi3d.dockerfile +++ b/bmi3d.dockerfile @@ -1,17 +1,26 @@ FROM python:3 -RUN echo $HOME +####### Set up directories and copy source code +RUN mkdir -v -p /code/src/ -RUN apt-get install apt +COPY . /code/bmi3d/ +WORKDIR /code/bmi3d/ -RUN mkdir $HOME/code/ -COPY . $HOME/code/bmi3d/ -WORKDIR $HOME/code/bmi3d/ +RUN mkdir -v -p /backup && chown root /backup +RUN mkdir -v -p /storage/plots && chown -R root /storage +RUN mkdir -v logs -RUN echo $HOME -RUN echo $PWD +# Prepare scripts: Fix line endings because windows breaks bash +RUN sed -i 's/\r$//' install/docker/package_install.sh +RUN sed -i 's/\r$//' install/docker/src_code_install.sh -RUN sed -i 's/\r$//' install/docker_install.sh # Fix line endings because windows breaks bash -RUN ./install/docker_install.sh +# Install required ubuntu packages +RUN ./install/docker/package_install.sh +# Install python dependencies +RUN pip install --upgrade pip +RUN pip install -r requirements.txt + +RUN ./install/docker/src_code_install.sh + CMD [ "python", "./your-daemon-or-script.py" ] \ No newline at end of file diff --git a/install/docker/package_install.sh b/install/docker/package_install.sh new file mode 100644 index 00000000..c6bb27d4 --- /dev/null +++ b/install/docker/package_install.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# Add the repository to get the rabbitmq server +curl -s https://packagecloud.io/install/repositories/rabbitmq/rabbitmq-server/script.deb.sh | bash +apt-get update +apt-get -y upgrade + + +####### Install Ubuntu dependencies +# apt-get -y install python-pip libhdf5-serial-dev +# setup the CIFS +apt-get -y install smbclient cifs-utils +# pygame +# apt-get -y install mercurial python-dev python-numpy ffmpeg libsdl-image1.2-dev libsdl-mixer1.2-dev libsdl-ttf2.0-dev libsmpeg-dev libsdl1.2-dev libportmidi-dev libswscale-dev libavformat-dev libavcodec-dev +# install tools +apt-get -y install bison flex +# ssh +apt-get -y install openssh-server +# text editors +# apt-get -y install sublime-text vim-gnome +apt-get -y install rabbitmq-server +apt-get -y install libusb-dev +# apt-get -y install ipython +# NIDAQ +apt-get -y install libcomedi-dev python-comedilib swig +# DHCP server +apt-get -y install isc-dhcp-server +apt-get -y install sqlite3 +# Arduino IDE +# apt-get -y install arduino arduino-core +# Serial lib +# apt-get -y install setserial \ No newline at end of file diff --git a/install/docker_install.sh b/install/docker/src_code_install.sh similarity index 53% rename from install/docker_install.sh rename to install/docker/src_code_install.sh index 1a138db0..621f4b2b 100644 --- a/install/docker_install.sh +++ b/install/docker/src_code_install.sh @@ -4,77 +4,26 @@ CODE=/code BMI3D=$CODE/bmi3d ### Directory in which to install the bmi3d software USER=root # We're in a docker container so root is safe -####### Set up directories -mkdir -p $CODE -mkdir /backup -chown $USER /backup - -mkdir /storage -chown -R $USER /storage -mkdir /storage/plots -mkdir $CODE/src/ - -# make log directory -mkdir $BMI3D/log - -# Add the repository to get the rabbitmq server -curl -s https://packagecloud.io/install/repositories/rabbitmq/rabbitmq-server/script.deb.sh | bash -apt-get update - - -####### Install Ubuntu dependencies -# apt-get -y install python-pip libhdf5-serial-dev -# setup the CIFS -apt-get -y install smbclient cifs-utils -# pygame -# apt-get -y install mercurial python-dev python-numpy ffmpeg libsdl-image1.2-dev libsdl-mixer1.2-dev libsdl-ttf2.0-dev libsmpeg-dev libsdl1.2-dev libportmidi-dev libswscale-dev libavformat-dev libavcodec-dev -# install tools -apt-get -y install bison flex -# ssh -apt-get -y install openssh-server -# text editors -# apt-get -y install sublime-text vim-gnome -apt-get -y install rabbitmq-server -apt-get -y install libusb-dev -apt-get -y install ipython -# NIDAQ -apt-get -y install libcomedi-dev python-comedilib swig -# DHCP server -apt-get -y install isc-dhcp-server -apt-get -y install sqlite3 -# Arduino IDE -apt-get install arduino arduino-core -# Serial lib -apt-get install setserial - -####### Install Python dependencies -pip install -r requirements.txt - - ####### Download any src code git clone https://github.com/sgowda/plot $HOME/code/plotutil git clone https://github.com/sgowda/robotics_toolbox $HOME/code/robotics # pygame hg clone https://bitbucket.org/pygame/pygame $HOME/code/pygame # Phidgets code -wget http://www.phidgets.com/downloads/libraries/libphidget.tar.gz -wget http://www.phidgets.com/downloads/libraries/PhidgetsPython.zip - - +wget https://www.phidgets.com/downloads/phidget22/libraries/linux/libphidget22/libphidget22-1.1.20190417.tar.gz +wget https://www.phidgets.com/downloads/phidget22/libraries/any/Phidget22Python/Phidget22Python_1.1.20190418.zip ####### Install source code, configure software # plexread module cd $BMI3D/riglib +echo which python python setup.py install # pygame cd $HOME/code/pygame python setup.py install -# symlink for iPython -ln -s /usr/bin/ipython /usr/bin/ipy - # NIDAQ software -- deprecated! # $HOME/code/bmi3d/riglib/nidaq/build.sh diff --git a/requirements.txt b/requirements.txt index 0c4278ce..f950a8cf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,6 @@ pandas patsy statsmodels PyOpenGL -PyOpenGL_accelerate Django pylibftdi nitime From b3904eedc41c3907f4bce8674608ee2dfe5ba603 Mon Sep 17 00:00:00 2001 From: Tomek Fraczek Date: Thu, 9 May 2019 15:10:40 -0700 Subject: [PATCH 04/22] Move package install to docker file, re-order to use cache --- bmi3d.dockerfile | 53 +++++++++++++++++++++++-------- install/docker/package_install.sh | 32 ------------------- 2 files changed, 40 insertions(+), 45 deletions(-) delete mode 100644 install/docker/package_install.sh diff --git a/bmi3d.dockerfile b/bmi3d.dockerfile index d8595d99..5a277db8 100644 --- a/bmi3d.dockerfile +++ b/bmi3d.dockerfile @@ -1,26 +1,53 @@ FROM python:3 +##### Install required ubuntu packages +# Add the repository to get the rabbitmq server +RUN curl -s https://packagecloud.io/install/repositories/rabbitmq/rabbitmq-server/script.deb.sh | bash +RUN apt-get update && apt-get -y upgrade + +# Install dependencies +RUN apt-get install -y \ + smbclient \ + cifs-utils \ + bison \ + flex \ + openssh-server \ + libusb-dev\ + libcomedi-dev \ + python-comedilib \ + swig \ + isc-dhcp-server \ + sqlite3 + +# Rabbitmq can run into issues so run as a separate command +RUN apt-get install rabbitmq-server + + +###### Install python dependencies +RUN pip install --upgrade pip +RUN pip install -r requirements.txt + + ####### Set up directories and copy source code -RUN mkdir -v -p /code/src/ +#RUN mkdir -v -p /code/src/ +#RUN mkdir -v -p /backup && chown root /backup +#RUN mkdir -v -p /storage/plots && chown -R root /storage + + +# ----- Expected cache invalidation here ------- # COPY . /code/bmi3d/ WORKDIR /code/bmi3d/ - -RUN mkdir -v -p /backup && chown root /backup -RUN mkdir -v -p /storage/plots && chown -R root /storage RUN mkdir -v logs + + + # Prepare scripts: Fix line endings because windows breaks bash -RUN sed -i 's/\r$//' install/docker/package_install.sh -RUN sed -i 's/\r$//' install/docker/src_code_install.sh +# RUN sed -i 's/\r$//' install/docker/src_code_install.sh -# Install required ubuntu packages -RUN ./install/docker/package_install.sh -# Install python dependencies -RUN pip install --upgrade pip -RUN pip install -r requirements.txt -RUN ./install/docker/src_code_install.sh +#RUN ./install/docker/src_code_install.sh -CMD [ "python", "./your-daemon-or-script.py" ] \ No newline at end of file +#CMD [ "python", "./your-daemon-or-script.py" ] \ No newline at end of file diff --git a/install/docker/package_install.sh b/install/docker/package_install.sh deleted file mode 100644 index c6bb27d4..00000000 --- a/install/docker/package_install.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash - -# Add the repository to get the rabbitmq server -curl -s https://packagecloud.io/install/repositories/rabbitmq/rabbitmq-server/script.deb.sh | bash -apt-get update -apt-get -y upgrade - - -####### Install Ubuntu dependencies -# apt-get -y install python-pip libhdf5-serial-dev -# setup the CIFS -apt-get -y install smbclient cifs-utils -# pygame -# apt-get -y install mercurial python-dev python-numpy ffmpeg libsdl-image1.2-dev libsdl-mixer1.2-dev libsdl-ttf2.0-dev libsmpeg-dev libsdl1.2-dev libportmidi-dev libswscale-dev libavformat-dev libavcodec-dev -# install tools -apt-get -y install bison flex -# ssh -apt-get -y install openssh-server -# text editors -# apt-get -y install sublime-text vim-gnome -apt-get -y install rabbitmq-server -apt-get -y install libusb-dev -# apt-get -y install ipython -# NIDAQ -apt-get -y install libcomedi-dev python-comedilib swig -# DHCP server -apt-get -y install isc-dhcp-server -apt-get -y install sqlite3 -# Arduino IDE -# apt-get -y install arduino arduino-core -# Serial lib -# apt-get -y install setserial \ No newline at end of file From 0d937f59ed43e093dfb8443abd12562e22f38afc Mon Sep 17 00:00:00 2001 From: Tomek Fraczek Date: Mon, 13 May 2019 12:11:35 -0700 Subject: [PATCH 05/22] Fix rabbitmq install, reactivate python installs --- bmi3d.dockerfile | 54 +++++++++++++++++++++++++++++++----------------- 1 file changed, 35 insertions(+), 19 deletions(-) diff --git a/bmi3d.dockerfile b/bmi3d.dockerfile index 5a277db8..8dc93ddb 100644 --- a/bmi3d.dockerfile +++ b/bmi3d.dockerfile @@ -1,11 +1,13 @@ FROM python:3 -##### Install required ubuntu packages -# Add the repository to get the rabbitmq server +#### Connect up third-party repositories (rabbitmq and erlang) RUN curl -s https://packagecloud.io/install/repositories/rabbitmq/rabbitmq-server/script.deb.sh | bash -RUN apt-get update && apt-get -y upgrade +RUN echo "deb http://dl.bintray.com/rabbitmq-erlang/debian bionic erlang" \ + >> /etc/apt/sources.list.d/bintray.erlang.list +RUN apt-get -y update + -# Install dependencies +##### Install required ubuntu packages RUN apt-get install -y \ smbclient \ cifs-utils \ @@ -19,35 +21,49 @@ RUN apt-get install -y \ isc-dhcp-server \ sqlite3 -# Rabbitmq can run into issues so run as a separate command -RUN apt-get install rabbitmq-server - - -###### Install python dependencies -RUN pip install --upgrade pip -RUN pip install -r requirements.txt +# Install rabbitmq with it's erlang dependencies +RUN apt-get install -y --allow-unauthenticated \ + erlang-base-hipe \ + erlang-asn1 \ + erlang-crypto \ + erlang-eldap \ + erlang-ftp \ + erlang-inets \ + erlang-mnesia \ + erlang-os-mon \ + erlang-parsetools \ + erlang-public-key \ + erlang-runtime-tools \ + erlang-snmp \ + erlang-ssl \ + erlang-syntax-tools \ + erlang-tftp \ + erlang-tools \ + erlang-xmerl \ + rabbitmq-server ####### Set up directories and copy source code -#RUN mkdir -v -p /code/src/ -#RUN mkdir -v -p /backup && chown root /backup -#RUN mkdir -v -p /storage/plots && chown -R root /storage - - -# ----- Expected cache invalidation here ------- # +RUN mkdir -v -p /code/src/ +RUN mkdir -v -p /backup && chown root /backup +RUN mkdir -v -p /storage/plots && chown -R root /storage COPY . /code/bmi3d/ WORKDIR /code/bmi3d/ RUN mkdir -v logs +###### Install python dependencies +RUN echo $PWD +RUN pip install --upgrade pip +RUN pip install -r requirements.txt # Prepare scripts: Fix line endings because windows breaks bash -# RUN sed -i 's/\r$//' install/docker/src_code_install.sh +RUN sed -i 's/\r$//' install/docker/src_code_install.sh -#RUN ./install/docker/src_code_install.sh +RUN ./install/docker/src_code_install.sh #CMD [ "python", "./your-daemon-or-script.py" ] \ No newline at end of file From 10b19a91645853546647e5a64e6fa2641d8a5c1b Mon Sep 17 00:00:00 2001 From: Tomek Fraczek Date: Tue, 14 May 2019 14:09:00 -0700 Subject: [PATCH 06/22] Comment out all plexon setup, seems to try and use python2 --- install/docker/src_code_install.sh | 38 ++++++++++++++++-------------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/install/docker/src_code_install.sh b/install/docker/src_code_install.sh index 621f4b2b..641465f8 100644 --- a/install/docker/src_code_install.sh +++ b/install/docker/src_code_install.sh @@ -10,15 +10,15 @@ git clone https://github.com/sgowda/robotics_toolbox $HOME/code/robotics # pygame hg clone https://bitbucket.org/pygame/pygame $HOME/code/pygame # Phidgets code -wget https://www.phidgets.com/downloads/phidget22/libraries/linux/libphidget22/libphidget22-1.1.20190417.tar.gz -wget https://www.phidgets.com/downloads/phidget22/libraries/any/Phidget22Python/Phidget22Python_1.1.20190418.zip +#wget https://www.phidgets.com/downloads/phidget22/libraries/linux/libphidget22/libphidget22-1.1.20190417.tar.gz +#wget https://www.phidgets.com/downloads/phidget22/libraries/any/Phidget22Python/Phidget22Python_1.1.20190418.zip ####### Install source code, configure software # plexread module -cd $BMI3D/riglib -echo which python -python setup.py install +#cd $BMI3D/riglib +which python +#python setup.py install # pygame cd $HOME/code/pygame @@ -27,25 +27,27 @@ python setup.py install # NIDAQ software -- deprecated! # $HOME/code/bmi3d/riglib/nidaq/build.sh +echo "TESTED IF HERE" + # Phidgets libraries -cd $CODE/src/ -tar xzf libphidget.tar.gz -cd libphidget* -./configure -make -make install +#cd $CODE/src/ +#tar xzf libphidget.tar.gz +#cd libphidget* +#./configure +#make +#make install -cd $CODE/src/ -unzip PhidgetsPython.zip -cd PhidgetsPython -python setup.py install +#cd $CODE/src/ +#unzip PhidgetsPython.zip +#cd PhidgetsPython +#python setup.py install ####### Configure udev rules, permissions # Phidgets -cp $CODE/src/libphidget*/udev/99-phidgets.rules /etc/udev/rules.d -chmod a+r /etc/udev/rules.d/99-phidgets.rules +#cp $CODE/src/libphidget*/udev/99-phidgets.rules /etc/udev/rules.d +#chmod a+r /etc/udev/rules.d/99-phidgets.rules # NIDAQ cp $HOME/code/bmi3d/install/udev/comedi.rules /etc/udev/rules.d/ chmod a+r /etc/udev/rules.d/comedi.rules @@ -61,7 +63,7 @@ sed -i '$a export BMI3D=/home/lab/code/bmi3d' $HOME/.bashrc sed -i '$a source $HOME/code/bmi3d/pathconfig.sh' $HOME/.bashrc source $HOME/.bashrc -chown -R $USER ~/.matplotlib +chown -R $USER ~/.matplotlibs cd $HOME/code/bmi3d/db python manage.py syncdb From ed6143d3cf181eaeb758245fe5a6538cbe0d9d66 Mon Sep 17 00:00:00 2001 From: Tomek Fraczek Date: Tue, 14 May 2019 14:13:14 -0700 Subject: [PATCH 07/22] Multiple dockerfile install bugfixes: - Fix task symlinks (must run from parent dir) - Fix all bash scripts with windows line endings - Save env vars (BMI3D path and python path) for future use - Add default entrypoint to run server - Fix config file path Signed-off-by: Tomek Fraczek --- bmi3d.dockerfile | 33 ++++++++++++++++++++++++++------- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/bmi3d.dockerfile b/bmi3d.dockerfile index 8dc93ddb..b89d631d 100644 --- a/bmi3d.dockerfile +++ b/bmi3d.dockerfile @@ -19,7 +19,8 @@ RUN apt-get install -y \ python-comedilib \ swig \ isc-dhcp-server \ - sqlite3 + sqlite3 \ + vim # Install rabbitmq with it's erlang dependencies RUN apt-get install -y --allow-unauthenticated \ @@ -48,22 +49,40 @@ RUN mkdir -v -p /code/src/ RUN mkdir -v -p /backup && chown root /backup RUN mkdir -v -p /storage/plots && chown -R root /storage -COPY . /code/bmi3d/ +COPY bmi3d/ /code/bmi3d/ +COPY bmi3d_tasks_analysis/ /code/bmi3d_tasks_analysis/ + +RUN ls +RUN ls /code/bmi3d/ + +# Replace windows symlinks with unix ones in the new env +RUN rm /code/bmi3d/analysis && ln -s /code/bmi3d_tasks_analysis/analysis/ /code/bmi3d/analysis \ + && rm /code/bmi3d/tasks && ln -s /code/bmi3d_tasks_analysis/tasks/ /code/bmi3d/tasks + +# Fix all .sh files that might have aquired windows line endings +RUN for f in $(find ./code/ -name "*.sh"); \ + do echo "fixing: $f" && sed -i 's/\r$//' $f; \ + done + WORKDIR /code/bmi3d/ RUN mkdir -v logs ###### Install python dependencies -RUN echo $PWD RUN pip install --upgrade pip RUN pip install -r requirements.txt # Prepare scripts: Fix line endings because windows breaks bash -RUN sed -i 's/\r$//' install/docker/src_code_install.sh +#RUN sed -i 's/\r$//' install/docker/src_code_install.sh +#RUN ./install/docker/src_code_install.sh + + +# Set env vars for future reference +ENV BMI3D="/code/bmi3d" \ + PYTHONPATH="${PYTHONPATH}:/code/bmi3d/:/code/bmi3d_tasks_analysis" +RUN python config/make_config.py -RUN ./install/docker/src_code_install.sh - -#CMD [ "python", "./your-daemon-or-script.py" ] \ No newline at end of file +CMD [ "/bin/bash", "./db/runserver.sh" ] From 5b2bf8f7ac44ed85bb5489b9897ecde487aad6e8 Mon Sep 17 00:00:00 2001 From: Tomek Fraczek Date: Tue, 14 May 2019 14:51:48 -0700 Subject: [PATCH 08/22] Fix references to config file after directory name change --- db/runserver.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/db/runserver.sh b/db/runserver.sh index d0e37a2d..06bfc14b 100755 --- a/db/runserver.sh +++ b/db/runserver.sh @@ -15,7 +15,7 @@ if [ -z "$BMI3D" ] fi #Check /storage (exist ) -storage=$(python $BMI3D/config_files/check_storage.py 2>&1) +storage=$(python $BMI3D/config/check_storage.py 2>&1) if [ $storage == 'False' ]; then echo "/storage does not exist --> if on Ismore, must mount" exit 1 @@ -28,8 +28,8 @@ if [ `ps aux | grep "manage.py runserver" | grep python | wc -l` -gt 0 ]; then fi # Check that a config file is in the correct place, $BMI3D/config -if [ ! -e $BMI3D/config_files/config ]; then - echo "ERROR: cannot find config file! Did you run $BMI3D/config_files/make_config.py?" +if [ ! -e $BMI3D/config/config ]; then + echo "ERROR: cannot find config file! Did you run $BMI3D/config/make_config.py?" exit 1 fi From eb095a46375033846f436dbdc89bccc94794dc34 Mon Sep 17 00:00:00 2001 From: Tomek Fraczek Date: Tue, 14 May 2019 14:52:23 -0700 Subject: [PATCH 09/22] Remove debug statements in dockerfile --- bmi3d.dockerfile | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/bmi3d.dockerfile b/bmi3d.dockerfile index b89d631d..a39df441 100644 --- a/bmi3d.dockerfile +++ b/bmi3d.dockerfile @@ -49,12 +49,10 @@ RUN mkdir -v -p /code/src/ RUN mkdir -v -p /backup && chown root /backup RUN mkdir -v -p /storage/plots && chown -R root /storage +# --- Expect cache invalidation here if source files have changed --- # COPY bmi3d/ /code/bmi3d/ COPY bmi3d_tasks_analysis/ /code/bmi3d_tasks_analysis/ -RUN ls -RUN ls /code/bmi3d/ - # Replace windows symlinks with unix ones in the new env RUN rm /code/bmi3d/analysis && ln -s /code/bmi3d_tasks_analysis/analysis/ /code/bmi3d/analysis \ && rm /code/bmi3d/tasks && ln -s /code/bmi3d_tasks_analysis/tasks/ /code/bmi3d/tasks @@ -72,17 +70,11 @@ RUN mkdir -v logs RUN pip install --upgrade pip RUN pip install -r requirements.txt - -# Prepare scripts: Fix line endings because windows breaks bash -#RUN sed -i 's/\r$//' install/docker/src_code_install.sh -#RUN ./install/docker/src_code_install.sh - - # Set env vars for future reference - ENV BMI3D="/code/bmi3d" \ PYTHONPATH="${PYTHONPATH}:/code/bmi3d/:/code/bmi3d_tasks_analysis" -RUN python config/make_config.py +# RUN python config/make_config.py + CMD [ "/bin/bash", "./db/runserver.sh" ] From d3d584e6c2af9ff769f6b617ace752d0125619df Mon Sep 17 00:00:00 2001 From: Tomek Fraczek Date: Tue, 14 May 2019 14:54:52 -0700 Subject: [PATCH 10/22] Made make_config.py optionally non-interactive, run defaults in build --- bmi3d.dockerfile | 2 +- config/make_config.py | 27 ++++++++++++++++++--------- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/bmi3d.dockerfile b/bmi3d.dockerfile index a39df441..9e40b9d5 100644 --- a/bmi3d.dockerfile +++ b/bmi3d.dockerfile @@ -74,7 +74,7 @@ RUN pip install -r requirements.txt ENV BMI3D="/code/bmi3d" \ PYTHONPATH="${PYTHONPATH}:/code/bmi3d/:/code/bmi3d_tasks_analysis" -# RUN python config/make_config.py +RUN python config/make_config.py --use-defaults CMD [ "/bin/bash", "./db/runserver.sh" ] diff --git a/config/make_config.py b/config/make_config.py index 50eaacc2..0b6e9f97 100644 --- a/config/make_config.py +++ b/config/make_config.py @@ -1,8 +1,9 @@ #!/usr/bin/python ''' -Executable script to create the configuration file for the BMI3D code, a text file called '$BMI3D/config_files/config' +Executable script to create the configuration file for the BMI3D code, a text file called '$BMI3D/config/config' ''' import os +import sys from collections import OrderedDict stuff = OrderedDict() @@ -14,24 +15,32 @@ stuff['plexon IP address'] = dict(addr='10.0.0.13', port=6000) stuff['update_rates'] = dict(hdf_hz=60) +# Add an optional commandline flag to make setup non-interactive +use_defaults = '-y' in sys.argv or '--use-defaults' in sys.argv + from db import settings databases = list(settings.DATABASES.keys()) -for dbname in databases: - stuff['db_config_%s' % dbname] = dict(data_path='/storage') +for db_name in databases: + stuff[f'db_config_{db_name}'] = dict(data_path='/storage') -config_filename = '$BMI3D/config_files/config' +config_filename = '$BMI3D/config/config' config_fh = open(os.path.expandvars(config_filename), 'w') for system_name, system_opts in list(stuff.items()): - config_fh.write('[%s]\n' % system_name) + config_fh.write(f'[{system_name}]\n') print(system_name) for option, default in list(system_opts.items()): - print(option, default) - opt_val = input("Enter value for '%s' (default=%s): " % (option, str(default))) - if opt_val == '': + + if use_defaults: + print(f"Using default ({default}) for {option}") opt_val = default - config_fh.write('%s = %s\n' % (option, opt_val)) + else: + opt_val = input(f" Enter value for '{option}' (default={default}): ") + if opt_val == '': + opt_val = default + + config_fh.write(f'{option} = {opt_val}\n') config_fh.write('\n') print() From 1d42eb186c0e63d61645c8138cbcef088bb30ab2 Mon Sep 17 00:00:00 2001 From: Tomek Fraczek Date: Tue, 14 May 2019 16:01:19 -0700 Subject: [PATCH 11/22] Add bash and batch scripts to run interactive config on docker container --- config.bat | 9 +++++++++ config.sh | 7 +++++++ 2 files changed, 16 insertions(+) create mode 100644 config.bat create mode 100644 config.sh diff --git a/config.bat b/config.bat new file mode 100644 index 00000000..7b003886 --- /dev/null +++ b/config.bat @@ -0,0 +1,9 @@ +@echo off + +IF "%~1"=="" ( + echo "No docker container name given!" + EXIT /B 2 +) ELSE ( + docker run -it --name configuration-inator %1 python config/make_config.py + docker commit configuration-inator %1 +) \ No newline at end of file diff --git a/config.sh b/config.sh new file mode 100644 index 00000000..aa9cea1d --- /dev/null +++ b/config.sh @@ -0,0 +1,7 @@ +#! /bin/bash +if [$# -eq 0]; then + echo "No docker image name given!" +else + docker run -it --name configuration-inator $1 python config/make_config.py; + docker commit configuration-inator $1; +fi \ No newline at end of file From 5cf9d60fcbd6b40a765f09d939ef36793dbd2f0a Mon Sep 17 00:00:00 2001 From: Tomek Fraczek Date: Tue, 14 May 2019 16:01:51 -0700 Subject: [PATCH 12/22] Indent default value output in make_config.py --- config/make_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/make_config.py b/config/make_config.py index 0b6e9f97..cd4ed6b8 100644 --- a/config/make_config.py +++ b/config/make_config.py @@ -33,7 +33,7 @@ for option, default in list(system_opts.items()): if use_defaults: - print(f"Using default ({default}) for {option}") + print(f" Using default ({default}) for {option}") opt_val = default else: opt_val = input(f" Enter value for '{option}' (default={default}): ") From d93291856720fc5ae2f7ea86d235ba7d8db35763 Mon Sep 17 00:00:00 2001 From: Tomek Fraczek Date: Tue, 14 May 2019 16:10:59 -0700 Subject: [PATCH 13/22] Bugfix: was making directory 'logs' instead of 'log' --- bmi3d.dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bmi3d.dockerfile b/bmi3d.dockerfile index 9e40b9d5..0248cb10 100644 --- a/bmi3d.dockerfile +++ b/bmi3d.dockerfile @@ -63,7 +63,7 @@ RUN for f in $(find ./code/ -name "*.sh"); \ done WORKDIR /code/bmi3d/ -RUN mkdir -v logs +RUN mkdir -v log ###### Install python dependencies From 4cedf144e71580f9445c14c4c26d99fc3c107652 Mon Sep 17 00:00:00 2001 From: Tomek Fraczek Date: Tue, 14 May 2019 16:31:57 -0700 Subject: [PATCH 14/22] Use makemigrations/migrate isntead of syncdb --- bmi3d.dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bmi3d.dockerfile b/bmi3d.dockerfile index 0248cb10..5eef8847 100644 --- a/bmi3d.dockerfile +++ b/bmi3d.dockerfile @@ -74,7 +74,9 @@ RUN pip install -r requirements.txt ENV BMI3D="/code/bmi3d" \ PYTHONPATH="${PYTHONPATH}:/code/bmi3d/:/code/bmi3d_tasks_analysis" -RUN python config/make_config.py --use-defaults +RUN python config/make_config.py --use-defaults && + python db/manage.py makemigrations && + python db/manage.py migrate CMD [ "/bin/bash", "./db/runserver.sh" ] From a086f66ac8055d2b470e5ab144f0d57c5aabb490 Mon Sep 17 00:00:00 2001 From: Tomek Fraczek Date: Tue, 14 May 2019 18:49:13 -0700 Subject: [PATCH 15/22] Split dockerfile into 3 to allow re-building of only tail --- bmi3d.base.dockerfile | 51 +++++++++++++++++++++++++ bmi3d.code.dockerfile | 17 +++++++++ bmi3d.dockerfile | 82 ----------------------------------------- bmi3d.python.dockerfile | 35 ++++++++++++++++++ 4 files changed, 103 insertions(+), 82 deletions(-) create mode 100644 bmi3d.base.dockerfile create mode 100644 bmi3d.code.dockerfile delete mode 100644 bmi3d.dockerfile create mode 100644 bmi3d.python.dockerfile diff --git a/bmi3d.base.dockerfile b/bmi3d.base.dockerfile new file mode 100644 index 00000000..35df5f29 --- /dev/null +++ b/bmi3d.base.dockerfile @@ -0,0 +1,51 @@ +##### ################################################################### #### +##### ---- Starting point when any os dependencies have been changed ---- #### +##### ################################################################### #### +FROM python:3 + +#### Connect up third-party repositories (rabbitmq and erlang) +RUN curl -s https://packagecloud.io/install/repositories/rabbitmq/rabbitmq-server/script.deb.sh | bash +RUN echo "deb http://dl.bintray.com/rabbitmq-erlang/debian bionic erlang" \ + >> /etc/apt/sources.list.d/bintray.erlang.list +RUN apt-get -y update + + +##### Install required ubuntu packages +RUN apt-get install -y \ + smbclient \ + cifs-utils \ + bison \ + flex \ + openssh-server \ + libusb-dev\ + libcomedi-dev \ + python-comedilib \ + swig \ + isc-dhcp-server \ + sqlite3 \ + vim + +# Install rabbitmq with it's erlang dependencies +RUN apt-get install -y --allow-unauthenticated \ + erlang-base-hipe \ + erlang-asn1 \ + erlang-crypto \ + erlang-eldap \ + erlang-ftp \ + erlang-inets \ + erlang-mnesia \ + erlang-os-mon \ + erlang-parsetools \ + erlang-public-key \ + erlang-runtime-tools \ + erlang-snmp \ + erlang-ssl \ + erlang-syntax-tools \ + erlang-tftp \ + erlang-tools \ + erlang-xmerl \ + rabbitmq-server + + + + diff --git a/bmi3d.code.dockerfile b/bmi3d.code.dockerfile new file mode 100644 index 00000000..47ffabd2 --- /dev/null +++ b/bmi3d.code.dockerfile @@ -0,0 +1,17 @@ +#### ##################################################################### ##### +#### ---- Starting point when only small local changes have been made ---- ##### +#### ##################################################################### ##### +FROM bmi3d:python +COPY bmi3d/ /code/bmi3d/ +COPY bmi3d_tasks_analysis/ /code/bmi3d_tasks_analysis/ + +RUN python config/make_config.py --use-defaults && \ + python db/manage.py makemigrations && \ + python db/manage.py migrate + +# Fix all .sh files that might have aquired windows line endings +RUN for f in $(find /code/ -name "*.sh"); \ + do echo "fixing: $f" && sed -i 's/\r$//' $f; \ + done + +CMD [ "/bin/bash", "./db/runserver.sh" ] \ No newline at end of file diff --git a/bmi3d.dockerfile b/bmi3d.dockerfile deleted file mode 100644 index 5eef8847..00000000 --- a/bmi3d.dockerfile +++ /dev/null @@ -1,82 +0,0 @@ -FROM python:3 - -#### Connect up third-party repositories (rabbitmq and erlang) -RUN curl -s https://packagecloud.io/install/repositories/rabbitmq/rabbitmq-server/script.deb.sh | bash -RUN echo "deb http://dl.bintray.com/rabbitmq-erlang/debian bionic erlang" \ - >> /etc/apt/sources.list.d/bintray.erlang.list -RUN apt-get -y update - - -##### Install required ubuntu packages -RUN apt-get install -y \ - smbclient \ - cifs-utils \ - bison \ - flex \ - openssh-server \ - libusb-dev\ - libcomedi-dev \ - python-comedilib \ - swig \ - isc-dhcp-server \ - sqlite3 \ - vim - -# Install rabbitmq with it's erlang dependencies -RUN apt-get install -y --allow-unauthenticated \ - erlang-base-hipe \ - erlang-asn1 \ - erlang-crypto \ - erlang-eldap \ - erlang-ftp \ - erlang-inets \ - erlang-mnesia \ - erlang-os-mon \ - erlang-parsetools \ - erlang-public-key \ - erlang-runtime-tools \ - erlang-snmp \ - erlang-ssl \ - erlang-syntax-tools \ - erlang-tftp \ - erlang-tools \ - erlang-xmerl \ - rabbitmq-server - - -####### Set up directories and copy source code -RUN mkdir -v -p /code/src/ -RUN mkdir -v -p /backup && chown root /backup -RUN mkdir -v -p /storage/plots && chown -R root /storage - -# --- Expect cache invalidation here if source files have changed --- # -COPY bmi3d/ /code/bmi3d/ -COPY bmi3d_tasks_analysis/ /code/bmi3d_tasks_analysis/ - -# Replace windows symlinks with unix ones in the new env -RUN rm /code/bmi3d/analysis && ln -s /code/bmi3d_tasks_analysis/analysis/ /code/bmi3d/analysis \ - && rm /code/bmi3d/tasks && ln -s /code/bmi3d_tasks_analysis/tasks/ /code/bmi3d/tasks - -# Fix all .sh files that might have aquired windows line endings -RUN for f in $(find ./code/ -name "*.sh"); \ - do echo "fixing: $f" && sed -i 's/\r$//' $f; \ - done - -WORKDIR /code/bmi3d/ -RUN mkdir -v log - - -###### Install python dependencies -RUN pip install --upgrade pip -RUN pip install -r requirements.txt - -# Set env vars for future reference -ENV BMI3D="/code/bmi3d" \ - PYTHONPATH="${PYTHONPATH}:/code/bmi3d/:/code/bmi3d_tasks_analysis" - -RUN python config/make_config.py --use-defaults && - python db/manage.py makemigrations && - python db/manage.py migrate - - -CMD [ "/bin/bash", "./db/runserver.sh" ] diff --git a/bmi3d.python.dockerfile b/bmi3d.python.dockerfile new file mode 100644 index 00000000..ecc7d2e6 --- /dev/null +++ b/bmi3d.python.dockerfile @@ -0,0 +1,35 @@ +##### ################################################################### #### +##### ---- Starting point when python dependencies have been changed ---- #### +##### ################################################################### #### +FROM bmi3d:base + +####### Set up directories and copy source code +RUN mkdir -v -p /code/src/ +RUN mkdir -v -p /backup && chown root /backup +RUN mkdir -v -p /storage/plots && chown -R root /storage + +# --- Expect cache invalidation here if source files have changed --- # +COPY bmi3d/ /code/bmi3d/ +COPY bmi3d_tasks_analysis/ /code/bmi3d_tasks_analysis/ + +# Replace windows symlinks with unix ones in the new env +RUN rm /code/bmi3d/analysis && ln -s /code/bmi3d_tasks_analysis/analysis/ /code/bmi3d/analysis \ + && rm /code/bmi3d/tasks && ln -s /code/bmi3d_tasks_analysis/tasks/ /code/bmi3d/tasks + +# Fix all .sh files that might have aquired windows line endings +RUN for f in $(find ./code/ -name "*.sh"); \ + do echo "fixing: $f" && sed -i 's/\r$//' $f; \ + done + +WORKDIR /code/bmi3d/ +RUN mkdir -v log + + +###### Install python dependencies +RUN pip install --upgrade pip +RUN pip install -r requirements.txt + +# Set env vars for future reference +ENV BMI3D="/code/bmi3d" \ + PYTHONPATH="${PYTHONPATH}:/code/bmi3d/:/code/bmi3d_tasks_analysis" + From 6574f3072a4b650e0d39bb6c1f6705d171dfd502 Mon Sep 17 00:00:00 2001 From: Tomek Fraczek Date: Tue, 14 May 2019 18:55:20 -0700 Subject: [PATCH 16/22] Tmp fix of feature and task lists: use empty dicts on import error --- config/namelist.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/config/namelist.py b/config/namelist.py index de2c86c2..3f72c3d7 100644 --- a/config/namelist.py +++ b/config/namelist.py @@ -2,12 +2,17 @@ Lookup table for features, generators and tasks for experiments ''' -## Get the list of experiment features -from .featurelist import features - -## Get the list of tasks -from .tasklist import tasks +# Get the list of experiment features +try: + from .featurelist import features +except (ImportError, ModuleNotFoundError): + features = {} +# Get the list of tasks +try: + from .tasklist import tasks +except (ImportError, ModuleNotFoundError): + tasks = {} # Derive generator functions from the tasklist (all generatorfunctions should be staticmethods of a task) generator_names = [] @@ -67,4 +72,4 @@ def __getitem__(self, name): ################################################################################ ################################################################################ -from .bmilist import * \ No newline at end of file +# from .bmilist import * \ No newline at end of file From 97865c72049e8be069488d4f61e56fe219b5a047 Mon Sep 17 00:00:00 2001 From: Tomek Fraczek Date: Wed, 15 May 2019 10:33:22 -0700 Subject: [PATCH 17/22] Split docker images into three separate images to help with partial re-building --- bmi3d.base.dockerfile | 5 ++- bmi3d.python.dockerfile | 15 -------- install/docker/bmi3d.base.dockerfile | 51 ++++++++++++++++++++++++++ install/docker/bmi3d.code.dockerfile | 17 +++++++++ install/docker/bmi3d.python.dockerfile | 35 ++++++++++++++++++ 5 files changed, 107 insertions(+), 16 deletions(-) create mode 100644 install/docker/bmi3d.base.dockerfile create mode 100644 install/docker/bmi3d.code.dockerfile create mode 100644 install/docker/bmi3d.python.dockerfile diff --git a/bmi3d.base.dockerfile b/bmi3d.base.dockerfile index 35df5f29..67483763 100644 --- a/bmi3d.base.dockerfile +++ b/bmi3d.base.dockerfile @@ -46,6 +46,9 @@ RUN apt-get install -y --allow-unauthenticated \ erlang-xmerl \ rabbitmq-server - +####### Set up directories +RUN mkdir -v -p /code/src/ +RUN mkdir -v -p /backup && chown root /backup +RUN mkdir -v -p /storage/plots && chown -R root /storage diff --git a/bmi3d.python.dockerfile b/bmi3d.python.dockerfile index ecc7d2e6..3946fc4c 100644 --- a/bmi3d.python.dockerfile +++ b/bmi3d.python.dockerfile @@ -3,28 +3,13 @@ ##### ################################################################### #### FROM bmi3d:base -####### Set up directories and copy source code -RUN mkdir -v -p /code/src/ -RUN mkdir -v -p /backup && chown root /backup -RUN mkdir -v -p /storage/plots && chown -R root /storage - # --- Expect cache invalidation here if source files have changed --- # COPY bmi3d/ /code/bmi3d/ COPY bmi3d_tasks_analysis/ /code/bmi3d_tasks_analysis/ -# Replace windows symlinks with unix ones in the new env -RUN rm /code/bmi3d/analysis && ln -s /code/bmi3d_tasks_analysis/analysis/ /code/bmi3d/analysis \ - && rm /code/bmi3d/tasks && ln -s /code/bmi3d_tasks_analysis/tasks/ /code/bmi3d/tasks - -# Fix all .sh files that might have aquired windows line endings -RUN for f in $(find ./code/ -name "*.sh"); \ - do echo "fixing: $f" && sed -i 's/\r$//' $f; \ - done - WORKDIR /code/bmi3d/ RUN mkdir -v log - ###### Install python dependencies RUN pip install --upgrade pip RUN pip install -r requirements.txt diff --git a/install/docker/bmi3d.base.dockerfile b/install/docker/bmi3d.base.dockerfile new file mode 100644 index 00000000..35df5f29 --- /dev/null +++ b/install/docker/bmi3d.base.dockerfile @@ -0,0 +1,51 @@ +##### ################################################################### #### +##### ---- Starting point when any os dependencies have been changed ---- #### +##### ################################################################### #### +FROM python:3 + +#### Connect up third-party repositories (rabbitmq and erlang) +RUN curl -s https://packagecloud.io/install/repositories/rabbitmq/rabbitmq-server/script.deb.sh | bash +RUN echo "deb http://dl.bintray.com/rabbitmq-erlang/debian bionic erlang" \ + >> /etc/apt/sources.list.d/bintray.erlang.list +RUN apt-get -y update + + +##### Install required ubuntu packages +RUN apt-get install -y \ + smbclient \ + cifs-utils \ + bison \ + flex \ + openssh-server \ + libusb-dev\ + libcomedi-dev \ + python-comedilib \ + swig \ + isc-dhcp-server \ + sqlite3 \ + vim + +# Install rabbitmq with it's erlang dependencies +RUN apt-get install -y --allow-unauthenticated \ + erlang-base-hipe \ + erlang-asn1 \ + erlang-crypto \ + erlang-eldap \ + erlang-ftp \ + erlang-inets \ + erlang-mnesia \ + erlang-os-mon \ + erlang-parsetools \ + erlang-public-key \ + erlang-runtime-tools \ + erlang-snmp \ + erlang-ssl \ + erlang-syntax-tools \ + erlang-tftp \ + erlang-tools \ + erlang-xmerl \ + rabbitmq-server + + + + diff --git a/install/docker/bmi3d.code.dockerfile b/install/docker/bmi3d.code.dockerfile new file mode 100644 index 00000000..47ffabd2 --- /dev/null +++ b/install/docker/bmi3d.code.dockerfile @@ -0,0 +1,17 @@ +#### ##################################################################### ##### +#### ---- Starting point when only small local changes have been made ---- ##### +#### ##################################################################### ##### +FROM bmi3d:python +COPY bmi3d/ /code/bmi3d/ +COPY bmi3d_tasks_analysis/ /code/bmi3d_tasks_analysis/ + +RUN python config/make_config.py --use-defaults && \ + python db/manage.py makemigrations && \ + python db/manage.py migrate + +# Fix all .sh files that might have aquired windows line endings +RUN for f in $(find /code/ -name "*.sh"); \ + do echo "fixing: $f" && sed -i 's/\r$//' $f; \ + done + +CMD [ "/bin/bash", "./db/runserver.sh" ] \ No newline at end of file diff --git a/install/docker/bmi3d.python.dockerfile b/install/docker/bmi3d.python.dockerfile new file mode 100644 index 00000000..ecc7d2e6 --- /dev/null +++ b/install/docker/bmi3d.python.dockerfile @@ -0,0 +1,35 @@ +##### ################################################################### #### +##### ---- Starting point when python dependencies have been changed ---- #### +##### ################################################################### #### +FROM bmi3d:base + +####### Set up directories and copy source code +RUN mkdir -v -p /code/src/ +RUN mkdir -v -p /backup && chown root /backup +RUN mkdir -v -p /storage/plots && chown -R root /storage + +# --- Expect cache invalidation here if source files have changed --- # +COPY bmi3d/ /code/bmi3d/ +COPY bmi3d_tasks_analysis/ /code/bmi3d_tasks_analysis/ + +# Replace windows symlinks with unix ones in the new env +RUN rm /code/bmi3d/analysis && ln -s /code/bmi3d_tasks_analysis/analysis/ /code/bmi3d/analysis \ + && rm /code/bmi3d/tasks && ln -s /code/bmi3d_tasks_analysis/tasks/ /code/bmi3d/tasks + +# Fix all .sh files that might have aquired windows line endings +RUN for f in $(find ./code/ -name "*.sh"); \ + do echo "fixing: $f" && sed -i 's/\r$//' $f; \ + done + +WORKDIR /code/bmi3d/ +RUN mkdir -v log + + +###### Install python dependencies +RUN pip install --upgrade pip +RUN pip install -r requirements.txt + +# Set env vars for future reference +ENV BMI3D="/code/bmi3d" \ + PYTHONPATH="${PYTHONPATH}:/code/bmi3d/:/code/bmi3d_tasks_analysis" + From f12340990d4fbb9bc34f2bf40ab65a20baab4b78 Mon Sep 17 00:00:00 2001 From: Tomek Fraczek Date: Wed, 15 May 2019 13:13:45 -0700 Subject: [PATCH 18/22] Re-organize build code between dockerfiles --- bmi3d.base.dockerfile | 54 -------------------------- bmi3d.code.dockerfile | 17 -------- bmi3d.python.dockerfile | 20 ---------- install/docker/bmi3d.base.dockerfile | 5 ++- install/docker/bmi3d.python.dockerfile | 15 ------- 5 files changed, 4 insertions(+), 107 deletions(-) delete mode 100644 bmi3d.base.dockerfile delete mode 100644 bmi3d.code.dockerfile delete mode 100644 bmi3d.python.dockerfile diff --git a/bmi3d.base.dockerfile b/bmi3d.base.dockerfile deleted file mode 100644 index 67483763..00000000 --- a/bmi3d.base.dockerfile +++ /dev/null @@ -1,54 +0,0 @@ -##### ################################################################### #### -##### ---- Starting point when any os dependencies have been changed ---- #### -##### ################################################################### #### -FROM python:3 - -#### Connect up third-party repositories (rabbitmq and erlang) -RUN curl -s https://packagecloud.io/install/repositories/rabbitmq/rabbitmq-server/script.deb.sh | bash -RUN echo "deb http://dl.bintray.com/rabbitmq-erlang/debian bionic erlang" \ - >> /etc/apt/sources.list.d/bintray.erlang.list -RUN apt-get -y update - - -##### Install required ubuntu packages -RUN apt-get install -y \ - smbclient \ - cifs-utils \ - bison \ - flex \ - openssh-server \ - libusb-dev\ - libcomedi-dev \ - python-comedilib \ - swig \ - isc-dhcp-server \ - sqlite3 \ - vim - -# Install rabbitmq with it's erlang dependencies -RUN apt-get install -y --allow-unauthenticated \ - erlang-base-hipe \ - erlang-asn1 \ - erlang-crypto \ - erlang-eldap \ - erlang-ftp \ - erlang-inets \ - erlang-mnesia \ - erlang-os-mon \ - erlang-parsetools \ - erlang-public-key \ - erlang-runtime-tools \ - erlang-snmp \ - erlang-ssl \ - erlang-syntax-tools \ - erlang-tftp \ - erlang-tools \ - erlang-xmerl \ - rabbitmq-server - -####### Set up directories -RUN mkdir -v -p /code/src/ -RUN mkdir -v -p /backup && chown root /backup -RUN mkdir -v -p /storage/plots && chown -R root /storage - - diff --git a/bmi3d.code.dockerfile b/bmi3d.code.dockerfile deleted file mode 100644 index 47ffabd2..00000000 --- a/bmi3d.code.dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -#### ##################################################################### ##### -#### ---- Starting point when only small local changes have been made ---- ##### -#### ##################################################################### ##### -FROM bmi3d:python -COPY bmi3d/ /code/bmi3d/ -COPY bmi3d_tasks_analysis/ /code/bmi3d_tasks_analysis/ - -RUN python config/make_config.py --use-defaults && \ - python db/manage.py makemigrations && \ - python db/manage.py migrate - -# Fix all .sh files that might have aquired windows line endings -RUN for f in $(find /code/ -name "*.sh"); \ - do echo "fixing: $f" && sed -i 's/\r$//' $f; \ - done - -CMD [ "/bin/bash", "./db/runserver.sh" ] \ No newline at end of file diff --git a/bmi3d.python.dockerfile b/bmi3d.python.dockerfile deleted file mode 100644 index 3946fc4c..00000000 --- a/bmi3d.python.dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -##### ################################################################### #### -##### ---- Starting point when python dependencies have been changed ---- #### -##### ################################################################### #### -FROM bmi3d:base - -# --- Expect cache invalidation here if source files have changed --- # -COPY bmi3d/ /code/bmi3d/ -COPY bmi3d_tasks_analysis/ /code/bmi3d_tasks_analysis/ - -WORKDIR /code/bmi3d/ -RUN mkdir -v log - -###### Install python dependencies -RUN pip install --upgrade pip -RUN pip install -r requirements.txt - -# Set env vars for future reference -ENV BMI3D="/code/bmi3d" \ - PYTHONPATH="${PYTHONPATH}:/code/bmi3d/:/code/bmi3d_tasks_analysis" - diff --git a/install/docker/bmi3d.base.dockerfile b/install/docker/bmi3d.base.dockerfile index 35df5f29..67483763 100644 --- a/install/docker/bmi3d.base.dockerfile +++ b/install/docker/bmi3d.base.dockerfile @@ -46,6 +46,9 @@ RUN apt-get install -y --allow-unauthenticated \ erlang-xmerl \ rabbitmq-server - +####### Set up directories +RUN mkdir -v -p /code/src/ +RUN mkdir -v -p /backup && chown root /backup +RUN mkdir -v -p /storage/plots && chown -R root /storage diff --git a/install/docker/bmi3d.python.dockerfile b/install/docker/bmi3d.python.dockerfile index ecc7d2e6..3946fc4c 100644 --- a/install/docker/bmi3d.python.dockerfile +++ b/install/docker/bmi3d.python.dockerfile @@ -3,28 +3,13 @@ ##### ################################################################### #### FROM bmi3d:base -####### Set up directories and copy source code -RUN mkdir -v -p /code/src/ -RUN mkdir -v -p /backup && chown root /backup -RUN mkdir -v -p /storage/plots && chown -R root /storage - # --- Expect cache invalidation here if source files have changed --- # COPY bmi3d/ /code/bmi3d/ COPY bmi3d_tasks_analysis/ /code/bmi3d_tasks_analysis/ -# Replace windows symlinks with unix ones in the new env -RUN rm /code/bmi3d/analysis && ln -s /code/bmi3d_tasks_analysis/analysis/ /code/bmi3d/analysis \ - && rm /code/bmi3d/tasks && ln -s /code/bmi3d_tasks_analysis/tasks/ /code/bmi3d/tasks - -# Fix all .sh files that might have aquired windows line endings -RUN for f in $(find ./code/ -name "*.sh"); \ - do echo "fixing: $f" && sed -i 's/\r$//' $f; \ - done - WORKDIR /code/bmi3d/ RUN mkdir -v log - ###### Install python dependencies RUN pip install --upgrade pip RUN pip install -r requirements.txt From daffa0ddd84e8a2a8e986648842717c5a2d03756 Mon Sep 17 00:00:00 2001 From: Tomek Fraczek Date: Fri, 17 May 2019 17:00:24 -0700 Subject: [PATCH 19/22] Actually remove OpenGL-accelerate --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index eb0b5180..a07bb2eb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,6 @@ pandas patsy statsmodels PyOpenGL -PyOpenGL-accelerate Django pylibftdi nitime From 175531e93c5338dda93450b1d77b5c912e64c6d8 Mon Sep 17 00:00:00 2001 From: Tomek Fraczek Date: Fri, 17 May 2019 17:03:14 -0700 Subject: [PATCH 20/22] Preven clearing of logs on python level re-build --- install/docker/bmi3d.base.dockerfile | 3 ++- install/docker/bmi3d.python.dockerfile | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/install/docker/bmi3d.base.dockerfile b/install/docker/bmi3d.base.dockerfile index 67483763..bd363a8c 100644 --- a/install/docker/bmi3d.base.dockerfile +++ b/install/docker/bmi3d.base.dockerfile @@ -49,6 +49,7 @@ RUN apt-get install -y --allow-unauthenticated \ ####### Set up directories RUN mkdir -v -p /code/src/ RUN mkdir -v -p /backup && chown root /backup -RUN mkdir -v -p /storage/plots && chown -R root /storage +RUN mkdir -v -p /storage/plots && chown -R root /storage\ +RUN mkdir -v -p /code/bmi3d/log diff --git a/install/docker/bmi3d.python.dockerfile b/install/docker/bmi3d.python.dockerfile index 3946fc4c..aadd37d6 100644 --- a/install/docker/bmi3d.python.dockerfile +++ b/install/docker/bmi3d.python.dockerfile @@ -8,7 +8,6 @@ COPY bmi3d/ /code/bmi3d/ COPY bmi3d_tasks_analysis/ /code/bmi3d_tasks_analysis/ WORKDIR /code/bmi3d/ -RUN mkdir -v log ###### Install python dependencies RUN pip install --upgrade pip From 68ceee156b12159625a90640901c264443e9c0ca Mon Sep 17 00:00:00 2001 From: Tomek Fraczek Date: Fri, 17 May 2019 17:04:16 -0700 Subject: [PATCH 21/22] Prevent pip install if requirements have not changed --- install/docker/bmi3d.python.dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/install/docker/bmi3d.python.dockerfile b/install/docker/bmi3d.python.dockerfile index aadd37d6..ad6e49f6 100644 --- a/install/docker/bmi3d.python.dockerfile +++ b/install/docker/bmi3d.python.dockerfile @@ -4,8 +4,7 @@ FROM bmi3d:base # --- Expect cache invalidation here if source files have changed --- # -COPY bmi3d/ /code/bmi3d/ -COPY bmi3d_tasks_analysis/ /code/bmi3d_tasks_analysis/ +COPY bmi3d/requirements.txt /code/bmi3d/requirements.txt WORKDIR /code/bmi3d/ From d2cb2acca6f15dc9f8a07dbc119a02f72dcaf1b8 Mon Sep 17 00:00:00 2001 From: Tomek Fraczek Date: Mon, 20 May 2019 11:07:01 -0700 Subject: [PATCH 22/22] Removed redundant installs of plotutils and pygame --- install/docker/src_code_install.sh | 7 ------- 1 file changed, 7 deletions(-) diff --git a/install/docker/src_code_install.sh b/install/docker/src_code_install.sh index 641465f8..e53845c2 100644 --- a/install/docker/src_code_install.sh +++ b/install/docker/src_code_install.sh @@ -5,10 +5,7 @@ BMI3D=$CODE/bmi3d ### Directory in which to install the bmi3d software USER=root # We're in a docker container so root is safe ####### Download any src code -git clone https://github.com/sgowda/plot $HOME/code/plotutil git clone https://github.com/sgowda/robotics_toolbox $HOME/code/robotics -# pygame -hg clone https://bitbucket.org/pygame/pygame $HOME/code/pygame # Phidgets code #wget https://www.phidgets.com/downloads/phidget22/libraries/linux/libphidget22/libphidget22-1.1.20190417.tar.gz #wget https://www.phidgets.com/downloads/phidget22/libraries/any/Phidget22Python/Phidget22Python_1.1.20190418.zip @@ -20,10 +17,6 @@ hg clone https://bitbucket.org/pygame/pygame $HOME/code/pygame which python #python setup.py install -# pygame -cd $HOME/code/pygame -python setup.py install - # NIDAQ software -- deprecated! # $HOME/code/bmi3d/riglib/nidaq/build.sh