From c44050edb67428009c86a1055237185d5feca648 Mon Sep 17 00:00:00 2001 From: Ming Rutar Date: Fri, 16 Jun 2017 15:09:20 -0700 Subject: [PATCH 1/7] added --vid_mapping_file and --version option --- docker/vcf_combiner/Dockerfile | 9 +- docker/vcf_combiner/usr/bin/combine_vcf.py | 85 +++++++++---------- .../vcf_combiner/usr/bin/container-entrypoint | 18 +++- 3 files changed, 66 insertions(+), 46 deletions(-) diff --git a/docker/vcf_combiner/Dockerfile b/docker/vcf_combiner/Dockerfile index 8558f572..105cb920 100644 --- a/docker/vcf_combiner/Dockerfile +++ b/docker/vcf_combiner/Dockerfile @@ -1,5 +1,5 @@ # The MIT License (MIT) -# Copyright (c) 2016 Intel Corporation +# Copyright (c) 2017 Intel Corporation # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in @@ -41,4 +41,11 @@ WORKDIR /tmp ENV BASH_ENV=/etc/profile.d/cont-env.sh HOME=/home/default PATH=$PATH:/usr/lib64/mpich/bin +RUN groupadd -r default -f -g 5658 && \ + useradd -u 5658 -g default -o -c "Default User" default -s /sbin/nologin + +#USER default + +ENTRYPOINT ["/usr/bin/container-entrypoint"] + CMD ["container-usage"] diff --git a/docker/vcf_combiner/usr/bin/combine_vcf.py b/docker/vcf_combiner/usr/bin/combine_vcf.py index 77679276..a0f8bcf3 100755 --- a/docker/vcf_combiner/usr/bin/combine_vcf.py +++ b/docker/vcf_combiner/usr/bin/combine_vcf.py @@ -2,25 +2,25 @@ # pylint: disable=missing-docstring, invalid-name, broad-except, too-many-branches, too-many-locals, line-too-long """ - * The MIT License (MIT) - * Copyright (c) 2016 Intel Corporation - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of - * this software and associated documentation files (the "Software"), to deal in - * the Software without restriction, including without limitation the rights to - * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of - * the Software, and to permit persons to whom the Software is furnished to do so, - * subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS - * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR - * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER - * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + The MIT License (MIT) + Copyright (c) 2017 Intel Corporation + + Permission is hereby granted, free of charge, to any person obtaining a copy of + this software and associated documentation files (the "Software"), to deal in + the Software without restriction, including without limitation the rights to + use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + the Software, and to permit persons to whom the Software is furnished to do so, + subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ import sys import os @@ -41,7 +41,6 @@ COL_PARTITION_SIZE_UNIT = 16384 DefaultVIDFile = "/usr/share/cont-intel/vid.json" -DefaultVIDEnv = 'HOST_VID_PATH' # to run the script at host, specify the path to a vid file logging.basicConfig(stream=sys.stdout, level=logging.INFO) ##### loader config file tags @@ -58,7 +57,7 @@ "offload_vcf_output_processing" : True, "produce_GT_field" : False, "size_per_column_partition" : COL_PARTITION_SIZE_UNIT, - "vid_mapping_file": "" + "vid_mapping_file": '' } cp_pos = namedtuple('pos_only', "begin, vcf_output_filename") cp_chr = namedtuple('chromosome', 'begin, end, vcf_output_filename') @@ -83,17 +82,15 @@ class CombineVCF(object): ''' VCF file combiner ''' logger = logging.getLogger("CombineVCF") brief_options = "i:o:R:c:p" - full_options = ['samples=', 'output=', 'reference=', 'callsets=', 'produce_GT_field', \ - 'chromosome=', 'begin=', 'end=', 'dryrun', 'version'] + full_options = ['samples=', 'output=', 'reference=', 'callsets=', 'vid_mapping_file=', \ + 'produce_GT_field', 'chromosome=', 'begin=', 'end=', 'dryrun'] def __init__(self): self.dryrun = False self.output_file = None + self.vid_mapping_file = DefaultVIDFile def _parse_args(self, args): - def get_vid_mapping_file(): - return DefaultVIDFile if os.path.isfile(DefaultVIDFile) else os.environ[DefaultVIDEnv] - vmf_args = {"vid_mapping_file": get_vid_mapping_file} def check_chromosome(): assert begin, 'No begin position is given' if end: @@ -106,14 +103,12 @@ def check_chromosome(): chromosome = None begin = None end = None + vid_mapping_file = DefaultVIDFile for opt, user_input in myopts: if opt == '-p' or opt == '--produce_GT_field': produce_GT_field = True elif opt == '--dryrun': self.dryrun = True - elif opt == '--version': - print(self.get_version()) - exit() else: assert user_input, 'specify a value for option %s' % opt if opt == '-i' or opt == '--samples': @@ -128,6 +123,9 @@ def check_chromosome(): assert os.path.isfile(user_input), "specify a valid callset file name" callset_mapping_file = user_input num_part_units = self.__check_callset(callset_mapping_file) + elif opt == '--vid_mapping_file': + assert os.path.isfile(user_input), "specify a valid vid mapping file" + vid_mapping_file = user_input elif opt == '--chromosome': chromosome = user_input elif opt == '--begin': @@ -135,8 +133,7 @@ def check_chromosome(): elif opt == '--end': end = int(user_input) else: - print(self.get_version()) - exit() + print("WARN: unknown option %s, ignored", opt) if not callset_mapping_file: callset_mapping_file = "callsets_%s.json" % datetime.now().strftime("%y%m%d%H%M") @@ -148,12 +145,13 @@ def check_chromosome(): col_par_setting = check_chromosome() if chromosome \ else get_col_partition(self.output_file, begin if begin else 0) - loader_cfg = get_loader_cfg(**vmf_args) + loader_cfg = get_loader_cfg() loader_cfg = loader_cfg._replace(reference_genome=reference_genome, \ vcf_output_filename=self.output_file, \ column_partitions=[col_par_setting], \ size_per_column_partition=int(abs(num_part_units)) * COL_PARTITION_SIZE_UNIT, \ callset_mapping_file=callset_mapping_file, \ + vid_mapping_file=vid_mapping_file, \ produce_GT_field=True if produce_GT_field else False) return loader_cfg @@ -214,9 +212,9 @@ def __get_inputs(self, inputfiles): inputs = [line.strip() for line in inputfiles if os.path.isfile(line)] return inputs if inputs else RuntimeError("No valid samples input files found") - @staticmethod - def generate_loader_config(nt_loader): - json_fname = "lc_%s" % datetime.now().strftime("%y%m%d%H%M") + def generate_loader_config(self, nt_loader): + json_fname = os.path.join(os.path.dirname(self.output_file), \ + "loader_config_%s.json" % datetime.now().strftime("%y%m%d%H%M")) with open(json_fname, 'w') as ofd: json.dump(nt_loader._asdict(), ofd) return json_fname @@ -254,18 +252,19 @@ def run(self): raise CombineVCFException("Failed to combining VCF files: %s" % (err)) @staticmethod - def get_version(): - return "%s %s" % (ProductName, Version) + def get_my_name(): + return ProductName def test_quick_test(): ''' a quick test for github check-in ''' - x = CombineVCF().get_version() + print("Run a quick test ...") + x = CombineVCF().get_my_name() assert x - x = x.split() - assert len(x) == 2 - x = x[1] - assert x == Version + assert x == ProductName if __name__ == "__main__": combiner = CombineVCF() - combiner.run() + if len(sys.argv) > 1: + combiner.run() + else: + combiner.get_my_name() diff --git a/docker/vcf_combiner/usr/bin/container-entrypoint b/docker/vcf_combiner/usr/bin/container-entrypoint index a26a32bc..cd3a5982 100755 --- a/docker/vcf_combiner/usr/bin/container-entrypoint +++ b/docker/vcf_combiner/usr/bin/container-entrypoint @@ -1,10 +1,24 @@ #!/bin/bash +if [ "$#" -gt 0 -a "$1" == "--version" ]; then + echo "INFO CONTAINER: vcf2tiledb version is $(vcf2tiledb --version)" + exit 0 +fi +my_uid=$(id -u) source /usr/share/cont-lib/cont-lib.sh - cont_debug "command: $*" - __cont_source_scripts "/usr/share/cont-entry" test -z "$*" && set -- bash +if [ "$1" == "combine_vcf" ]; then + echo "INFO CONTAINER: run combine_vcf as user id $my_uid, vcf2tiledb version is $(vcf2tiledb --version)" + echo +fi + +if [ $my_uid == 5658 ]; then + cd $HOME +else + export HOME=/ +fi + exec "$@" From fd47f0569bc7e83be24d869ac66f6799047e4ea6 Mon Sep 17 00:00:00 2001 From: Ming Rutar Date: Fri, 16 Jun 2017 16:00:04 -0700 Subject: [PATCH 2/7] remove white spaces --- docker/vcf_combiner/usr/bin/combine_vcf.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/vcf_combiner/usr/bin/combine_vcf.py b/docker/vcf_combiner/usr/bin/combine_vcf.py index a0f8bcf3..2e5bfce6 100755 --- a/docker/vcf_combiner/usr/bin/combine_vcf.py +++ b/docker/vcf_combiner/usr/bin/combine_vcf.py @@ -4,17 +4,17 @@ """ The MIT License (MIT) Copyright (c) 2017 Intel Corporation - + Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - + The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR From 763a3737bc09067d674b7dcb4bddd789f938b927 Mon Sep 17 00:00:00 2001 From: Ming Rutar Date: Mon, 19 Jun 2017 15:23:15 -0700 Subject: [PATCH 3/7] refactory --- docker/vcf_combiner/usr/bin/container-entrypoint | 13 +++++++------ .../usr/share/cont-lib/autoload/genomicsdb-env.sh | 6 +++++- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/docker/vcf_combiner/usr/bin/container-entrypoint b/docker/vcf_combiner/usr/bin/container-entrypoint index cd3a5982..10fe7bd0 100755 --- a/docker/vcf_combiner/usr/bin/container-entrypoint +++ b/docker/vcf_combiner/usr/bin/container-entrypoint @@ -1,17 +1,18 @@ #!/bin/bash +. /usr/share/cont-lib/cont-lib.sh + +ls /usr/bin/vcf2tiledb > /dev/null 2>&1 +test $? -ne 0 && cont_error "cannot find vcf2tiledb... exiting" && exit 2 + if [ "$#" -gt 0 -a "$1" == "--version" ]; then - echo "INFO CONTAINER: vcf2tiledb version is $(vcf2tiledb --version)" + __cont_msg "info vcf2tiledb version is $(vcf2tiledb --version)" exit 0 fi my_uid=$(id -u) -source /usr/share/cont-lib/cont-lib.sh -cont_debug "command: $*" -__cont_source_scripts "/usr/share/cont-entry" - test -z "$*" && set -- bash if [ "$1" == "combine_vcf" ]; then - echo "INFO CONTAINER: run combine_vcf as user id $my_uid, vcf2tiledb version is $(vcf2tiledb --version)" + __cont_msg "info run combine_vcf as user id $my_uid, vcf2tiledb version is $(vcf2tiledb --version)" echo fi diff --git a/docker/vcf_combiner/usr/share/cont-lib/autoload/genomicsdb-env.sh b/docker/vcf_combiner/usr/share/cont-lib/autoload/genomicsdb-env.sh index bbe1fe83..c79945ca 100644 --- a/docker/vcf_combiner/usr/share/cont-lib/autoload/genomicsdb-env.sh +++ b/docker/vcf_combiner/usr/share/cont-lib/autoload/genomicsdb-env.sh @@ -1,2 +1,6 @@ -export HOME=/home/default +if [ $(id -u) == 5658 ]; then + export HOME=/home/default +else + export HOME=/ +fi export PATH=$PATH:/usr/lib64/mpich/bin From ff3903117b86c16cd2ddd3884665369073e895ff Mon Sep 17 00:00:00 2001 From: Ming Rutar Date: Tue, 20 Jun 2017 12:55:13 -0700 Subject: [PATCH 4/7] addressed review issues --- docker/vcf_combiner/usr/bin/combine_vcf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/vcf_combiner/usr/bin/combine_vcf.py b/docker/vcf_combiner/usr/bin/combine_vcf.py index 2e5bfce6..0bdd4bfa 100755 --- a/docker/vcf_combiner/usr/bin/combine_vcf.py +++ b/docker/vcf_combiner/usr/bin/combine_vcf.py @@ -3,7 +3,7 @@ """ The MIT License (MIT) - Copyright (c) 2017 Intel Corporation + Copyright (c) 2016-2017 Intel Corporation Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in @@ -255,7 +255,7 @@ def run(self): def get_my_name(): return ProductName -def test_quick_test(): +def test_code_runs_after_pylint(): ''' a quick test for github check-in ''' print("Run a quick test ...") x = CombineVCF().get_my_name() From 7063c4c1c5a065a75b0cebb68cf713c3e32245af Mon Sep 17 00:00:00 2001 From: Ming Rutar Date: Tue, 20 Jun 2017 13:52:36 -0700 Subject: [PATCH 5/7] Docker source for GenomicsDB builder --- .travis.yml | 1 + docker/genomicsdb_builder/.dockerignore | 2 + docker/genomicsdb_builder/Dockerfile | 45 +++++++++ docker/genomicsdb_builder/README.md | 11 +++ docker/genomicsdb_builder/root/.bashrc | 1 + .../usr/bin/build_genomicsdb | 99 +++++++++++++++++++ .../usr/bin/container-entrypoint | 10 ++ .../usr/bin/container-usage | 6 ++ .../usr/share/cont-docs/70-general.txt | 20 ++++ .../usr/share/cont-docs/README | 1 + .../usr/share/cont-entry/cont-env.sh | 1 + .../common/env/enabledevtoolset-4.sh | 3 + .../usr/share/cont-lib/autoload/README | 2 + .../usr/share/cont-lib/cont-env.sh | 6 ++ .../usr/share/cont-lib/cont-lib.sh | 97 ++++++++++++++++++ .../share/cont-lib/parser-simple-config.sh | 83 ++++++++++++++++ docker/requirements.txt | 3 - 17 files changed, 388 insertions(+), 3 deletions(-) create mode 100644 docker/genomicsdb_builder/.dockerignore create mode 100644 docker/genomicsdb_builder/Dockerfile create mode 100644 docker/genomicsdb_builder/README.md create mode 100644 docker/genomicsdb_builder/root/.bashrc create mode 100755 docker/genomicsdb_builder/usr/bin/build_genomicsdb create mode 100755 docker/genomicsdb_builder/usr/bin/container-entrypoint create mode 100755 docker/genomicsdb_builder/usr/bin/container-usage create mode 100644 docker/genomicsdb_builder/usr/share/cont-docs/70-general.txt create mode 100644 docker/genomicsdb_builder/usr/share/cont-docs/README create mode 100644 docker/genomicsdb_builder/usr/share/cont-entry/cont-env.sh create mode 100755 docker/genomicsdb_builder/usr/share/cont-layer/common/env/enabledevtoolset-4.sh create mode 100644 docker/genomicsdb_builder/usr/share/cont-lib/autoload/README create mode 100644 docker/genomicsdb_builder/usr/share/cont-lib/cont-env.sh create mode 100644 docker/genomicsdb_builder/usr/share/cont-lib/cont-lib.sh create mode 100644 docker/genomicsdb_builder/usr/share/cont-lib/parser-simple-config.sh delete mode 100644 docker/requirements.txt diff --git a/.travis.yml b/.travis.yml index 54c2f5e1..d446bcd4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -48,6 +48,7 @@ before_script: script: - pylint $TRAVIS_BUILD_DIR/docker/vcf_combiner/usr/bin/combine_vcf.py - pytest $TRAVIS_BUILD_DIR/docker/vcf_combiner/usr/bin/combine_vcf.py + - bash -n $TRAVIS_BUILD_DIR/docker/genomicsdb_builder/usr/bin/build_genomicsdb - cd $GENOMICSDB_BUILD_DIR - cmake $TRAVIS_BUILD_DIR -DBUILD_JAVA=1 -DCMAKE_BUILD_TYPE=Coverage -DCMAKE_INSTALL_PREFIX=$GENOMICSDB_INSTALL_DIR -DLIBCSV_DIR=$TRAVIS_BUILD_DIR/dependencies/libcsv -DGENOMICSDB_RELEASE_VERSION=$GENOMICSDB_RELEASE_VERSION -DMAVEN_QUIET=True - ln -s $TRAVIS_BUILD_DIR/tests diff --git a/docker/genomicsdb_builder/.dockerignore b/docker/genomicsdb_builder/.dockerignore new file mode 100644 index 00000000..b084cee3 --- /dev/null +++ b/docker/genomicsdb_builder/.dockerignore @@ -0,0 +1,2 @@ +# no build +/build_src/ diff --git a/docker/genomicsdb_builder/Dockerfile b/docker/genomicsdb_builder/Dockerfile new file mode 100644 index 00000000..97c59c34 --- /dev/null +++ b/docker/genomicsdb_builder/Dockerfile @@ -0,0 +1,45 @@ +# The MIT License (MIT) +# Copyright (c) 2017 Intel Corporation +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of +# this software and associated documentation files (the "Software"), to deal in +# the Software without restriction, including without limitation the rights to +# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +# the Software, and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +# The code utilized some features of RedHat pre-built container + +FROM centos:7 + +LABEL vendor="Intel Inc." name="GenomicsDB Builder" version="1.0" description="Build GenomicsDB" + +RUN yum install -y --setopt=tsflags=nodocs centos-release-scl && \ + yum-config-manager --enable rhel-server-rhscl-7-rpms && \ + yum install -y devtoolset-4 && \ + yum install -y --setopt=tsflags=nodocs epel-release && \ + yum repolist && \ + yum install -y --setopt=tsflags=nodocs cmake git.x86_64 libcsv libcsv-devel mpich-devel openssl-devel zlib-devel unzip.x86_64 && \ + yum install -y python34.x86_64 && \ + yum clean all + +ENV BASH_ENV=/etc/profile.d/cont-env.sh PATH=$PATH:/usr/lib64/mpich/bin HOME=/home/default + +WORKDIR /home/default + +ADD ./usr /usr +ADD ./etc /etc +ADD ./root /root + +ENTRYPOINT ["/usr/bin/container-entrypoint"] +CMD ["build_genomicsdb"] diff --git a/docker/genomicsdb_builder/README.md b/docker/genomicsdb_builder/README.md new file mode 100644 index 00000000..95b51081 --- /dev/null +++ b/docker/genomicsdb_builder/README.md @@ -0,0 +1,11 @@ +### GenomicsDB Building Docker + +The centos-build is the source code for building a GenomicsDB building environment Docker image. The details about GenomicsDB building environment can be found at [GenomicsDB Wiki]( https://github.com/Intel-HLS/GenomicsDB/wiki/Compiling-GenomicsDB#building.) + +To create a GenomicsDB building environment Docker image, run the following command: + +docker build -t name_of_the_builder_image --no-cache . + +Once the image is created. You can build the latest GenomicsDB executables by running: + +docker run -it -v /path/2/output/:/output/ [-e GDB_BRANCH=branch_name_default_is_master] name_of_the_builder_image diff --git a/docker/genomicsdb_builder/root/.bashrc b/docker/genomicsdb_builder/root/.bashrc new file mode 100644 index 00000000..01a215ef --- /dev/null +++ b/docker/genomicsdb_builder/root/.bashrc @@ -0,0 +1 @@ +source /usr/share/cont-lib/cont-env.sh diff --git a/docker/genomicsdb_builder/usr/bin/build_genomicsdb b/docker/genomicsdb_builder/usr/bin/build_genomicsdb new file mode 100755 index 00000000..b21c7c27 --- /dev/null +++ b/docker/genomicsdb_builder/usr/bin/build_genomicsdb @@ -0,0 +1,99 @@ +#! /bin/bash +# The MIT License (MIT) +# Copyright (c) 2017 Intel Corporation +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of +# this software and associated documentation files (the "Software"), to deal in +# the Software without restriction, including without limitation the rights to +# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +# the Software, and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +if [ ! -d /output ]; then + echo 'can not find /output/ exit...' + return 2 +fi +export BUILD_ROOT=${HOME}/build_src +export PROTOBUF_LIBRARY=$BUILD_ROOT/protobuf_build +echo "build_home=$build_home, PROTOBUF_LIBRARY=$PROTOBUF_LIBRARY, GenomicsDB_HOME=$GenomicsDB_HOME" + +protobuf_to_dir=/output +genomicsdb_to_dir=/output + +# make protobuf +build_proto_buf() { + echo "+++ Building protobuf at ${PROTOBUF_LIBRARY}..." + mkdir -p /output/protobuf + mkdir -p ${PROTOBUF_LIBRARY} && pushd ${PROTOBUF_LIBRARY} >/dev/null 2>&1 + git clone https://github.com/google/protobuf.git + cd protobuf + git checkout 3.0.x + ./autogen.sh + ./configure --prefix=$protobuf_to_dir --with-pic + if [ -f ./Makefile ]; then + make && make install + basename $(ls $protobuf_to_dir/bin/protoc) + find $protobuf_to_dir/lib/ -name 'libproto*' -type f -exec basename {} \; + echo "--- Done building protobuf" + popd >/dev/null 2>&1 + return 0 + else + popd >/dev/null 2>&1 + echo "ERROR: build_proto_buf not find Makefile" + return -1 + fi +} + +# make genomicsdb +build_gdb() { + echo + echo "+++ Building GenomicsDB at ${GenomicsDB_HOME}..." + git clone --recursive https://github.com/Intel-HLS/GenomicsDB.git + ws=GenomicsDB/build + mkdir -p $ws && pushd $ws >/dev/null 2>&1 + branch=${GDB_BRANCH:=master} + git checkout $branch + git branch + if [ $# -gt 0 ]; then + cmake --warn-uninitialized --debug-output .. -DCMAKE_INSTALL_PREFIX=$genomicsdb_to_dir -DPROTOBUF_LIBRARY=$protobuf_to_dir $@ + else + cmake --warn-uninitialized --debug-output .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$genomicsdb_to_dir -DDO_PROFILING=False -DPROTOBUF_LIBRARY=$protobuf_to_dir -DDISABLE_OPENMP=True -DBUILD_JAVA=False -DDO_PROFILING=False + fi + + if [ -f ./Makefile ]; then + make && make install + echo "INFO: Successfully built GenomicsDB... run test" + ../tests/run.py $PWD $genomicsdb_to_dir + popd >/dev/null 2>&1 + return 0 + else + echo "ERROR: build_gdb not find Makefile" + popd >/dev/null 2>&1 + return -1 + fi +} + +#source /opt/rh/devtoolset-4/enable +gcc --version + +mkdir -p ${BUILD_ROOT} && pushd ${BUILD_ROOT} >/dev/null 2>&1 +build_proto_buf +retst=$? +if [ $retst -eq 0 ]; then + build_gdb $@ + retst=$? + rsync -az $genomicsdb_to_dir /usr +fi +[[ $retst -eq 0 ]] && echo "DONE: built GenomicsDB" || echo "FAIL: cannot build GenomicsDB" +popd >/dev/null 2>&1 +return $retst diff --git a/docker/genomicsdb_builder/usr/bin/container-entrypoint b/docker/genomicsdb_builder/usr/bin/container-entrypoint new file mode 100755 index 00000000..d21b6fa0 --- /dev/null +++ b/docker/genomicsdb_builder/usr/bin/container-entrypoint @@ -0,0 +1,10 @@ +#!/bin/bash + +. /usr/share/cont-lib/cont-lib.sh + +cont_debug "command: $*" + +__cont_source_scripts "/usr/share/cont-entry" + +test -z "$*" && set -- bash +exec "$@" diff --git a/docker/genomicsdb_builder/usr/bin/container-usage b/docker/genomicsdb_builder/usr/bin/container-usage new file mode 100755 index 00000000..062e5200 --- /dev/null +++ b/docker/genomicsdb_builder/usr/bin/container-usage @@ -0,0 +1,6 @@ +#!/bin/bash + +# TODO: support API for executable help scripts +cat /usr/share/cont-docs/*.txt 2>/dev/null + + diff --git a/docker/genomicsdb_builder/usr/share/cont-docs/70-general.txt b/docker/genomicsdb_builder/usr/share/cont-docs/70-general.txt new file mode 100644 index 00000000..4fa749cc --- /dev/null +++ b/docker/genomicsdb_builder/usr/share/cont-docs/70-general.txt @@ -0,0 +1,20 @@ +General container help +---------------------- + +Run `docker run THIS_IMAGE container-usage` to get this help. + +Run `docker run -ti THIS_IMAGE bash` to obtain interactive shell. + +Run `docker exec -ti CONTAINERID container-entrypoint` to access already running container. + +In order to get the container ID after running the image, pass `--cidfile=` +option to the `docker run` command. That will instruct Docker to write +a file with the container ID. + +You may try `-e CONT_DEBUG=VAL` with VAL up to 3 to get more verbose debugging +info. + + +Report bugs to . + + diff --git a/docker/genomicsdb_builder/usr/share/cont-docs/README b/docker/genomicsdb_builder/usr/share/cont-docs/README new file mode 100644 index 00000000..511e0c9b --- /dev/null +++ b/docker/genomicsdb_builder/usr/share/cont-docs/README @@ -0,0 +1 @@ +Files '*.txt' are automatically read and added to 'container-usage' output. diff --git a/docker/genomicsdb_builder/usr/share/cont-entry/cont-env.sh b/docker/genomicsdb_builder/usr/share/cont-entry/cont-env.sh new file mode 100644 index 00000000..01a215ef --- /dev/null +++ b/docker/genomicsdb_builder/usr/share/cont-entry/cont-env.sh @@ -0,0 +1 @@ +source /usr/share/cont-lib/cont-env.sh diff --git a/docker/genomicsdb_builder/usr/share/cont-layer/common/env/enabledevtoolset-4.sh b/docker/genomicsdb_builder/usr/share/cont-layer/common/env/enabledevtoolset-4.sh new file mode 100755 index 00000000..3677466c --- /dev/null +++ b/docker/genomicsdb_builder/usr/share/cont-layer/common/env/enabledevtoolset-4.sh @@ -0,0 +1,3 @@ +#!/bin/bash +# M:R: this file was part of Red Hat samples +source scl_source enable devtoolset-4 diff --git a/docker/genomicsdb_builder/usr/share/cont-lib/autoload/README b/docker/genomicsdb_builder/usr/share/cont-lib/autoload/README new file mode 100644 index 00000000..b52db739 --- /dev/null +++ b/docker/genomicsdb_builder/usr/share/cont-lib/autoload/README @@ -0,0 +1,2 @@ +All '*.sh' files in this directory will be automatically sourced together with +cont-lib.sh script. diff --git a/docker/genomicsdb_builder/usr/share/cont-lib/cont-env.sh b/docker/genomicsdb_builder/usr/share/cont-lib/cont-env.sh new file mode 100644 index 00000000..74682e1c --- /dev/null +++ b/docker/genomicsdb_builder/usr/share/cont-lib/cont-env.sh @@ -0,0 +1,6 @@ +. /usr/share/cont-lib/cont-lib.sh + +cont_debug "changing environment variables" + +cont_source_hooks env common + diff --git a/docker/genomicsdb_builder/usr/share/cont-lib/cont-lib.sh b/docker/genomicsdb_builder/usr/share/cont-lib/cont-lib.sh new file mode 100644 index 00000000..8b76a382 --- /dev/null +++ b/docker/genomicsdb_builder/usr/share/cont-lib/cont-lib.sh @@ -0,0 +1,97 @@ +__cont_source_scripts() +{ + local i + local dir="$1" + for i in "$dir"/*.sh; do + if test -r "$i"; then + . "$i" + fi + done +} + + +# CONT_SOURCE_HOOKS HOOKDIR [PROJECT] +# ----------------------------------- +# Source '*.sh' files from the following directories (in this order): +# a. /usr/share/cont-layer/PROJECT/HOOK/ +# b. /usr/share/cont-volume/PROJECT/HOOK/ +# +# The PROJECT argument is optional because it may be set globally by +# $CONT_PROJECT environment variable. The need for PROJECT argument is +# basically to push people to install script into theirs own directories, +# which will allow easier multi-project containers maintenance. +cont_source_hooks() +{ + local i dir + local hook="$1" + local project="$CONT_PROJECT" + local dir + + test -z "$hook" && return + test -n "$2" && project="$2" + + for dir in /usr/share/cont-layer /usr/share/cont-volume; do + dir="$dir/$project/$hook" + cont_debug2 "loading scripts from $dir" + __cont_source_scripts "$dir" + done +} + +__cont_msg() +{ + echo "$*" >&2 +} + + +__cont_dbg() +{ + test -z "$CONT_DEBUG" && CONT_DEBUG=0 + test "$CONT_DEBUG" -lt "$1" && return + local lvl="$1" + shift + __cont_msg "debug_$lvl: $*" +} + + +cont_warn() { __cont_msg "warn: $*" ; } +cont_error() { __cont_msg "error: $*"; } +cont_debug() { __cont_dbg 1 "$*" ; } +cont_debug2() { __cont_dbg 2 "$*" ; } +cont_debug3() { __cont_dbg 3 "$*" ; } + + +__cont_encode_env() +{ + local i + for i in $1 + do + eval local val="\$$i" + printf ": \${%s=%q}\n" "$i" "$val" + done +} + + +# CONT_STORE_ENV VARIABLES FILENAME +# --------------------------------- +# Create source-able script conditionally setting specified VARIABLES by +# inheritting the values from current environment; Create the file on path +# FILENAME. Already existing variables will not be changed by sourcing the +# resulting script. The argument VARIABLES expects list of space separated +# variable names. +# +# Usage: +# $ my_var=my_value +# $ my_var2="my value2" +# $ cont_store_env "my_var my_var2" ~/.my-environment +# $ cat ~/.my-environment +# : ${my_var=my_value} +# : ${my_var2=my\ value2} +cont_store_env() +{ + cont_debug "creating env file '$2'" + __cont_encode_env "$1" > "$2" \ + || cont_warn "can't store environment $1 into $2 file" +} + + +__cont_source_scripts "/usr/share/cont-lib/autoload" diff --git a/docker/genomicsdb_builder/usr/share/cont-lib/parser-simple-config.sh b/docker/genomicsdb_builder/usr/share/cont-lib/parser-simple-config.sh new file mode 100644 index 00000000..4455957d --- /dev/null +++ b/docker/genomicsdb_builder/usr/share/cont-lib/parser-simple-config.sh @@ -0,0 +1,83 @@ +. "/usr/share/cont-lib/cont-lib.sh" + +# SEMICOLON_SPLIT VAR +# ------------------- +# Split the contents of string variable VAR into list of strings (on separate +# line), each of those strings will be printed to standard output. The ';' and +# newline characters are used as split separators. You can use quadrigraph @.,@ +# for ';' character not splitting the VAR (and use @&t@ to expand into empty +# string). More info about quadrigraphs may be found in autoconf info page. +cont_semicolon_split() +{ + eval set -- "\"\$$1\"" + test x = x"$1" && return 0 + + echo "$1" \ + | sed \ + -e 's/[[:space:]]*;[[:space:]]*/\n/g' \ + -e 's/^[[:space:]]*//g' \ + -e 's/\([^\\]\)[[:space:]]*$/\1/g' \ + | sed \ + -e 's|@.,@|;|g' \ + -e 's|@&t@||g' +} + +# CONT_PARSER_SIMPLE_CONFIG CONFIG_VAR CALLBACK [ARGS] +# ---------------------------------------------------- +# Parse contents of variable of name CONFIG_VAR, call CALLBACK function (or +# command) for each parsed configuration option. +# +# The format of configuration file is: +# +# = [; ...] +# +# Content of is not limited, but keep it sane please (lets say we support +# the C syntax of identifiers). The must be single-line string. Should +# the contain ';' or '\' character, it must be escaped by '\'. +# +# The semantics of CALLBACK you *must* provide is: +# +# CALLBACK KEY VALUE [ARGS] +# ------------------------- +# KEY and VALUE are strings with parsed result. ARGS is additional payload +# you may provide during CONTAINER_SIMPLE_CONFIG_PARSER call. +# +# Example of usage: +# +# $ cat script.sh +# callback() +# { +# local var="$1" val="$2" +# shift 2 +# test -n "$*" && local payload=" [$*]" +# echo "$var=$val$payload" +# } +# config='URL = "http://example.com"; semicolon = "@.,@"' +# cont_parser_simple_config config callback additional data +# +# $ ./script +# URL="http://example.com" [additional data] +# semicolon=";" [additional data] +cont_parser_simple_config() +{ + local conf_var="$1" + local callback="$2" + shift 2 + + while read line; do + test -z "$line" && continue + if [[ $line =~ ^([^[:space:]]+)[[:space:]]*=[[:space:]]*(.*)$ ]] + then + local k="${BASH_REMATCH[1]}" v="${BASH_REMATCH[2]}" + cont_debug3 "calling callback with: $k = $v" + "$callback" "$k" "$v" "$@" || { + cont_error "$FUNCNAME: callback failed" + return 1 + } + else + cont_warn "wrong config: $line" + fi + done < <(cont_semicolon_split "$conf_var") + + return 0 +} diff --git a/docker/requirements.txt b/docker/requirements.txt deleted file mode 100644 index 06c44966..00000000 --- a/docker/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -pytest ~= 3.0.7 -pylint ~= 1.7.1 - From e0155860f1cd1818ba1060ccf6053034f9cbfdbd Mon Sep 17 00:00:00 2001 From: Ming Rutar Date: Tue, 20 Jun 2017 14:12:30 -0700 Subject: [PATCH 6/7] more readme --- docker/genomicsdb_builder/Dockerfile | 2 +- docker/genomicsdb_builder/README.md | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/docker/genomicsdb_builder/Dockerfile b/docker/genomicsdb_builder/Dockerfile index 97c59c34..b52947db 100644 --- a/docker/genomicsdb_builder/Dockerfile +++ b/docker/genomicsdb_builder/Dockerfile @@ -1,5 +1,5 @@ # The MIT License (MIT) -# Copyright (c) 2017 Intel Corporation +# Copyright (c) 2016-2017 Intel Corporation # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/docker/genomicsdb_builder/README.md b/docker/genomicsdb_builder/README.md index 95b51081..2fd51a95 100644 --- a/docker/genomicsdb_builder/README.md +++ b/docker/genomicsdb_builder/README.md @@ -9,3 +9,15 @@ To create a GenomicsDB building environment Docker image, run the following comm Once the image is created. You can build the latest GenomicsDB executables by running: docker run -it -v /path/2/output/:/output/ [-e GDB_BRANCH=branch_name_default_is_master] name_of_the_builder_image + +In this case, the docker container builds GenomicsDB with the following options: + +* CMAKE_BUILD_TYPE=Release +* DO_PROFILING=False +* DISABLE_OPENMP=True +* BUILD_JAVA=False +* DO_PROFILING=False +* CMAKE_INSTALL_PREFIX=/output/ +* PROTOBUF_LIBRARY=dont_worry_the_docker_manage_it + +You can pass GenomicsDB building options via docker run command arguments. The docker will pass your arguments to cmake. Since the docker builds protobuf library internally, you just ignore the PROTOBUF_LIBRARY option. From b5a8bf528c8cad66c235d70a93253ca239b3471a Mon Sep 17 00:00:00 2001 From: Ming Rutar Date: Tue, 20 Jun 2017 14:17:54 -0700 Subject: [PATCH 7/7] add missed files --- docker/genomicsdb_builder/etc/profile.d/cont-env.sh | 1 + 1 file changed, 1 insertion(+) create mode 100644 docker/genomicsdb_builder/etc/profile.d/cont-env.sh diff --git a/docker/genomicsdb_builder/etc/profile.d/cont-env.sh b/docker/genomicsdb_builder/etc/profile.d/cont-env.sh new file mode 100644 index 00000000..01a215ef --- /dev/null +++ b/docker/genomicsdb_builder/etc/profile.d/cont-env.sh @@ -0,0 +1 @@ +source /usr/share/cont-lib/cont-env.sh