From 1cfddd8638c3b4daad615d5c1072d46712394692 Mon Sep 17 00:00:00 2001 From: fsanaulla Date: Sat, 14 Dec 2019 17:24:26 +0200 Subject: [PATCH 01/10] update livy and spark --- Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 44f75b6..2947a34 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,14 +28,14 @@ RUN mkdir /var/apache-spark-binaries/ # binaries # apache livy -RUN wget http://mirror.23media.de/apache/incubator/livy/0.5.0-incubating/livy-0.5.0-incubating-bin.zip -O /tmp/livy.zip +RUN wget http://mirror.23media.de/apache/incubator/livy/0.6.0-incubating/livy-0.6.0-incubating-bin.zip -O /tmp/livy.zip RUN unzip /tmp/livy.zip -d /opt/ # Logging dir -RUN mkdir /opt/livy-0.5.0-incubating-bin/logs +RUN mkdir /opt/livy-0.6.0-incubating-bin/logs # apache spark -RUN wget https://archive.apache.org/dist/spark/spark-2.3.1/spark-2.3.1-bin-hadoop2.7.tgz -O /tmp/spark-2.3.1-bin-hadoop2.7.tgz -RUN tar -xvzf /tmp/spark-2.3.1-bin-hadoop2.7.tgz -C /opt/ +RUN wget https://archive.apache.org/dist/spark/spark-2.4.4/spark-2.4.4-bin-hadoop2.7.tgz -O /tmp/spark-2.4.4-bin-hadoop2.7.tgz +RUN tar -xvzf /tmp/spark-2.4.4-bin-hadoop2.7.tgz -C /opt/ # set Python3 as default RUN rm /usr/bin/python From 3386e6177e251978b25b1935c96daf18dfdd11ce Mon Sep 17 00:00:00 2001 From: fsanaulla Date: Sat, 14 Dec 2019 17:25:54 +0200 Subject: [PATCH 02/10] update readme --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 2409f44..83a7198 100644 --- a/README.md +++ b/README.md @@ -10,9 +10,9 @@ Based on the latest release of the [Apache Livy project](https://livy.incubator. ### Supported Versions: -Livy-Server: 0.5.0 +Livy-Server: 0.6.0 -Apache Spark: 2.3.1 +Apache Spark: 2.4.4 Python: Python3 (including boto3 1.9) @@ -37,6 +37,6 @@ Livy Server start on default port 8998 ### Usage: ```bash -docker run -d -p 8998:8998 -e SPARK_MASTER_ENDPOINT=1.2.3.4 -e SPARK_MASTER_PORT=7077 -v /tmp:/tmp cloudiator/livy-server-docker:latest +docker run -d -p 8998:8998 -e SPARK_MASTER_ENDPOINT=1.2.3.4 -e SPARK_MASTER_PORT=7077 -v /tmp:/tmp cloudiator/livy-server:latest ``` From 5ee50f7ca88d0c5db2f2e18f8ed82b0575eb6cbb Mon Sep 17 00:00:00 2001 From: fsanaulla Date: Fri, 20 Dec 2019 20:10:47 +0200 Subject: [PATCH 03/10] additional fix --- Dockerfile | 6 ++---- init/entrypoint | 6 +++--- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2947a34..712a7c2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,10 +28,10 @@ RUN mkdir /var/apache-spark-binaries/ # binaries # apache livy -RUN wget http://mirror.23media.de/apache/incubator/livy/0.6.0-incubating/livy-0.6.0-incubating-bin.zip -O /tmp/livy.zip +RUN wget https://www-eu.apache.org/dist/incubator/livy/0.6.0-incubating/apache-livy-0.6.0-incubating-bin.zip -O /tmp/livy.zip RUN unzip /tmp/livy.zip -d /opt/ # Logging dir -RUN mkdir /opt/livy-0.6.0-incubating-bin/logs +RUN mkdir /opt/apache-livy-0.6.0-incubating-bin/logs # apache spark RUN wget https://archive.apache.org/dist/spark/spark-2.4.4/spark-2.4.4-bin-hadoop2.7.tgz -O /tmp/spark-2.4.4-bin-hadoop2.7.tgz @@ -41,8 +41,6 @@ RUN tar -xvzf /tmp/spark-2.4.4-bin-hadoop2.7.tgz -C /opt/ RUN rm /usr/bin/python RUN ln -s /usr/bin/python3 /usr/bin/python - - # expose ports EXPOSE 8998 diff --git a/init/entrypoint b/init/entrypoint index 53252eb..b9ce553 100755 --- a/init/entrypoint +++ b/init/entrypoint @@ -35,7 +35,7 @@ function writeConfigOptions(){ export SPARK_MASTER_PORT=$SPARK_MASTER_PORT export DEPLOY_MODE=$DEPLOY_MODE - cat /opt/docker-conf/livy.conf | envsubst > /opt/livy-0.5.0-incubating-bin/conf/livy.conf + cat /opt/docker-conf/livy.conf | envsubst > /opt/apache-livy-0.6.0-incubating-bin/conf/livy.conf } function init(){ @@ -58,9 +58,9 @@ function init(){ function livy_server_service(){ - export SPARK_HOME=/opt/spark-2.3.1-bin-hadoop2.7/ + export SPARK_HOME=/opt/spark-2.4.4-bin-hadoop2.7/ echo "starting Livy Server!" - /opt/livy-0.5.0-incubating-bin/bin/livy-server start + /opt/apache-livy-0.6.0-incubating-bin/bin/livy-server start # whatever blocking call tail -f /dev/null From eb8543ec93127612659d0043673f1157c5616cc7 Mon Sep 17 00:00:00 2001 From: fsanaulla Date: Tue, 25 Feb 2020 16:33:06 +0200 Subject: [PATCH 04/10] update livy image --- Dockerfile | 42 ++++++++++-------------------------------- init/entrypoint | 22 +++++++++++++++------- 2 files changed, 25 insertions(+), 39 deletions(-) diff --git a/Dockerfile b/Dockerfile index 712a7c2..7f4dc83 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,21 +1,8 @@ # select operating system -FROM ubuntu:16.04 +FROM alpine # install operating system packages -RUN apt-get update -y && apt-get install git curl gettext unzip wget software-properties-common python python-software-properties python-pip python3-pip dnsutils make -y - -## add more packages, if necessary -# install Java8 -RUN add-apt-repository ppa:webupd8team/java -y && apt-get update && apt-get -y install openjdk-8-jdk-headless - -# install boto3 library for PySpark applications to connect to S3 -RUN pip install boto3==1.9 - - -# use bpkg to handle complex bash entrypoints -RUN curl -Lo- "https://raw.githubusercontent.com/bpkg/bpkg/master/setup.sh" | bash -RUN bpkg install cha87de/bashutil -g -## add more bash dependencies, if necessary +RUN apk add unzip wget curl git bash openjdk8 gettext make coreutils procps && apk update # add config, init and source files # entrypoint @@ -23,23 +10,14 @@ ADD init /opt/docker-init ADD conf /opt/docker-conf # folders -RUN mkdir /opt/apache-livy -RUN mkdir /var/apache-spark-binaries/ - -# binaries -# apache livy -RUN wget https://www-eu.apache.org/dist/incubator/livy/0.6.0-incubating/apache-livy-0.6.0-incubating-bin.zip -O /tmp/livy.zip -RUN unzip /tmp/livy.zip -d /opt/ -# Logging dir -RUN mkdir /opt/apache-livy-0.6.0-incubating-bin/logs - -# apache spark -RUN wget https://archive.apache.org/dist/spark/spark-2.4.4/spark-2.4.4-bin-hadoop2.7.tgz -O /tmp/spark-2.4.4-bin-hadoop2.7.tgz -RUN tar -xvzf /tmp/spark-2.4.4-bin-hadoop2.7.tgz -C /opt/ - -# set Python3 as default -RUN rm /usr/bin/python -RUN ln -s /usr/bin/python3 /usr/bin/python +RUN mkdir /opt/apache-livy \ + && mkdir /var/apache-spark-binaries/ \ + && wget https://www-eu.apache.org/dist/incubator/livy/0.6.0-incubating/apache-livy-0.6.0-incubating-bin.zip -O /tmp/livy.zip \ + && unzip /tmp/livy.zip -d /opt/ \ + && mkdir /opt/apache-livy-0.6.0-incubating-bin/logs \ + && wget https://archive.apache.org/dist/spark/spark-2.4.4/spark-2.4.4-bin-hadoop2.7.tgz -O /tmp/spark-2.4.4-bin-hadoop2.7.tgz \ + && tar -xvzf /tmp/spark-2.4.4-bin-hadoop2.7.tgz -C /opt/ \ + && git clone https://github.com/cha87de/bashutil.git # expose ports EXPOSE 8998 diff --git a/init/entrypoint b/init/entrypoint index b9ce553..03e971c 100755 --- a/init/entrypoint +++ b/init/entrypoint @@ -1,18 +1,20 @@ #!/bin/bash # include bpkg dependencies -source /usr/local/bin/retry -source /usr/local/bin/bgo -source /usr/local/bin/bgowait +source /bashutil/retry +source /bashutil/bgo +source /bashutil/bgowait -# global variables -GLOBAL_VAR="xyz" +[[ -z "${LIVY_EXECUTION_MODE}" ]] && mode='local' || mode="${LIVY_EXECUTION_MODE}" ############################################################################## # validate if all container variables are set ############################################################################## function validate(){ + if [[ $mode == "local" ]]; then + return 0 + elif [[ $mode == "cluster" ]]; then vars="SPARK_MASTER_ENDPOINT SPARK_MASTER_PORT DEPLOY_MODE" for var in $vars; do if [[ $(env | awk -F "=" '{print $1}' | grep "^$var$") != "$var" ]]; then @@ -23,19 +25,25 @@ function validate(){ if [[ -z ${GLOBAL_VAR+x} ]]; then echo "GLOBAL_VAR variable cannot be looked up." return 1 - fi + fi + else + echo -e "Unsupported value $LIVY_EXECUTION_MODE" + return 1 + fi } ############################################################################## # write config vars with configfile template ############################################################################## function writeConfigOptions(){ + if [ $mode == "cluster" ]; then echo "write config options" export SPARK_MASTER_ENDPOINT=$SPARK_MASTER_ENDPOINT export SPARK_MASTER_PORT=$SPARK_MASTER_PORT export DEPLOY_MODE=$DEPLOY_MODE - + cat /opt/docker-conf/livy.conf | envsubst > /opt/apache-livy-0.6.0-incubating-bin/conf/livy.conf + fi } function init(){ From 731bc279e501cfe43f14751089bf8e160153e8bb Mon Sep 17 00:00:00 2001 From: fsanaulla Date: Tue, 25 Feb 2020 16:58:43 +0200 Subject: [PATCH 05/10] add postgres jar to livy --- Dockerfile | 13 ++++++------- examples/pi.py | 47 ----------------------------------------------- 2 files changed, 6 insertions(+), 54 deletions(-) delete mode 100644 examples/pi.py diff --git a/Dockerfile b/Dockerfile index 7f4dc83..f9f721a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,7 @@ # select operating system FROM alpine -# install operating system packages -RUN apk add unzip wget curl git bash openjdk8 gettext make coreutils procps && apk update +# install operating system packages # add config, init and source files # entrypoint @@ -10,13 +9,13 @@ ADD init /opt/docker-init ADD conf /opt/docker-conf # folders -RUN mkdir /opt/apache-livy \ - && mkdir /var/apache-spark-binaries/ \ +RUN apk add unzip wget curl git bash openjdk8 gettext make coreutils procps \ + && apk update \ && wget https://www-eu.apache.org/dist/incubator/livy/0.6.0-incubating/apache-livy-0.6.0-incubating-bin.zip -O /tmp/livy.zip \ + && wget https://archive.apache.org/dist/spark/spark-2.4.4/spark-2.4.4-bin-hadoop2.7.tgz -O /tmp/spark.tgz \ && unzip /tmp/livy.zip -d /opt/ \ - && mkdir /opt/apache-livy-0.6.0-incubating-bin/logs \ - && wget https://archive.apache.org/dist/spark/spark-2.4.4/spark-2.4.4-bin-hadoop2.7.tgz -O /tmp/spark-2.4.4-bin-hadoop2.7.tgz \ - && tar -xvzf /tmp/spark-2.4.4-bin-hadoop2.7.tgz -C /opt/ \ + && tar -xvzf /tmp/spark.tgz -C /opt/ \ + && wget https://jdbc.postgresql.org/download/postgresql-42.2.10.jar -P /opt/apache-livy-0.6.0-incubating-bin/jars \ && git clone https://github.com/cha87de/bashutil.git # expose ports diff --git a/examples/pi.py b/examples/pi.py deleted file mode 100644 index 5839cc2..0000000 --- a/examples/pi.py +++ /dev/null @@ -1,47 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from __future__ import print_function - -import sys -from random import random -from operator import add - -from pyspark.sql import SparkSession - - -if __name__ == "__main__": - """ - Usage: pi [partitions] - """ - spark = SparkSession\ - .builder\ - .appName("PythonPi")\ - .getOrCreate() - - partitions = int(sys.argv[1]) if len(sys.argv) > 1 else 2 - n = 100000 * partitions - - def f(_): - x = random() * 2 - 1 - y = random() * 2 - 1 - return 1 if x ** 2 + y ** 2 <= 1 else 0 - - count = spark.sparkContext.parallelize(range(1, n + 1), partitions).map(f).reduce(add) - print("Pi is roughly %f" % (4.0 * count / n)) - - spark.stop() From 55aba30b69e36db9a94a5833a281b10b1082263b Mon Sep 17 00:00:00 2001 From: fsanaulla Date: Wed, 26 Feb 2020 14:19:23 +0200 Subject: [PATCH 06/10] update image --- Dockerfile | 2 +- init/entrypoint | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index f9f721a..d990446 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,7 @@ RUN apk add unzip wget curl git bash openjdk8 gettext make coreutils procps \ && wget https://archive.apache.org/dist/spark/spark-2.4.4/spark-2.4.4-bin-hadoop2.7.tgz -O /tmp/spark.tgz \ && unzip /tmp/livy.zip -d /opt/ \ && tar -xvzf /tmp/spark.tgz -C /opt/ \ - && wget https://jdbc.postgresql.org/download/postgresql-42.2.10.jar -P /opt/apache-livy-0.6.0-incubating-bin/jars \ + && wget https://jdbc.postgresql.org/download/postgresql-42.2.10.jar -P /opt/spark-2.4.4-bin-hadoop2.7/jars \ && git clone https://github.com/cha87de/bashutil.git # expose ports diff --git a/init/entrypoint b/init/entrypoint index 03e971c..ffde719 100755 --- a/init/entrypoint +++ b/init/entrypoint @@ -7,7 +7,6 @@ source /bashutil/bgowait [[ -z "${LIVY_EXECUTION_MODE}" ]] && mode='local' || mode="${LIVY_EXECUTION_MODE}" - ############################################################################## # validate if all container variables are set ############################################################################## @@ -22,10 +21,6 @@ function validate(){ return 1 fi done - if [[ -z ${GLOBAL_VAR+x} ]]; then - echo "GLOBAL_VAR variable cannot be looked up." - return 1 - fi else echo -e "Unsupported value $LIVY_EXECUTION_MODE" return 1 From 5a055706718e310f7fd0054e6995477c502184fb Mon Sep 17 00:00:00 2001 From: fsanaulla Date: Tue, 31 Mar 2020 10:47:46 +0300 Subject: [PATCH 07/10] bump to spark 2.4.3 --- Dockerfile | 4 ++-- init/entrypoint | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index d990446..78a9438 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,10 +12,10 @@ ADD conf /opt/docker-conf RUN apk add unzip wget curl git bash openjdk8 gettext make coreutils procps \ && apk update \ && wget https://www-eu.apache.org/dist/incubator/livy/0.6.0-incubating/apache-livy-0.6.0-incubating-bin.zip -O /tmp/livy.zip \ - && wget https://archive.apache.org/dist/spark/spark-2.4.4/spark-2.4.4-bin-hadoop2.7.tgz -O /tmp/spark.tgz \ + && wget https://archive.apache.org/dist/spark/spark-2.4.3/spark-2.4.3-bin-hadoop2.7.tgz -O /tmp/spark.tgz \ && unzip /tmp/livy.zip -d /opt/ \ && tar -xvzf /tmp/spark.tgz -C /opt/ \ - && wget https://jdbc.postgresql.org/download/postgresql-42.2.10.jar -P /opt/spark-2.4.4-bin-hadoop2.7/jars \ + && wget https://jdbc.postgresql.org/download/postgresql-42.2.10.jar -P /opt/spark-2.4.3-bin-hadoop2.7/jars \ && git clone https://github.com/cha87de/bashutil.git # expose ports diff --git a/init/entrypoint b/init/entrypoint index ffde719..6c87218 100755 --- a/init/entrypoint +++ b/init/entrypoint @@ -61,7 +61,7 @@ function init(){ function livy_server_service(){ - export SPARK_HOME=/opt/spark-2.4.4-bin-hadoop2.7/ + export SPARK_HOME=/opt/spark-2.4.3-bin-hadoop2.7/ echo "starting Livy Server!" /opt/apache-livy-0.6.0-incubating-bin/bin/livy-server start From b8882b77da941de3764d18d3e8d64fd079b0af65 Mon Sep 17 00:00:00 2001 From: fsanaulla Date: Fri, 12 Jun 2020 22:51:57 +0300 Subject: [PATCH 08/10] bump to spark 2.4.3 --- Dockerfile | 7 +++++-- README.md | 2 +- init/entrypoint | 6 +++--- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 78a9438..02c4f95 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,11 +11,14 @@ ADD conf /opt/docker-conf # folders RUN apk add unzip wget curl git bash openjdk8 gettext make coreutils procps \ && apk update \ - && wget https://www-eu.apache.org/dist/incubator/livy/0.6.0-incubating/apache-livy-0.6.0-incubating-bin.zip -O /tmp/livy.zip \ + && wget https://www-eu.apache.org/dist/incubator/livy/0.7.0-incubating/apache-livy-0.7.0-incubating-bin.zip -O /tmp/livy.zip \ && wget https://archive.apache.org/dist/spark/spark-2.4.3/spark-2.4.3-bin-hadoop2.7.tgz -O /tmp/spark.tgz \ && unzip /tmp/livy.zip -d /opt/ \ && tar -xvzf /tmp/spark.tgz -C /opt/ \ - && wget https://jdbc.postgresql.org/download/postgresql-42.2.10.jar -P /opt/spark-2.4.3-bin-hadoop2.7/jars \ + # postgres jar + && wget https://jdbc.postgresql.org/download/postgresql-42.2.10.jar -P /opt/spark-2.4.4-bin-hadoop2.7/jars \ + # spark-excel support + && wget https://oss.sonatype.org/content/repositories/public/com/crealytics/spark-excel_2.12/0.13.1/spark-excel_2.12-0.13.1.jar -P /opt/spark-2.4.4-bin-hadoop2.7/jars \ && git clone https://github.com/cha87de/bashutil.git # expose ports diff --git a/README.md b/README.md index 83a7198..15e6619 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ Based on the latest release of the [Apache Livy project](https://livy.incubator. ### Supported Versions: -Livy-Server: 0.6.0 +Livy-Server: 0.7.0 Apache Spark: 2.4.4 diff --git a/init/entrypoint b/init/entrypoint index 6c87218..b8527ae 100755 --- a/init/entrypoint +++ b/init/entrypoint @@ -37,7 +37,7 @@ function writeConfigOptions(){ export SPARK_MASTER_PORT=$SPARK_MASTER_PORT export DEPLOY_MODE=$DEPLOY_MODE - cat /opt/docker-conf/livy.conf | envsubst > /opt/apache-livy-0.6.0-incubating-bin/conf/livy.conf + cat /opt/docker-conf/livy.conf | envsubst > /opt/apache-livy-0.7.0-incubating-bin/conf/livy.conf fi } @@ -61,9 +61,9 @@ function init(){ function livy_server_service(){ - export SPARK_HOME=/opt/spark-2.4.3-bin-hadoop2.7/ + export SPARK_HOME=/opt/spark-2.4.4-bin-hadoop2.7/ echo "starting Livy Server!" - /opt/apache-livy-0.6.0-incubating-bin/bin/livy-server start + /opt/apache-livy-0.7.0-incubating-bin/bin/livy-server start # whatever blocking call tail -f /dev/null From a8fef78eea63bc36fbce36f3388b98156a25062b Mon Sep 17 00:00:00 2001 From: fsanaulla Date: Fri, 12 Jun 2020 23:27:51 +0300 Subject: [PATCH 09/10] upgrade to live 0.7.0 --- Dockerfile | 2 ++ init/entrypoint | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 02c4f95..3d584aa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,6 +12,8 @@ ADD conf /opt/docker-conf RUN apk add unzip wget curl git bash openjdk8 gettext make coreutils procps \ && apk update \ && wget https://www-eu.apache.org/dist/incubator/livy/0.7.0-incubating/apache-livy-0.7.0-incubating-bin.zip -O /tmp/livy.zip \ + && apk add --no-cache libc6-compat \ + && ln -s /lib/libc.musl-x86_64.so.1 /lib/ld-linux-x86-64.so.2 \ && wget https://archive.apache.org/dist/spark/spark-2.4.3/spark-2.4.3-bin-hadoop2.7.tgz -O /tmp/spark.tgz \ && unzip /tmp/livy.zip -d /opt/ \ && tar -xvzf /tmp/spark.tgz -C /opt/ \ diff --git a/init/entrypoint b/init/entrypoint index b8527ae..41447ca 100755 --- a/init/entrypoint +++ b/init/entrypoint @@ -61,7 +61,7 @@ function init(){ function livy_server_service(){ - export SPARK_HOME=/opt/spark-2.4.4-bin-hadoop2.7/ + export SPARK_HOME=/opt/spark-2.4.3-bin-hadoop2.7/ echo "starting Livy Server!" /opt/apache-livy-0.7.0-incubating-bin/bin/livy-server start From 5d47fe47937ef947600473eaa99cf195d33a0465 Mon Sep 17 00:00:00 2001 From: fsanaulla Date: Sun, 14 Jun 2020 19:52:53 +0300 Subject: [PATCH 10/10] excel support --- Dockerfile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3d584aa..0763f55 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,9 +18,12 @@ RUN apk add unzip wget curl git bash openjdk8 gettext make coreutils procps \ && unzip /tmp/livy.zip -d /opt/ \ && tar -xvzf /tmp/spark.tgz -C /opt/ \ # postgres jar - && wget https://jdbc.postgresql.org/download/postgresql-42.2.10.jar -P /opt/spark-2.4.4-bin-hadoop2.7/jars \ + && wget https://jdbc.postgresql.org/download/postgresql-42.2.10.jar -P /opt/spark-2.4.3-bin-hadoop2.7/jars \ # spark-excel support - && wget https://oss.sonatype.org/content/repositories/public/com/crealytics/spark-excel_2.12/0.13.1/spark-excel_2.12-0.13.1.jar -P /opt/spark-2.4.4-bin-hadoop2.7/jars \ + && wget https://oss.sonatype.org/content/repositories/public/com/crealytics/spark-excel_2.11/0.13.1/spark-excel_2.11-0.13.1.jar -P /opt/spark-2.4.3-bin-hadoop2.7/jars \ + && wget https://repo1.maven.org/maven2/org/apache/commons/commons-collections4/4.1/commons-collections4-4.1.jar -P /opt/spark-2.4.3-bin-hadoop2.7/jars \ + && wget https://repo1.maven.org/maven2/org/apache/xmlbeans/xmlbeans/3.1.0/xmlbeans-3.1.0.jar -P /opt/spark-2.4.3-bin-hadoop2.7/jars \ + && wget https://repo1.maven.org/maven2/org/apache/poi/poi-ooxml-schemas/4.1.2/poi-ooxml-schemas-4.1.2.jar -P /opt/spark-2.4.3-bin-hadoop2.7/jars \ && git clone https://github.com/cha87de/bashutil.git # expose ports