diff --git a/.circleci/config.yml b/.circleci/config.yml index 106be5b..c0c7166 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,6 +7,8 @@ commands: - run: | sudo apt update sudo apt install -y make bash git jq docker + sudo python3 -m pip install --upgrade pip + sudo pip3 install boto3 jobs: test: @@ -19,26 +21,30 @@ jobs: - run: make test - run: make test-destroy - redhat: + dockerhub: parameters: package: type: string - machine: true - steps: - - setup_environment - - checkout - - run: docker login -u unused --password ${REDHAT_REGISTRY_KEY_NODE} scan.connect.redhat.com - - run: make build-<> redhat-verify-<> VARIANT=redhat - - redhat-ciab: - machine: true + machine: + # https://circleci.com/docs/2.0/images/linux-vm/16.04-to-20.04-migration/ + image: ubuntu-2004:202201-02 steps: - setup_environment - checkout - - run: docker login -u unused --password ${REDHAT_REGISTRY_KEY_CIAB} scan.connect.redhat.com - - run: make build-ciab redhat-verify-ciab VARIANT=redhat + - run: docker login -u ${DOCKER_HUB_USER} --password ${DOCKER_HUB_KEY} + - run: make build-<> publish-<> + - when: + condition: + matches: + pattern: "^node(-.+)?$" + value: <> + steps: + - run: docker logout + - run: echo "${GCLOUD_SERVICE_ACCOUNT_SINGLESTORE_PUBLIC}" | base64 -d >./dockerconfig.json + - run: cat ./dockerconfig.json | docker login -u _json_key --password-stdin https://gcr.io + - run: make publish-gcr-<> - dockerhub: + gcr-public: parameters: package: type: string @@ -48,7 +54,8 @@ jobs: steps: - setup_environment - checkout - - run: docker login -u ${DOCKER_HUB_USER} --password ${DOCKER_HUB_KEY} + - run: echo "${GCLOUD_SERVICE_ACCOUNT_SINGLESTORE_PUBLIC}" | base64 -d >./dockerconfig.json + - run: cat ./dockerconfig.json | docker login -u _json_key --password-stdin https://gcr.io - run: make build-<> publish-<> ubi-gcr-internal: @@ -58,7 +65,7 @@ jobs: - checkout # We can't use Bash variables in CircleCI so we just duplicate the ""./dockerconfig.json" # string here. - - run: echo "${GCLOUD_SERVICE_ACCOUNT_INTERNAL}" | base64 -d >./dockerconfig.json + - run: echo "${GCLOUD_SERVICE_ACCOUNT_SINGLESTORE_PUBLIC}" | base64 -d >./dockerconfig.json - run: cat ./dockerconfig.json | docker login -u _json_key --password-stdin https://gcr.io - run: make build-node redhat-verify-ubi-gcr-internal-node VARIANT=redhat @@ -83,6 +90,9 @@ workflows: "node-7-3", "node-7-5", "node-7-6", + "node-7-8", + "node-8-0", + "node-8-1", "dynamic-node", "ciab", "tools", @@ -90,29 +100,6 @@ workflows: requires: - test - - publish-redhat: - name: publish-redhat-<< matrix.package >> - type: approval - matrix: - parameters: - # Must match line 93. - package: - [ - "node", - "node-7-0", - "node-7-1", - "node-7-3", - "node-7-5", - "node-7-6" - ] - requires: - - test - - - publish-redhat-ciab: - name: publish-redhat-ciab - type: approval - requires: - - test - dockerhub: name: dockerhub-<< matrix.package >> @@ -128,6 +115,9 @@ workflows: "node-7-3", "node-7-5", "node-7-6", + "node-7-8", + "node-8-0", + "node-8-1", "dynamic-node", "ciab", "tools", @@ -135,27 +125,6 @@ workflows: requires: - publish-dockerhub-<< matrix.package >> - - redhat: - name: redhat-<< matrix.package >> - matrix: - parameters: - package: - [ - "node", - "node-7-0", - "node-7-1", - "node-7-3", - "node-7-5", - "node-7-6" - ] - requires: - - publish-redhat-<< matrix.package >> - - - redhat-ciab: - name: redhat-ciab - requires: - - publish-redhat-ciab - - publish-ubi-gcr-internal: name: publish-ubi-gcr-internal type: approval @@ -165,4 +134,28 @@ workflows: - ubi-gcr-internal: name: ubi-gcr-internal requires: - - publish-ubi-gcr-internal \ No newline at end of file + - publish-ubi-gcr-internal + + - publish-gcr-public: + name: publish-gcr-public-<< matrix.package >> + type: approval + matrix: + parameters: + # Must match line 71. + package: + [ + "node-cloud", + ] + requires: + - test + + - gcr-public: + name: gcr-public-<< matrix.package >> + matrix: + parameters: + package: + [ + "node-cloud", + ] + requires: + - publish-gcr-public-<< matrix.package >> \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c7904c6 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +.idea +.vscode +lib \ No newline at end of file diff --git a/Dockerfile-base b/Dockerfile-base index 7bb200b..ec84350 100644 --- a/Dockerfile-base +++ b/Dockerfile-base @@ -16,10 +16,12 @@ RUN yum install yum-utils -y RUN yum-config-manager --save --setopt=skip_missing_names_on_install=0 # updates and basic utilities -ARG RELEASE_CHANNEL=production +ARG RELEASE_CHANNEL +RUN rpm --import https://repo.almalinux.org/almalinux/RPM-GPG-KEY-AlmaLinux + # currently yum update-minimal fails on curl, so we need to update curl before RUN yum update curl -y && yum -y update-minimal --setopt=tsflags=nodocs \ - --security --sec-severity=Important --sec-severity=Critical \ + --security --sec-severity=Important --sec-severity=Critical --nobest \ && yum install -y yum-utils wget procps RUN yum-config-manager --add-repo https://release.memsql.com/${RELEASE_CHANNEL}/rpm/x86_64/repodata/memsql.repo \ @@ -28,17 +30,8 @@ RUN yum-config-manager --add-repo https://release.memsql.com/${RELEASE_CHANNEL}/ RUN dnf --enablerepo=* clean all && dnf update -y; exit 0 # address vulnarabilities found by `grype --only-fixed` command -RUN yum remove vim-minimal platform-python-pip.noarch -y && yum update expat libxml2 libgcrypt -y - -# jq 1.6 -ENV JQ_VERSION='1.6' -RUN wget --no-check-certificate https://raw.githubusercontent.com/stedolan/jq/master/sig/jq-release.key -O /tmp/jq-release.key && \ - wget --no-check-certificate https://raw.githubusercontent.com/stedolan/jq/master/sig/v${JQ_VERSION}/jq-linux64.asc -O /tmp/jq-linux64.asc && \ - wget --no-check-certificate https://github.com/stedolan/jq/releases/download/jq-${JQ_VERSION}/jq-linux64 -O /tmp/jq-linux64 && \ - gpg --import /tmp/jq-release.key && \ - gpg --verify /tmp/jq-linux64.asc /tmp/jq-linux64 && \ - cp /tmp/jq-linux64 /usr/bin/jq && \ - chmod +x /usr/bin/jq && \ - rm -f /tmp/jq-release.key && \ - rm -f /tmp/jq-linux64.asc && \ - rm -f /tmp/jq-linux64 +RUN yum update expat libxml2 gnupg2 libgcrypt openssl-libs pcre2 curl libcurl-minimal systemd platform-python python3-libs gnutls libksba sqlite-libs zlib libcom_err krb5-libs dbus libtasn1 libarchive unbound-libs tar platform-python-setuptools python3-setuptools-wheel sqlite-libs systemd systemd-libs systemd-pam oniguruma gnutls python3-rpm python3-unbound rpm rpm-build-libs rpm-libs rpm-plugin-systemd-inhibit unbound-libs -y +RUN yum remove vim-minimal platform-python-pip.noarch -y + +RUN yum install -y jq-1.6 +RUN yum install -y which \ No newline at end of file diff --git a/Dockerfile-ciab b/Dockerfile-ciab index 25a5942..128daaa 100644 --- a/Dockerfile-ciab +++ b/Dockerfile-ciab @@ -1,15 +1,20 @@ # vim: ft=dockerfile -ARG BASE_IMAGE +# CHANGED FROM SOURCE: set the base image to fit the LOCALLY BUILT image +# (which was built using `make`, based on my changes to the Makefile) +ARG BASE_IMAGE="memsql/cluster-in-a-box" FROM ${BASE_IMAGE} # install SingleStore DB packages and additional dependencies -ARG CLIENT_VERSION -ARG SERVER_VERSION -ARG STUDIO_VERSION -ARG TOOLBOX_VERSION +# CHANGED FROM SOURCE: switch to root, add specific versions according to Makefile +USER root +ARG CLIENT_VERSION="1.0.7" +ARG SERVER_PACKAGE="singlestoredb-server" +ARG SERVER_VERSION="8.1.31-d5850b922e" +ARG STUDIO_VERSION="4.0.16" +ARG TOOLBOX_VERSION="1.17.6" RUN yum install -y \ singlestore-client-${CLIENT_VERSION} \ - singlestoredb-server${SERVER_VERSION} \ + ${SERVER_PACKAGE}${SERVER_VERSION} \ singlestoredb-studio-${STUDIO_VERSION} \ singlestoredb-toolbox-${TOOLBOX_VERSION} \ && yum clean all @@ -18,7 +23,11 @@ RUN mkdir -p /home/memsql && chown memsql:memsql /home/memsql ADD ciab-assets/studio.hcl /var/lib/singlestoredb-studio/studio.hcl RUN chown memsql:memsql /var/lib/singlestoredb-studio/studio.hcl -VOLUME ["/var/lib/memsql"] +# CHANGED FROM SOURCE: switch back from root +USER memsql + +# CHANGED FROM SOURCE: commented the VOLUME line to enable persisting data in our automation-db-upload job +#VOLUME ["/var/lib/memsql"] LABEL name="SingleStore DB Cluster in a Box" LABEL vendor="SingleStore" diff --git a/Dockerfile-dynamic b/Dockerfile-dynamic index e519b47..933f941 100644 --- a/Dockerfile-dynamic +++ b/Dockerfile-dynamic @@ -5,6 +5,7 @@ FROM ${BASE_IMAGE} ARG CLIENT_VERSION RUN yum install -y \ singlestore-client-${CLIENT_VERSION} \ + java-11-openjdk \ && yum clean all VOLUME ["/var/lib/memsql"] diff --git a/Dockerfile-node b/Dockerfile-node index 5b0dea7..2e449c6 100644 --- a/Dockerfile-node +++ b/Dockerfile-node @@ -6,6 +6,7 @@ FROM ${BASE_IMAGE} ARG CLIENT_VERSION ARG SERVER_VERSION ARG LOCAL_SERVER_RPM +ARG JRE_PACKAGES ADD assets /assets RUN if [[ -z "${LOCAL_SERVER_RPM}" ]] ; then \ @@ -13,23 +14,12 @@ RUN if [[ -z "${LOCAL_SERVER_RPM}" ]] ; then \ else \ rpm -i /assets/${LOCAL_SERVER_RPM}; \ fi \ + && for pkg in ${JRE_PACKAGES} ; do \ + yum install -y ${pkg}; \ + done \ && yum install -y singlestore-client-${CLIENT_VERSION} \ && yum clean all -RUN yum install python39 -y && update-alternatives --set python /usr/bin/python3; if [ $? -ne 0 ]; then \ -wget https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm && \ - wget http://mirror.centos.org/centos/7/sclo/x86_64/rh/Packages/r/rh-python36-runtime-2.0-1.el7.x86_64.rpm && \ - wget http://mirror.centos.org/centos/7/sclo/x86_64/rh/Packages/r/rh-python36-python-pip-9.0.1-5.el7.noarch.rpm && \ - wget http://mirror.centos.org/centos/7/sclo/x86_64/rh/Packages/r/rh-python36-python-3.6.12-1.el7.x86_64.rpm && \ - wget http://mirror.centos.org/altarch/7/sclo/aarch64/rh/Packages/r/rh-python36-python-setuptools-36.5.0-1.el7.noarch.rpm && \ - wget http://mirror.centos.org/centos/7/sclo/x86_64/rh/Packages/r/rh-python36-python-libs-3.6.12-1.el7.x86_64.rpm && \ - yum install -y epel-release-latest-7.noarch.rpm rh-python36-runtime-2.0-1.el7.x86_64.rpm rh-python36-python-pip-9.0.1-5.el7.noarch.rpm rh-python36-python-3.6.12-1.el7.x86_64.rpm rh-python36-python-setuptools-36.5.0-1.el7.noarch.rpm rh-python36-python-libs-3.6.12-1.el7.x86_64.rpm;\ - echo '/opt/rh/rh-python36/root/usr/lib64/' >> /etc/ld.so.conf && ldconfig;\ - ln -s /opt/rh/rh-python36/root/usr/bin/python3 /usr/bin/python3;\ - /usr/bin/python3 -m pip install --upgrade pip;\ - /usr/bin/python3 -m pip install pymysql;\ -fi - VOLUME ["/var/lib/memsql"] LABEL name="SingleStore DB Node" diff --git a/Makefile b/Makefile index 6bc841a..3df28d4 100644 --- a/Makefile +++ b/Makefile @@ -1,14 +1,24 @@ -SERVER_VERSION=7.8.10-375bb758c4 -SERVER_VERSION_PREVIEW=7.9.4-1a8fd43c84 +# this is the latest version +# CHANGED FROM SOURCE: enforce the specific version that we need for Skai's use-case +SERVER_VERSION=8.1.31-d5850b922e +SERVER_RELEASE_BRANCH=origin/qtpie-8.5 + +# this is actually 7.9 which is cloud-only, it's named this way to distingush the fact +# it tags the image differently and pushes to a different repo +SERVER_VERSION_CLOUD=7.9.25-5635578da8 +SERVER_VERSION_PREVIEW=8.7.1-a4ba56b7ef SERVER_VERSION_6_8=6.8.24-8e110b7bed SERVER_VERSION_7_0=7.0.26-8999f1390b SERVER_VERSION_7_1=7.1.25-af0195880c -SERVER_VERSION_7_3=7.3.26-edbc115410 -SERVER_VERSION_7_5=7.5.20-f604d8a71d -SERVER_VERSION_7_6=7.6.18-29f3f6ac6e -CLIENT_VERSION=1.0.5 -TOOLBOX_VERSION=1.13.9 -STUDIO_VERSION=4.0.7 +SERVER_VERSION_7_3=7.3.27-9f2e2f81f1 +SERVER_VERSION_7_5=7.5.25-3aa4489895 +SERVER_VERSION_7_6=7.6.33-c59b12bc70 +SERVER_VERSION_7_8=7.8.42-1a507bf140 +SERVER_VERSION_8_0=8.0.32-4959d03ded +SERVER_VERSION_8_1=8.1.38-61bcb26831 +CLIENT_VERSION=1.0.7 +TOOLBOX_VERSION=1.17.6 +STUDIO_VERSION=4.0.16 KUBE_CLIENT_VERSION=v1.11.6 REVISION=$(shell git describe --dirty=-dirty --always --long --abbrev=40 --match='') @@ -24,6 +34,7 @@ else endif endif +NODE_TAG_CLOUD=${VARIANT}-${SERVER_VERSION_CLOUD} NODE_TAG=${VARIANT}-${SERVER_VERSION} NODE_TAG_PREVIEW=${VARIANT}-${SERVER_VERSION_PREVIEW}-preview NODE_TAG_6_8=${VARIANT}-${SERVER_VERSION_6_8} @@ -32,6 +43,9 @@ NODE_TAG_7_1=${VARIANT}-${SERVER_VERSION_7_1} NODE_TAG_7_3=${VARIANT}-${SERVER_VERSION_7_3} NODE_TAG_7_5=${VARIANT}-${SERVER_VERSION_7_5} NODE_TAG_7_6=${VARIANT}-${SERVER_VERSION_7_6} +NODE_TAG_7_8=${VARIANT}-${SERVER_VERSION_7_8} +NODE_TAG_8_0=${VARIANT}-${SERVER_VERSION_8_0} +NODE_TAG_8_1=${VARIANT}-${SERVER_VERSION_8_1} DYNAMIC_TAG=${VARIANT}-${REVISION} CIAB_TAG=${VARIANT}-${SERVER_VERSION}-${STUDIO_VERSION}-${TOOLBOX_VERSION} TOOLS_TAG=${VARIANT}-${KUBE_CLIENT_VERSION}-${TOOLBOX_VERSION}-${REVISION} @@ -73,6 +87,20 @@ test: # node-7-6 ${MAKE} build-node-7-6 ${MAKE} test-node-7-6 + # node-7-8 + ${MAKE} build-node-7-8 + ${MAKE} test-node-7-8 + # node-8-0 + ${MAKE} build-node-8-0 + ${MAKE} test-node-8-0 + # node-8-1 + ${MAKE} build-node-8-1 + ${MAKE} test-node-8-1 + + # node-cloud + ${MAKE} build-node-cloud + ${MAKE} test-node-cloud + # dynamic node ${MAKE} build-dynamic-node ${MAKE} test-dynamic-node @@ -84,6 +112,15 @@ test: build-base: docker build \ --build-arg BASE_IMAGE=${BASE_IMAGE} \ + --build-arg RELEASE_CHANNEL=production \ + -t s2-base:${VARIANT} \ + -f Dockerfile-base . + +.PHONY: build-base-cloud +build-base-cloud: + docker build \ + --build-arg BASE_IMAGE=${BASE_IMAGE} \ + --build-arg RELEASE_CHANNEL=cloud \ -t s2-base:${VARIANT} \ -f Dockerfile-base . @@ -113,12 +150,28 @@ build-node: build-base --build-arg BASE_IMAGE=s2-base:${VARIANT} \ --build-arg SERVER_VERSION=${SERVER_VERSION} \ --build-arg CLIENT_VERSION=${CLIENT_VERSION} \ + --build-arg JRE_PACKAGES="java-11-openjdk java-21-openjdk" \ -t singlestore/node:${NODE_TAG} \ -f Dockerfile-node . docker tag singlestore/node:${NODE_TAG} singlestore/node:latest docker tag singlestore/node:${NODE_TAG} memsql/node:${NODE_TAG} docker tag singlestore/node:${NODE_TAG} memsql/node:latest + +.PHONY: build-node-cloud +build-node-cloud: build-base-cloud + docker build \ + --build-arg BASE_IMAGE=s2-base:${VARIANT} \ + --build-arg SERVER_VERSION=${SERVER_VERSION_CLOUD} \ + --build-arg CLIENT_VERSION=${CLIENT_VERSION} \ + -t singlestore/node:${NODE_TAG_CLOUD} \ + -f Dockerfile-node . + docker tag singlestore/node:${NODE_TAG_CLOUD} gcr.io/singlestore-public/memsql/node:${NODE_TAG_CLOUD} + docker tag singlestore/node:${NODE_TAG_CLOUD} memsql/node:${NODE_TAG_CLOUD} + docker tag singlestore/node:${NODE_TAG_CLOUD} gcr.io/singlestore-public/memsql/node:latest + docker tag singlestore/node:${NODE_TAG_CLOUD} memsql/node:latest + docker tag singlestore/node:${NODE_TAG_CLOUD} singlestore/node:latest + .PHONY: build-node-custom build-node-custom: build-base docker build \ @@ -126,6 +179,7 @@ build-node-custom: build-base --build-arg SERVER_VERSION=${SERVER_VERSION_CUSTOM} \ --build-arg CLIENT_VERSION=${CLIENT_VERSION} \ --build-arg LOCAL_SERVER_RPM=${LOCAL_SERVER_RPM_CUSTOM} \ + --build-arg JRE_PACKAGES=java-1.8.0-openjdk \ -t ${REGISTRY_CUSTOM}/singlestore/node:${NODE_TAG_CUSTOM} \ -t ${REGISTRY_CUSTOM}/memsql/node:${NODE_TAG_CUSTOM} \ -f Dockerfile-node . @@ -136,6 +190,7 @@ build-node-preview: build-base-dev --build-arg BASE_IMAGE=s2-base-dev:${VARIANT} \ --build-arg SERVER_VERSION=${SERVER_VERSION_PREVIEW} \ --build-arg CLIENT_VERSION=${CLIENT_VERSION} \ + --build-arg JRE_PACKAGES="java-11-openjdk java-21-openjdk" \ -t singlestore/node:${NODE_TAG_PREVIEW} \ -f Dockerfile-node . docker tag singlestore/node:${NODE_TAG_PREVIEW} memsql/node:${NODE_TAG_PREVIEW} @@ -200,6 +255,37 @@ build-node-7-6: build-base -f Dockerfile-node . docker tag singlestore/node:${NODE_TAG_7_6} memsql/node:${NODE_TAG_7_6} +.PHONY: build-node-7-8 +build-node-7-8: build-base + docker build \ + --build-arg BASE_IMAGE=s2-base:${VARIANT} \ + --build-arg SERVER_VERSION=${SERVER_VERSION_7_8} \ + --build-arg CLIENT_VERSION=${CLIENT_VERSION} \ + -t singlestore/node:${NODE_TAG_7_8} \ + -f Dockerfile-node . + docker tag singlestore/node:${NODE_TAG_7_8} memsql/node:${NODE_TAG_7_8} + +.PHONY: build-node-8-0 +build-node-8-0: build-base + docker build \ + --build-arg BASE_IMAGE=s2-base:${VARIANT} \ + --build-arg SERVER_VERSION=${SERVER_VERSION_8_0} \ + --build-arg CLIENT_VERSION=${CLIENT_VERSION} \ + -t singlestore/node:${NODE_TAG_8_0} \ + -f Dockerfile-node . + docker tag singlestore/node:${NODE_TAG_8_0} memsql/node:${NODE_TAG_8_0} + +.PHONY: build-node-8-1 +build-node-8-1: build-base + docker build \ + --build-arg BASE_IMAGE=s2-base:${VARIANT} \ + --build-arg SERVER_VERSION=${SERVER_VERSION_8_1} \ + --build-arg CLIENT_VERSION=${CLIENT_VERSION} \ + --build-arg JRE_PACKAGES=java-1.8.0-openjdk \ + -t singlestore/node:${NODE_TAG_8_1} \ + -f Dockerfile-node . + docker tag singlestore/node:${NODE_TAG_8_1} memsql/node:${NODE_TAG_8_1} + .PHONY: test-node test-node: test-destroy IMAGE=singlestore/node:${NODE_TAG} ./test/node @@ -232,6 +318,22 @@ test-node-7-5: test-destroy test-node-7-6: test-destroy IMAGE=singlestore/node:${NODE_TAG_7_6} ./test/node +.PHONY: test-node-7-8 +test-node-7-8: test-destroy + IMAGE=singlestore/node:${NODE_TAG_7_8} ./test/node + +.PHONY: test-node-8-0 +test-node-8-0: test-destroy + IMAGE=singlestore/node:${NODE_TAG_8_0} ./test/node + +.PHONY: test-node-8-1 +test-node-8-1: test-destroy + IMAGE=singlestore/node:${NODE_TAG_8_1} ./test/node + +.PHONY: test-node-cloud +test-node-cloud: test-destroy + IMAGE=singlestore/node:${NODE_TAG_CLOUD} ./test/node + .PHONY: test-node-ssl test-node-ssl: test-destroy IMAGE=singlestore/node:${NODE_TAG} ./test/node-ssl @@ -247,6 +349,21 @@ publish-node: docker push singlestore/node:latest docker push memsql/node:latest +.PHONY: publish-gcr-node +publish-gcr-node: + docker tag memsql/node:${NODE_TAG} gcr.io/singlestore-public/mirror/docker.io/memsql/node:${NODE_TAG} + docker tag memsql/node:${NODE_TAG} gcr.io/singlestore-public/mirror/docker.io/memsql/node:latest + docker push gcr.io/singlestore-public/mirror/docker.io/memsql/node:${NODE_TAG} + docker push gcr.io/singlestore-public/mirror/docker.io/memsql/node:latest +# publish the bottle-version dependency of the engine release to Helios control-plane. + MEMSQL_SERVER_VERSION=${SERVER_VERSION} RELEASE_CHANNEL=dev RELEASE_BRANCH=${SERVER_RELEASE_BRANCH} ./assets/release-metadata/release-metadata.py + MEMSQL_SERVER_VERSION=${SERVER_VERSION} RELEASE_CHANNEL=prod RELEASE_BRANCH=${SERVER_RELEASE_BRANCH} ./assets/release-metadata/release-metadata.py + +.PHONY: publish-node-cloud +publish-node-cloud: + docker push gcr.io/singlestore-public/memsql/node:${NODE_TAG_CLOUD} + docker push gcr.io/singlestore-public/memsql/node:latest + .PHONY: publish-node-custom publish-node-custom: docker push ${REGISTRY_CUSTOM}/singlestore/node:${NODE_TAG_CUSTOM} @@ -276,66 +393,90 @@ publish-node-6-8: docker push singlestore/node:${NODE_TAG_6_8} docker push memsql/node:${NODE_TAG_6_8} +.PHONY: publish-gcr-node-6-8 +publish-gcr-node-6-8: + docker tag memsql/node:${NODE_TAG_6_8} gcr.io/singlestore-public/mirror/docker.io/memsql/node:${NODE_TAG_6_8} + docker push gcr.io/singlestore-public/mirror/docker.io/memsql/node:${NODE_TAG_6_8} + .PHONY: publish-node-7-0 publish-node-7-0: docker push singlestore/node:${NODE_TAG_7_0} docker push memsql/node:${NODE_TAG_7_0} +.PHONY: publish-gcr-node-7-0 +publish-gcr-node-7-0: + docker tag memsql/node:${NODE_TAG_7_0} gcr.io/singlestore-public/mirror/docker.io/memsql/node:${NODE_TAG_7_0} + docker push gcr.io/singlestore-public/mirror/docker.io/memsql/node:${NODE_TAG_7_0} + .PHONY: publish-node-7-1 publish-node-7-1: docker push singlestore/node:${NODE_TAG_7_1} docker push memsql/node:${NODE_TAG_7_1} +.PHONY: publish-gcr-node-7-1 +publish-gcr-node-7-1: + docker tag memsql/node:${NODE_TAG_7_1} gcr.io/singlestore-public/mirror/docker.io/memsql/node:${NODE_TAG_7_1} + docker push gcr.io/singlestore-public/mirror/docker.io/memsql/node:${NODE_TAG_7_1} + .PHONY: publish-node-7-3 publish-node-7-3: docker push singlestore/node:${NODE_TAG_7_3} docker push memsql/node:${NODE_TAG_7_3} +.PHONY: publish-gcr-node-7-3 +publish-gcr-node-7-3: + docker tag memsql/node:${NODE_TAG_7_3} gcr.io/singlestore-public/mirror/docker.io/memsql/node:${NODE_TAG_7_3} + docker push gcr.io/singlestore-public/mirror/docker.io/memsql/node:${NODE_TAG_7_3} + .PHONY: publish-node-7-5 publish-node-7-5: docker push singlestore/node:${NODE_TAG_7_5} docker push memsql/node:${NODE_TAG_7_5} +.PHONY: publish-gcr-node-7-5 +publish-gcr-node-7-5: + docker tag memsql/node:${NODE_TAG_7_5} gcr.io/singlestore-public/mirror/docker.io/memsql/node:${NODE_TAG_7_5} + docker push gcr.io/singlestore-public/mirror/docker.io/memsql/node:${NODE_TAG_7_5} + .PHONY: publish-node-7-6 publish-node-7-6: docker push singlestore/node:${NODE_TAG_7_6} docker push memsql/node:${NODE_TAG_7_6} -.PHONY: redhat-verify-node -redhat-verify-node: - docker tag singlestore/node:${NODE_TAG} scan.connect.redhat.com/ospid-faf4ba09-5344-40d5-b9c5-7c88ea143472/node:${NODE_TAG} - docker push scan.connect.redhat.com/ospid-faf4ba09-5344-40d5-b9c5-7c88ea143472/node:${NODE_TAG} - @echo "View results + publish: https://connect.redhat.com/project/1123901/view" - -.PHONY: redhat-verify-node-7-0 -redhat-verify-node-7-0: - docker tag singlestore/node:${NODE_TAG_7_0} scan.connect.redhat.com/ospid-faf4ba09-5344-40d5-b9c5-7c88ea143472/node:${NODE_TAG_7_0} - docker push scan.connect.redhat.com/ospid-faf4ba09-5344-40d5-b9c5-7c88ea143472/node:${NODE_TAG_7_0} - @echo "View results + publish: https://connect.redhat.com/project/1123901/view" - -.PHONY: redhat-verify-node-7-1 -redhat-verify-node-7-1: - docker tag singlestore/node:${NODE_TAG_7_1} scan.connect.redhat.com/ospid-faf4ba09-5344-40d5-b9c5-7c88ea143472/node:${NODE_TAG_7_1} - docker push scan.connect.redhat.com/ospid-faf4ba09-5344-40d5-b9c5-7c88ea143472/node:${NODE_TAG_7_1} - @echo "View results + publish: https://connect.redhat.com/project/1123901/view" - -.PHONY: redhat-verify-node-7-3 -redhat-verify-node-7-3: - docker tag singlestore/node:${NODE_TAG_7_3} scan.connect.redhat.com/ospid-faf4ba09-5344-40d5-b9c5-7c88ea143472/node:${NODE_TAG_7_3} - docker push scan.connect.redhat.com/ospid-faf4ba09-5344-40d5-b9c5-7c88ea143472/node:${NODE_TAG_7_3} - @echo "View results + publish: https://connect.redhat.com/project/1123901/view" - -.PHONY: redhat-verify-node-7-5 -redhat-verify-node-7-5: - docker tag singlestore/node:${NODE_TAG_7_5} scan.connect.redhat.com/ospid-faf4ba09-5344-40d5-b9c5-7c88ea143472/node:${NODE_TAG_7_5} - docker push scan.connect.redhat.com/ospid-faf4ba09-5344-40d5-b9c5-7c88ea143472/node:${NODE_TAG_7_5} - @echo "View results + publish: https://connect.redhat.com/project/1123901/view" - -.PHONY: redhat-verify-node-7-6 -redhat-verify-node-7-6: - docker tag singlestore/node:${NODE_TAG_7_6} scan.connect.redhat.com/ospid-faf4ba09-5344-40d5-b9c5-7c88ea143472/node:${NODE_TAG_7_6} - docker push scan.connect.redhat.com/ospid-faf4ba09-5344-40d5-b9c5-7c88ea143472/node:${NODE_TAG_7_6} - @echo "View results + publish: https://connect.redhat.com/project/1123901/view" +.PHONY: publish-gcr-node-7-6 +publish-gcr-node-7-6: + docker tag memsql/node:${NODE_TAG_7_6} gcr.io/singlestore-public/mirror/docker.io/memsql/node:${NODE_TAG_7_6} + docker push gcr.io/singlestore-public/mirror/docker.io/memsql/node:${NODE_TAG_7_6} + +.PHONY: publish-node-7-8 +publish-node-7-8: + docker push singlestore/node:${NODE_TAG_7_8} + docker push memsql/node:${NODE_TAG_7_8} + +.PHONY: publish-gcr-node-7-8 +publish-gcr-node-7-8: + docker tag memsql/node:${NODE_TAG_7_8} gcr.io/singlestore-public/mirror/docker.io/memsql/node:${NODE_TAG_7_8} + docker push gcr.io/singlestore-public/mirror/docker.io/memsql/node:${NODE_TAG_7_8} + +.PHONY: publish-node-8-0 +publish-node-8-0: + docker push singlestore/node:${NODE_TAG_8_0} + docker push memsql/node:${NODE_TAG_8_0} + +.PHONY: publish-gcr-node-8-0 +publish-gcr-node-8-0: + docker tag memsql/node:${NODE_TAG_8_0} gcr.io/singlestore-public/mirror/docker.io/memsql/node:${NODE_TAG_8_0} + docker push gcr.io/singlestore-public/mirror/docker.io/memsql/node:${NODE_TAG_8_0} + +.PHONY: publish-node-8-1 +publish-node-8-1: + docker push singlestore/node:${NODE_TAG_8_1} + docker push memsql/node:${NODE_TAG_8_1} + +.PHONY: publish-gcr-node-8-1 +publish-gcr-node-8-1: + docker tag memsql/node:${NODE_TAG_8_1} gcr.io/singlestore-public/mirror/docker.io/memsql/node:${NODE_TAG_8_1} + docker push gcr.io/singlestore-public/mirror/docker.io/memsql/node:${NODE_TAG_8_1} .PHONY: build-dynamic-node build-dynamic-node: build-base @@ -363,6 +504,7 @@ publish-dynamic-node: build-ciab: build-base docker build \ --build-arg BASE_IMAGE=s2-base:${VARIANT} \ + --build-arg SERVER_PACKAGE=singlestoredb-server \ --build-arg SERVER_VERSION=${SERVER_VERSION} \ --build-arg CLIENT_VERSION=${CLIENT_VERSION} \ --build-arg STUDIO_VERSION=${STUDIO_VERSION} \ @@ -377,6 +519,7 @@ build-ciab: build-base build-ciab-dev: build-base-dev docker build \ --build-arg BASE_IMAGE=s2-base-dev:${VARIANT} \ + --build-arg SERVER_PACKAGE=memsql-server \ --build-arg SERVER_VERSION=${SERVER_VERSION} \ --build-arg CLIENT_VERSION=${CLIENT_VERSION} \ --build-arg STUDIO_VERSION=${STUDIO_VERSION} \ @@ -398,18 +541,12 @@ publish-ciab: docker push memsql/cluster-in-a-box:${CIAB_TAG} docker push memsql/cluster-in-a-box:latest -.PHONY: redhat-verify-ciab -redhat-verify-ciab: - docker tag singlestore/cluster-in-a-box:${CIAB_TAG} scan.connect.redhat.com/ospid-6b69e5e1-d98a-4d75-a591-e300d4820ecb/cluster-in-a-box:${CIAB_TAG} - docker push scan.connect.redhat.com/ospid-6b69e5e1-d98a-4d75-a591-e300d4820ecb/cluster-in-a-box:${CIAB_TAG} - @echo "View results + publish: https://connect.redhat.com/project/923891/view" - # This is used to publish an UBI-based (known as redhat) node image to GCR.io. .PHONY: redhat-verify-ubi-gcr-internal-node redhat-verify-ubi-gcr-internal-node: - docker tag singlestore/node:${NODE_TAG} gcr.io/internal-freya/memsql/node:${NODE_TAG} - docker push gcr.io/internal-freya/memsql/node:${NODE_TAG} - @echo "View results + publish: https://console.cloud.google.com/gcr/images/internal-freya/global/memsql/node" + docker tag singlestore/node:${NODE_TAG} gcr.io/singlestore-public/memsql/node:${NODE_TAG} + docker push gcr.io/singlestore-public/memsql/node:${NODE_TAG} + @echo "View results + publish: https://console.cloud.google.com/gcr/images/singlestore-public/global/memsql/node" .PHONY: test-destroy test-destroy: @@ -434,4 +571,4 @@ stage-tools: requires-license: ifndef LICENSE_KEY $(error LICENSE_KEY is required) -endif +endif \ No newline at end of file diff --git a/ORIGINAL_README.md b/ORIGINAL_README.md new file mode 100644 index 0000000..4b14a7f --- /dev/null +++ b/ORIGINAL_README.md @@ -0,0 +1,93 @@ +# deployment-docker + +This repository contains our official deployment Docker images for various products. +If you are interested in contributing, please read `CONTRIBUTING.md`. + +| Image | SingleStore Packages Installed | +|--------------|-------------------------------------------------------------------| +| ciab | singlestoredb-server, singlestoredb-studio, singlestoredb-toolbox | +| ciab-redhat | singlestoredb-server, singlestoredb-studio, singlestoredb-toolbox | +| dynamic-node | (none) | +| node | singlestoredb-server | +| node-redhat | singlestoredb-server | +| tools | singlestoredb-toolbox | + +# Running the Cluster in a Box image + +To initialize a new cluster in a box: + +```bash +docker run -i --init \ + --name singlestore-ciab \ + -e LICENSE_KEY=${LICENSE_KEY} \ + -e ROOT_PASSWORD=${ROOT_PASSWORD} \ + -p 3306:3306 -p 8080:8080 \ + singlestore/cluster-in-a-box +``` + +To manage your cluster in a box: + +```bash +To start the container: + docker start singlestore-ciab + +To read logs from the container: + docker logs singlestore-ciab + +To stop the container (must be started): + docker stop singlestore-ciab + +To restart the container (must be started): + docker restart singlestore-ciab + +To remove the container (all data will be deleted): + docker rm singlestore-ciab +``` + +# Automatically run SQL when Cluster in a Box initializes + +If you want to automatically run SQL commands when creating a Cluster in a Box +container, you can mount a SQL file into the Docker container like so: + +```bash +docker run -i --init \ + --name singlestore-ciab \ + -e LICENSE_KEY=${LICENSE_KEY} \ + -e ROOT_PASSWORD=${ROOT_PASSWORD} \ + -v /PATH/TO/INIT.SQL:/init.sql \ + -p 3306:3306 -p 8080:8080 \ + singlestore/cluster-in-a-box +``` + +**Replace `/PATH/TO/INIT.SQL` with a valid path on your machine to the SQL file +you want to run when initializing Cluster in a Box.** + +# Enable the HTTP API or External Functions + +The [HTTP API][httpapi] and [External Functions][extfunc] features can be enabled when you create the container via passing environment variables. + +**HTTP API:** + +Add the following flags to your `docker run` command: + +```bash + -e HTTP_API=ON -p 9000:9000 +``` + +By default, the HTTP API runs on port 9000. If you want to use a different port you can instead run: + +```bash + -e HTTP_API=ON -e HTTP_API_PORT=$PORT -p $PORT:$PORT +``` + +**External Functions:** + +Add the following flag to your `docker run` command: + +```bash + -e EXTERNAL_FUNCTIONS=ON +``` + +[httpapi]: https://docs.singlestore.com/db/latest/en/reference/http-api.html +[extfunc]: https://docs.singlestore.com/db/latest/en/reference/sql-reference/procedural-sql-reference/create--or-replace--external-function.html + diff --git a/README.md b/README.md index d834766..ddcd94c 100644 --- a/README.md +++ b/README.md @@ -1,92 +1,49 @@ -# deployment-docker - -This repository contains our official deployment Docker images for various products. -If you are interested in contributing, please read `CONTRIBUTING.md`. - -| Image | SingleStore Packages Installed | -|--------------|-------------------------------------------------------------------| -| ciab | singlestoredb-server, singlestoredb-studio, singlestoredb-toolbox | -| ciab-redhat | singlestoredb-server, singlestoredb-studio, singlestoredb-toolbox | -| dynamic-node | (none) | -| node | singlestoredb-server | -| node-redhat | singlestoredb-server | -| tools | singlestoredb-toolbox | - -# Running the Cluster in a Box image - -To initialize a new cluster in a box: - -```bash -docker run -i --init \ - --name singlestore-ciab \ - -e LICENSE_KEY=${LICENSE_KEY} \ - -e ROOT_PASSWORD=${ROOT_PASSWORD} \ - -p 3306:3306 -p 8080:8080 \ - singlestore/cluster-in-a-box -``` - -To manage your cluster in a box: - -```bash -To start the container: - docker start singlestore-ciab - -To read logs from the container: - docker logs singlestore-ciab - -To stop the container (must be started): - docker stop singlestore-ciab - -To restart the container (must be started): - docker restart singlestore-ciab - -To remove the container (all data will be deleted): - docker rm singlestore-ciab -``` - -# Automatically run SQL when Cluster in a Box initializes - -If you want to automatically run SQL commands when creating a Cluster in a Box -container, you can mount a SQL file into the Docker container like so: - -```bash -docker run -i --init \ - --name singlestore-ciab \ - -e LICENSE_KEY=${LICENSE_KEY} \ - -e ROOT_PASSWORD=${ROOT_PASSWORD} \ - -v /PATH/TO/INIT.SQL:/init.sql \ - -p 3306:3306 -p 8080:8080 \ - singlestore/cluster-in-a-box -``` - -**Replace `/PATH/TO/INIT.SQL` with a valid path on your machine to the SQL file -you want to run when initializing Cluster in a Box.** - -# Enable the HTTP API or External Functions - -The [HTTP API][httpapi] and [External Functions][extfunc] features can be enabled when you create the container via passing environment variables. - -**HTTP API:** - -Add the following flags to your `docker run` command: - -```bash - -e HTTP_API=ON -p 9000:9000 -``` - -By default, the HTTP API runs on port 9000. If you want to use a different port you can instead run: - -```bash - -e HTTP_API=ON -e HTTP_API_PORT=$PORT -p $PORT:$PORT -``` - -**External Functions:** - -Add the following flag to your `docker run` command: - -```bash - -e EXTERNAL_FUNCTIONS=ON -``` - -[httpapi]: https://docs.singlestore.com/db/latest/en/reference/http-api.html -[extfunc]: https://docs.singlestore.com/db/latest/en/reference/sql-reference/procedural-sql-reference/create--or-replace--external-function.html +# Introduction & motivation +This repo is a fork of [memsql/deployment-docker](https://github.com/memsql/deployment-docker), customized for building the Skai KS base MemSQL image that is used for [building the automation image](https://jenkins-prod-search.internalk.com/job/automation-tests-db-upload/) for our local environments and testing environments on Jenkins. This process is comprised of (as of Dec 2022): +1. Starting a cluster-in-a-box memsql docker image. +2. Running several cucumbers to populate it with schemas, tables and data. +3. Committing the docker state and upload the image to an artifactory. + +Then we can download the image in different automation jobs, start it and run tests that rely on the committed data. + +Q: Aren't the [default SingleStore scripts](https://github.com/memsql/deployment-docker) sufficient for this use case? +A: No, because they don't allow the retention of data after having populated the image with data and committed it. +Additionally, with the image built by the default scripts, it's impossible to start the image twice (i.e. start the committed image), due to the usage of the `sdb-toolbox` tool in the startup script. + +For more information, read the [original readme](./ORIGINAL_README.md). + +# Differences with source repo +The main changes made to the repo are: +* Makefile: Explicitly specify our desired memsql version. +* Dockerfile-ciab: + * Specify the base image and other tool versions. + * Add permissions to run installations on the image. + * Skip the `VOLUME` command, presumably to allow data retention on the image once we use it in our Jenkins automations. +* Startup: + * Unregister host, as it prevents the committed image to be started again as a new docker in automation jobs. + * Add custom memsql variable values to reduce the final committed image size. + +# Building the base image + +Run the release job: https://jenkins-prod-microcosm.internalk.com/job/memsql-base-automation-image-builder-release. +If you've made changes to the DSL files, you will need to run https://jenkins-prod-microcosm.internalk.com/job/memsql-base-automation-image-builder-main-dsl/ first. + +# Recommendations - discovery for next version upgrade + +When it's time to upgrade from 7.6 to some new version, one of the steps will be to create a new base image for Skai's automation environment. + +I suggest trying it like this: +1. Check what changes have been made on SingleStore's `deployment-docker` scripts repository since [the version we forked](https://github.com/memsql/deployment-docker/tree/c8cdbebe123d456940dd5fcb22f3b094563ca40b). +2 methods for doing that: + * Via Github: https://github.com/kenshoo/memsql-base-automation-image-builder/compare/master...memsql:deployment-docker:master + * Locally: + 1. `git checkout` this repo. + 1. `git remote add -f memsql_src git@github.com:memsql/deployment-docker.git` + 1. `git remote update` + 1. `git diff master remotes/memsql_src/master` + 1. `git remote rm memsql_src` + If they seem necessary, apply them to the repo on your local machine. +3. Try to apply the same changes that PR https://github.com/kenshoo/memsql-base-automation-image-builder/pull/1 contains (see description for summary) to the current version of the official repo. Adjust version numbers accordingly. +4. If the repo has changed so much that this PR's changes are no longer applicable, know that the main changes were skipping the 'VOLUME' command, and running `sdb-toolbox-config unregister-host`. All the others were workarounds to issues we ran into. +5. Good luck! + diff --git a/assets/lib.sh b/assets/lib.sh index 848c6a7..c67fc4d 100644 --- a/assets/lib.sh +++ b/assets/lib.sh @@ -1,5 +1,5 @@ INIT_FLAG=/.initialized -CHANNELS=(production dev) +CHANNELS=(production dev cloud) log() { (>&2 echo "${@}") } @@ -38,6 +38,36 @@ installRelease() { return 1 } +isVersionGE() +{ + local arg_major=${1} + local arg_minor=${2} + + local version=$(memsqlctl version | sed -n 's/^Version: \(.*\)$/\1/p') + local versionParts=($(echo ${version//./ })) + local major=${versionParts[0]} + local minor=${versionParts[1]} + local patch=${versionParts[2]} + + if [[ "${major}" -ne ${arg_major} ]]; then + if [[ "${major}" -gt ${arg_major} ]]; then + return 0 + else + return 1 + fi + fi + + if [[ "${minor}" -ne ${arg_minor} ]]; then + if [[ "${minor}" -gt ${arg_minor} ]]; then + return 0 + else + return 1 + fi + fi + + return 0 +} + createNode() { log "Initializing SingleStore DB node" memsqlctl -y create-node --no-start --base-install-dir /var/lib/memsql/instance @@ -54,6 +84,24 @@ setMaximumMemory() { fi } +setJava11Path() { + if isVersionGE 8 5; then + memsqlctl -y update-config --key java_pipelines_java11_path --value $(which java) + fi +} + +setJava21Path() { + if isVersionGE 8 7; then + # Look up the installation path in the alternatives list. If we don't + # find one, do nothing. + local java21Path=$(update-alternatives --list | grep '^jre_21\b' | cut -f 3 2>/dev/null) + if [ -n "${java21Path}" ] ; then + memsqlctl -y update-config --key fts2_java_path --value "${java21Path}/bin/java" + memsqlctl -y update-config --key fts2_java_home --value "${java21Path}" + fi + fi +} + checkSSL() { local code=0 if [ ! -r /etc/memsql/ssl/server-cert.pem ]; then @@ -75,14 +123,22 @@ enableSSL() { } waitStart() { - local ctl=${1} + local pid=${1} while true; do local CONNECTABLE=$(memsqlctl describe-node --property IsConnectable || echo false) - local RECOVERY=$(memsqlctl describe-node --property RecoveryState || echo Offline) - if [[ ${CONNECTABLE} == "true" && ${RECOVERY} == "Online" ]]; then + if [[ ${CONNECTABLE} == "true" ]]; then break fi - sleep 0.2 + + if kill -0 $pid; then + # memsqld still running, retry + sleep 0.2 + else + # memsdl is gone + log "memsqld exited" + exit 1 + fi + done } @@ -91,7 +147,15 @@ startSingleStore() { local memsqldPath=${installDir}/memsqld local memsqldSafePath=${installDir}/memsqld_safe - ${memsqldSafePath} \ + # as a sanity check, make sure there isn't an existing process running memsqldSafePath + # use || true to not fail due to `set -e` if it does not exist + existing_memsql_pid=$(pgrep -f ${memsqldSafePath}) || true + if [[ "$existing_memsql_pid" != "" ]]; then + log "Error, there is an existing memsql process running '$memsqldSafePath' with PID $existing_memsql_pid" + exit 1 + fi + + env -u ROOT_PASSWORD ${memsqldSafePath} \ --auto-restart disable \ --defaults-file $(memsqlctl describe-node --property MemsqlConfig) \ --memsqld ${memsqldPath} \ @@ -118,4 +182,4 @@ updateRootPassword() { if ! memsqlctl query -e "SELECT 1"; then memsqlctl -y change-root-password --password "${password}" fi -} +} \ No newline at end of file diff --git a/assets/release-metadata/release-metadata.py b/assets/release-metadata/release-metadata.py new file mode 100644 index 0000000..58ad9c4 --- /dev/null +++ b/assets/release-metadata/release-metadata.py @@ -0,0 +1,74 @@ +#!/usr/bin/python3 + +import boto3 +import configparser +import json +import logging +import requests +import os + +def _must_get_env(var, redact_log=False): + logging.info("Extracting environment variable {}".format(var)) + val = os.getenv(var) + if val is None: + logging.error("Missing required environment variable {}".format(var)) + raise Exception("Environment variable {} is required".format(var)) + logging.info("Got {} = {}".format(var, val if not redact_log else "")) + return val + +def download_bottle_version(ref, githubToken): + logging.info("Downloading BOTTLE_VERSION file from GitHub") + headers = { + 'Authorization': 'token ' + githubToken.strip(), + } + + try: + response = requests.get( + 'https://raw.githubusercontent.com/memsql/memsql/%s/BOTTLE_VERSION' % ref.replace("origin/", ""), + headers=headers, + ) + response.raise_for_status() + return response.text.strip() + except requests.exceptions.HTTPError as err: + logging.error('failed to download BOTTLE_VERSION file: %s' % err) + raise + +def generate_release_metadata_file(fileName, data): + logging.info("Generating release metadata file %s with contents (%s)" % (fileName, data)) + with open(fileName, 'wb') as f: + f.write(json.dumps(data, indent=2).encode('utf-8')) + +def upload_release_metadata_file(bucket, region, accessKeyID, accessKeySecret, fileName, release): + key = "memsqlserver/%s/%s" % (release, fileName) + logging.info("Uploading release metadata file '%s' to S3" % key) + s3 = boto3.resource('s3', aws_access_key_id=accessKeyID, aws_secret_access_key=accessKeySecret, region_name=region) + s3.meta.client.upload_file(fileName, bucket, key) + +if __name__ == "__main__": + try: + logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO")) + memsqlServerVersion = _must_get_env("MEMSQL_SERVER_VERSION") + memsqlReleaseChannel = _must_get_env("RELEASE_CHANNEL") + releaseBranch = _must_get_env("RELEASE_BRANCH") + releaseMetadataS3Bucket = _must_get_env("RELEASE_METADATA_AWS_BUCKET_NAME") + releaseMetadataBucketRegion = _must_get_env("RELEASE_METADATA_AWS_REGION") + releaseMetadataAWSAccessKeyID = _must_get_env("RELEASE_METADATA_AWS_ACCESS_KEY_ID", redact_log=True) + releaseMetadataAWSAccessKeySecret = _must_get_env("RELEASE_METADATA_AWS_SECRET_ACCESS_KEY", redact_log=True) + githubToken = _must_get_env("GITHUB_TOKEN", redact_log=True) + bottleVersion = download_bottle_version(releaseBranch, githubToken) + config = configparser.ConfigParser() + config.read_string('[default]\n' + bottleVersion) + bottleVersionMajorStr = config['default']['BOTTLE_VERSION_MAJOR'] + bottleVersionMinorStr = config['default']['BOTTLE_VERSION_MINOR'] + bottleVersionPatchStr = config['default']['BOTTLE_VERSION_PATCH'] + if not bottleVersionMajorStr.isdigit() or not bottleVersionMinorStr.isdigit() or not bottleVersionPatchStr.isdigit(): + raise Exception("BOTTLE_VERSION file contents are invalid: %s" % bottleVersion) + releaseMetadataContents = { + 'bottleVersion': '%s.%s.%s' % (bottleVersionMajorStr, bottleVersionMinorStr, bottleVersionPatchStr), + } + releaseMetadataFile = "%s.json" % memsqlServerVersion + generate_release_metadata_file(releaseMetadataFile, releaseMetadataContents) + upload_release_metadata_file(releaseMetadataS3Bucket, releaseMetadataBucketRegion, releaseMetadataAWSAccessKeyID, releaseMetadataAWSAccessKeySecret, releaseMetadataFile, memsqlReleaseChannel) + finally: + if os.path.exists(releaseMetadataFile): + os.remove(releaseMetadataFile) \ No newline at end of file diff --git a/assets/report/memsql-report-kubectl.sh b/assets/report/memsql-report-kubectl.sh index c6ac974..67ec40e 100755 --- a/assets/report/memsql-report-kubectl.sh +++ b/assets/report/memsql-report-kubectl.sh @@ -54,7 +54,7 @@ fi REPORT_DIR=$(mktemp -d) RESULT_DIR=$(mktemp -d) -timeout -k 5 ${REPORT_TIMEOUT} sdb-report collect-kube ${COLLECTOR_FLAGS} ${OTHER_FLAGS} --cluster-name ${CLUSTER_NAME} --version v1 --disable-colors --disable-spinner -vvv -o "${RESULT_DIR}/${CLUSTER_NAME}.tar.gz" --opt memsqlTracelogs.tracelogSize=100mb +timeout -k 5 ${REPORT_TIMEOUT} sdb-report collect-kube ${COLLECTOR_FLAGS} ${OTHER_FLAGS} --cluster-name ${CLUSTER_NAME} --version v1 --shard-queries --disable-colors --disable-spinner -vvv -o "${RESULT_DIR}/${CLUSTER_NAME}.tar.gz" --opt memsqlTracelogs.tracelogSize=100mb # copy out before checking because to support a failed check we need to see the report aws s3 ${ENDPOINT_FLAG} cp "${RESULT_DIR}/${CLUSTER_NAME}.tar.gz" "s3://${S3_REPORT_BUCKET}/${S3_REPORT_PATH}/report.tar.gz" @@ -69,6 +69,6 @@ aws s3 ${ENDPOINT_FLAG} cp "${RESULT_DIR}/metadata.txt" "s3://${S3_REPORT_BUCKET if [[ "${REPORT_TYPE}" = "Admin" ]] || [[ "$REPORT_TYPE" = "" ]]; then CHECK_REPORT_FILE="${RESULT_DIR}/${CLUSTER_NAME}.txt" echo "Internal: Will write check report to ${CHECK_REPORT_FILE}" - sdb-report check -i "${RESULT_DIR}/${CLUSTER_NAME}.tar.gz" | tee "${CHECK_REPORT_FILE}" || echo "WARNING: CHECKS FAILED" + sdb-report check -i "${RESULT_DIR}/${CLUSTER_NAME}.tar.gz" --exclude swapEnabled,minFreeKbytes,vmSwappiness | tee "${CHECK_REPORT_FILE}" || echo "WARNING: CHECKS FAILED" aws s3 ${ENDPOINT_FLAG} cp "${RESULT_DIR}/${CLUSTER_NAME}.txt" "s3://${S3_REPORT_BUCKET}/${S3_REPORT_PATH}/check.txt" fi diff --git a/assets/startup-node b/assets/startup-node index db4b640..d0a7b65 100755 --- a/assets/startup-node +++ b/assets/startup-node @@ -24,6 +24,9 @@ fi # set the nodes' maximum memory if needed setMaximumMemory "${MAXIMUM_MEMORY}" +setJava11Path +setJava21Path + # if a pre-start script is defined in the environment, and the file exists + is # executable, run the file before starting SingleStore DB if [ -n "${PRE_START_SCRIPT}" ]; then @@ -39,7 +42,7 @@ trap 'log "Stopping singlestore ($PID)..."; kill $PID; wait $PID; exit 0' SIGTER # start SingleStore DB in the background startSingleStore; PID=$! -waitStart +waitStart $PID log "Started singlestore ($PID)" diff --git a/buildScripts/jenkins/dsl/pull_request.groovy b/buildScripts/jenkins/dsl/pull_request.groovy new file mode 100644 index 0000000..24fc706 --- /dev/null +++ b/buildScripts/jenkins/dsl/pull_request.groovy @@ -0,0 +1,100 @@ +def NAME = "memsql-base-automation-image-builder" +def EMAIL = "shmulik.gutman@skai.io,shaked.nahum@skai.io" +def JOB_NAME = "${NAME}-pull-request" + +job(JOB_NAME) { + label("microcosm-ubuntu-base") + + logRotator(10,10) + concurrentBuild(true) + + throttleConcurrentBuilds{ + maxPerNode 1 + maxTotal 10 + } + + scm { + git { + remote { + url("https://github.com/kenshoo/${NAME}.git") + credentials('jenkins-microcosm-github-app') + refspec('+refs/pull/*:refs/remotes/origin/pr/*') + } + + configure { node -> + node / 'extensions' / 'hudson.plugins.git.extensions.impl.CleanBeforeCheckout' {} + } + + branch("\${sha1}") + } + } + + configure { project -> + def properties = project / 'properties' + properties<< { + 'com.coravy.hudson.plugins.github.GithubProjectProperty'{ + projectUrl "https://github.com/kenshoo/${NAME}/" + } + } + } + + wrappers { + preBuildCleanup() + timestamps() + injectPasswords() + colorizeOutput() + timeout { + absolute(120) + } + sshAgent('kgithub-build-jenkins-microcosm-key') + credentialsBinding { + usernamePassword('MICROSERVICES_ARTIFACTORY_USER', 'MICROSERVICES_ARTIFACTORY_PASSWORD', 'jcasc_deployer-microcosm') + } + } + + triggers { + githubPullRequest { + orgWhitelist('Kenshoo') + useGitHubHooks() + } + } + + steps { + shell(""" + make + docker build -t 668139184987.dkr.ecr.us-east-1.amazonaws.com/ks-db-memsql-cluster-in-a-box-base-guyh-8:8.1.31 -f Dockerfile-ciab . && echo "Successfully built base image" + """) + } + + publishers { + + extendedEmail { + recipientList("${EMAIL}") + triggers { + unstable { + sendTo { + requester() + developers() + } + } + failure { + sendTo { + requester() + developers() + } + } + statusChanged { + sendTo { + requester() + developers() + } + } + + configure { node -> + node / contentType << 'text/html' + } + } + } + + } +} \ No newline at end of file diff --git a/buildScripts/jenkins/dsl/release.groovy b/buildScripts/jenkins/dsl/release.groovy new file mode 100644 index 0000000..b3d30a6 --- /dev/null +++ b/buildScripts/jenkins/dsl/release.groovy @@ -0,0 +1,95 @@ +def NAME = "memsql-base-automation-image-builder" +def EMAIL = "shmulik.gutman@skai.io,shaked.nahum@skai.io" +def JOB_NAME = "${NAME}-release" +def BRANCH_NAME = "master" + +job(JOB_NAME) { + label("microcosm-ubuntu-base") + + logRotator(10,10) + concurrentBuild(true) + + throttleConcurrentBuilds{ + maxPerNode 1 + maxTotal 10 + } + + scm { + git { + remote { + url("https://github.com/kenshoo/${NAME}.git") + credentials('jenkins-microcosm-github-app') + refspec("+refs/heads/${BRANCH_NAME}:refs/remotes/origin/${BRANCH_NAME}") + } + + configure { node -> + node / 'extensions' / 'hudson.plugins.git.extensions.impl.CleanBeforeCheckout' {} + } + + branch(BRANCH_NAME) + } + } + + configure { project -> + def properties = project / 'properties' + properties<< { + 'com.coravy.hudson.plugins.github.GithubProjectProperty'{ + projectUrl "https://github.com/kenshoo/${NAME}/" + } + } + } + + wrappers { + preBuildCleanup() + timestamps() + injectPasswords() + colorizeOutput() + timeout { + absolute(120) + } + sshAgent('kgithub-build-jenkins-microcosm-key') + credentialsBinding { + usernamePassword('MICROSERVICES_ARTIFACTORY_USER', 'MICROSERVICES_ARTIFACTORY_PASSWORD', 'jcasc_deployer-microcosm') + } + } + + steps { + shell(""" + make + docker build -t 668139184987.dkr.ecr.us-east-1.amazonaws.com/ks-db-memsql-cluster-in-a-box-base-guyh-8:8.1.31 -f Dockerfile-ciab . && echo "Successfully built base image" + docker push 668139184987.dkr.ecr.us-east-1.amazonaws.com/ks-db-memsql-cluster-in-a-box-base-guyh-8:8.1.31 && echo "Successfully pushed image" + """) + } + + publishers { + + extendedEmail { + recipientList("${EMAIL}") + triggers { + unstable { + sendTo { + requester() + developers() + } + } + failure { + sendTo { + requester() + developers() + } + } + statusChanged { + sendTo { + requester() + developers() + } + } + + configure { node -> + node / contentType << 'text/html' + } + } + } + + } +} \ No newline at end of file diff --git a/ciab-assets/startup b/ciab-assets/startup index f38279b..12456ba 100644 --- a/ciab-assets/startup +++ b/ciab-assets/startup @@ -56,9 +56,18 @@ else #initialize the cluster fi echo Done. - echo Configuring Toolbox... - sdb-toolbox-config register-host -y --localhost --host 127.0.0.1 - echo Done. + echo Configuring Toolbox... + sdb-toolbox-config register-host -y --localhost --host 127.0.0.1 + +# CHANGED FROM SOURCE: added configurations to prevent huge docker image size. + memsqlctl -y update-config --all --set-global --key DEFAULT_PARTITIONS_PER_LEAF --value 2 + memsqlctl -y update-config --all --set-global --key SNAPSHOT_TRIGGER_SIZE --value 268435456 + memsqlctl -y update-config --all --set-global --key LOG_FILE_SIZE_PARTITIONS --value 268435456 + memsqlctl -y update-config --all --set-global --key LOG_FILE_SIZE_REF_DBS --value 67108864 + +# CHANGED FROM SOURCE: unregistered the host, because it prevents raising the docker twice. + sdb-toolbox-config unregister-host -y --host 127.0.0.1 + echo Done. if [ -f /init.sql ]; then echo Running /init.sql... @@ -95,15 +104,7 @@ singlestoredb-studio --port 8080 1>/dev/null 2>/dev/null & studio_pid=$! log_files+=("/var/lib/singlestoredb-studio/studio.log") -args_file="/etc/memsql/memsql_exporter_ma.args" -if [ ! -f $args_file ]; then - install_path=$(dirname $(realpath "/usr/bin/memsql_exporter")) - args_file=$install_path"/conf/memsql_exporter_ma.args" -fi -explog="/var/lib/memsql/memsql_exporter.log" -memsql_exporter @$args_file 1>$explog 2>$explog & -exporter_pid=$! -log_files+=($explog) +sdb-admin configure-monitoring -y --exporter-port=9104 --exporter-password=$ROOT_PASSWORD & tail -F $(printf '%s ' "${log_files[@]}") & tail_pid=$! @@ -112,7 +113,6 @@ cleanup() { echo Stopping Cluster... memsqlctl -y stop-node --all kill -15 $studio_pid - kill -15 $exporter_pid kill -15 $tail_pid echo Stopped. }