diff --git a/concourse/pipeline/build_tarball_open.yml b/concourse/pipeline/build_tarball_open.yml new file mode 100644 index 00000000000..095fb3492ab --- /dev/null +++ b/concourse/pipeline/build_tarball_open.yml @@ -0,0 +1,53 @@ +resources: + - name: eloqsql_src_main + type: git + source: + branch: main + uri: git@github.com:eloqdata/eloqsql.git + private_key: ((git-key)) + + - name: image_ubuntu2404_main + type: registry-image + source: + repository: ubuntu + tag: "24.04" + + - name: logservice_src_main + type: git + source: + branch: main + private_key: ((git-key)) + uri: git@github.com:eloqdata/log_service.git + + - name: raft_host_manager_src_main + type: git + source: + branch: main + private_key: ((git-key)) + uri: git@github.com:eloqdata/raft_host_manager.git + +jobs: + - name: ubuntu2404-debug-openlog + plan: + - in_parallel: + limit: 2 + steps: + - get: eloqsql_src + resource: eloqsql_src_main + - get: logservice_src + resource: logservice_src_main + - get: raft_host_manager_src + resource: raft_host_manager_src_main + - get: image_ubuntu2404 + resource: image_ubuntu2404_main + + - task: build-debug-openlog + file: eloqsql_src/concourse/tasks/build_tarball_open.yml + image: image_ubuntu2404 + params: + GIT_SSH_KEY: ((git-key)) + BUILD_TYPE: Debug + OUT_NAME: debug-openlog + + serial: true + diff --git a/concourse/scripts/build_tarball_open.bash b/concourse/scripts/build_tarball_open.bash new file mode 100755 index 00000000000..7535644eb14 --- /dev/null +++ b/concourse/scripts/build_tarball_open.bash @@ -0,0 +1,152 @@ +#!/bin/bash +set -exo pipefail + +# This script builds EloqSQL tarball with open_log_service enabled, using Debug build on Ubuntu 24.04 +# It follows the build configuration outlined in README.md, with packaging similar to build_tarball.bash + +export WORKSPACE=$PWD +export AWS_PAGER="" +export DEBIAN_FRONTEND=noninteractive +export TZ=UTC + +apt-get update && apt-get install -y sudo openssh-client + + +current_user=$(whoami) +sudo chown -R "$current_user" "$PWD" + +# Setup SSH for private submodules if provided +if [ -n "${GIT_SSH_KEY}" ]; then + mkdir -p ~/.ssh + echo "${GIT_SSH_KEY}" > ~/.ssh/id_rsa + chmod 600 ~/.ssh/id_rsa + ssh-keyscan github.com >> ~/.ssh/known_hosts 2>/dev/null || true +fi + +# Defaults aligned with README.md +: "${BUILD_TYPE:=Debug}" +: "${ASAN:=OFF}" +: "${DATA_STORE_TYPE:=ELOQDSS_ROCKSDB_CLOUD_S3}" +: "${NCORE:=8}" + +DEST_DIR="${HOME}/EloqSQL" +OUT_NAME="${OUT_NAME:-debug-openlog}" + +# Prepare workspace layout expected by scripts +cd "$HOME" +ln -sfn "${WORKSPACE}/eloqsql_src" eloqsql +ln -sfn "${WORKSPACE}/logservice_src" logservice_src || true +mkdir -p eloqsql/storage/eloq/tx_service +ln -sfn "${WORKSPACE}/raft_host_manager_src" eloqsql/storage/eloq/tx_service/raft_host_manager || true + +ELOQSQL_SRC="${HOME}/eloqsql" + +# Install all dependencies using Ubuntu 24.04 script +source /etc/os-release +if [[ "$ID" == ubuntu* ]]; then + echo "Installing dependencies for Ubuntu ${VERSION_ID}..." + cd "$ELOQSQL_SRC" + bash scripts/install_dependency_ubuntu2404.sh + + # Activate the Python virtual environment created by the dependency script + source $HOME/venv/bin/activate +fi + +# Initialize submodules per README +cd "$ELOQSQL_SRC" +git submodule sync +git submodule update --init --recursive + +# Also ensure log_service submodules +if [ -d "storage/eloq/log_service" ]; then + pushd storage/eloq/log_service + git submodule sync + git submodule update --init --recursive + popd +fi + +# Build EloqSQL per README with OPEN_LOG_SERVICE enabled +mkdir build +cd build + +cmake -DCMAKE_INSTALL_PREFIX="${DEST_DIR}" \ + -DCMAKE_BUILD_TYPE="${BUILD_TYPE}" \ + -DWITH_READLINE=1 \ + -DPLUGIN_HANDLERSOCKET=NO \ + -DPLUGIN_ROCKSDB=NO \ + -DPLUGIN_ARIA=NO \ + -DPLUGIN_ARCHIVE=NO \ + -DPLUGIN_CVS=NO \ + -DPLUGIN_FEDERATEDX=NO \ + -DPLUGIN_TOKUDB=NO \ + -DPLUGIN_MROONGA=NO \ + -DPLUGIN_OQGRAPH=NO \ + -DPLUGIN_CONNECT=NO \ + -DPLUGIN_SPIDER=NO \ + -DPLUGIN_SPHINX=NO \ + -DPLUGIN_HEAP=NO \ + -DPLUGIN_MYISAMMRG=NO \ + -DPLUGIN_SEQUENCE=NO \ + -DINSTALL_MYSQLTESTDIR= \ + -DMYSQL_MAINTAINER_MODE=OFF \ + -DWITH_SSL=system \ + -DCOROUTINE_ENABLED=ON \ + -DBRPC_WITH_GLOG=ON \ + -DMARIA_WITH_GLOG=ON \ + -DWITH_ASAN="${ASAN}" \ + -DCMAKE_C_FLAGS_DEBUG="-O0 -g -DDBUG_ON -fno-omit-frame-pointer -fno-strict-aliasing" \ + -DCMAKE_CXX_FLAGS_DEBUG="-O0 -g -DDBUG_ON -fno-omit-frame-pointer -fno-strict-aliasing -felide-constructors -Wno-error" \ + -DWITH_DATA_STORE="${DATA_STORE_TYPE}" \ + -DOPEN_LOG_SERVICE=ON \ + ../ + +cmake --build . --config "${BUILD_TYPE}" -j"${NCORE}" +cmake --install . --config "${BUILD_TYPE}" + +# Helper to copy dependent libraries for portability +copy_libraries() { + local executable="$1" + local path="$2" + libraries=$(ldd "$executable" | awk 'NF==4{print $(NF-1)}{}') + mkdir -p "$path" + for lib in $libraries; do + rsync -avL --ignore-existing "$lib" "$path/" + done +} + +# Ensure runtime libs packaged +mkdir -p "${DEST_DIR}/lib" "${DEST_DIR}/bin" "${DEST_DIR}/conf" +copy_libraries "${DEST_DIR}/bin/mariadbd" "${DEST_DIR}/lib" +copy_libraries "${DEST_DIR}/bin/mariadb" "${DEST_DIR}/lib" + +# Build and include dss_server component +pushd "${ELOQSQL_SRC}/storage/eloq/store_handler/eloq_data_store_service" +rm -rf bld +mkdir bld && cd bld +cmake -DCMAKE_BUILD_TYPE="${BUILD_TYPE}" -DWITH_DATA_STORE="${DATA_STORE_TYPE}" ../ +cmake --build . --config "${BUILD_TYPE}" -j"${NCORE}" +copy_libraries dss_server "${DEST_DIR}/lib" +mv dss_server "${DEST_DIR}/bin/" +popd + +# Build and include log_service (launch_sv) +pushd "${ELOQSQL_SRC}/storage/eloq/log_service" +rm -rf bld +mkdir bld && cd bld +cmake -DCMAKE_BUILD_TYPE="${BUILD_TYPE}" -DUSE_ROCKSDB_LOG_STATE=ON ../ +cmake --build . --config "${BUILD_TYPE}" -j"${NCORE}" +copy_libraries launch_sv "${DEST_DIR}/lib" +mv launch_sv "${DEST_DIR}/bin/" +popd + +# Create tarball +cd "${HOME}" +tar -czvf eloqsql.tar.gz -C "${HOME}" EloqSQL + +# Cleanup build directories +cd "${ELOQSQL_SRC}" +rm -rf build +rm -rf storage/eloq/store_handler/eloq_data_store_service/bld +rm -rf storage/eloq/log_service/bld + +echo "Build completed. Tarball created at: ${HOME}/eloqsql.tar.gz" diff --git a/concourse/tasks/build_tarball_open.yml b/concourse/tasks/build_tarball_open.yml new file mode 100644 index 00000000000..b0a966c9283 --- /dev/null +++ b/concourse/tasks/build_tarball_open.yml @@ -0,0 +1,13 @@ +platform: linux +image_resource: + type: registry-image +inputs: + - name: eloqsql_src + - name: logservice_src + - name: raft_host_manager_src +outputs: + - name: the-output +run: + path: /bin/bash + args: + - eloqsql_src/concourse/scripts/build_tarball_open.bash diff --git a/scripts/install_dependency_ubuntu2404.sh b/scripts/install_dependency_ubuntu2404.sh index 0f1de4c7e1d..fc72f0151f5 100644 --- a/scripts/install_dependency_ubuntu2404.sh +++ b/scripts/install_dependency_ubuntu2404.sh @@ -2,8 +2,27 @@ set -ex +# Ensure noninteractive apt; keep TZ default +export DEBIAN_FRONTEND=noninteractive +export TZ=${TZ:-UTC} + +needs_tz_config=false +if [ ! -f /etc/timezone ] || ! grep -qE '^(Etc/UTC|UTC)$' /etc/timezone; then + needs_tz_config=true +fi +if [ ! -L /etc/localtime ] || [ "$(readlink -f /etc/localtime)" != "/usr/share/zoneinfo/Etc/UTC" ]; then + needs_tz_config=true +fi + +if $needs_tz_config; then + echo 'tzdata tzdata/Areas select Etc' | sudo debconf-set-selections || true + echo 'tzdata tzdata/Zones/Etc select UTC' | sudo debconf-set-selections || true + echo 'Etc/UTC' | sudo tee /etc/timezone >/dev/null + sudo ln -sf /usr/share/zoneinfo/Etc/UTC /etc/localtime +fi + # Install system packages -sudo apt-get update +DEBIAN_FRONTEND=noninteractive sudo apt-get update DEBIAN_FRONTEND=noninteractive sudo apt-get install -y --no-install-recommends \ jq sudo vim wget curl apt-utils python3 python3-dev python3-pip python3-venv \ python3-venv gdb libcurl4-openssl-dev build-essential libncurses5-dev \ @@ -49,7 +68,7 @@ sudo ldconfig cd ../ && rm -rf protobuf # Install glog -git clone https://github.com/monographdb/glog.git glog +git clone https://github.com/eloqdata/glog.git glog cd glog cmake -S . -B build -G "Unix Makefiles" cmake --build build -j6 @@ -65,7 +84,7 @@ make -j4 && sudo make install cd .. && rm -rf liburing # Install brpc -git clone https://github.com/monographdb/brpc.git brpc +git clone https://github.com/eloqdata/brpc.git brpc cd brpc mkdir build && cd build cmake .. \ @@ -78,7 +97,7 @@ sudo cp ./output/lib/* /usr/lib/ cd ../../ && rm -rf brpc # Install braft -git clone https://github.com/monographdb/braft.git braft +git clone https://github.com/eloqdata/braft.git braft cd braft sed -i 's/libbrpc.a//g' CMakeLists.txt mkdir bld && cd bld @@ -89,7 +108,7 @@ sudo cp ./output/lib/* /usr/lib/ cd ../../ && rm -rf braft # Install mimalloc -git clone https://github.com/monographdb/mimalloc.git mimalloc +git clone https://github.com/eloqdata/mimalloc.git mimalloc cd mimalloc git checkout eloq-v2.1.2 mkdir bld && cd bld @@ -97,7 +116,7 @@ cmake .. && make && sudo make install cd ../../ && rm -rf mimalloc # Install cuckoo filter -git clone https://github.com/monographdb/cuckoofilter.git cuckoofilter +git clone https://github.com/eloqdata/cuckoofilter.git cuckoofilter cd cuckoofilter sudo make install cd .. && rm -rf cuckoofilter @@ -239,10 +258,10 @@ sudo cmake --build cmake-out --target install cd ../ && rm -rf google-cloud-cpp # Install Google Cloud CLI -sudo apt-get install -y apt-transport-https ca-certificates gnupg curl sudo +DEBIAN_FRONTEND=noninteractive sudo apt-get install -y apt-transport-https ca-certificates gnupg curl sudo echo "deb https://packages.cloud.google.com/apt cloud-sdk main" | sudo tee -a /etc/apt/sources.list.d/google-cloud-sdk.list curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add - -sudo apt-get update && sudo apt-get install -y google-cloud-cli +DEBIAN_FRONTEND=noninteractive sudo apt-get update && DEBIAN_FRONTEND=noninteractive sudo apt-get install -y google-cloud-cli # Install FakeIt git clone https://github.com/eranpeer/FakeIt.git @@ -251,9 +270,9 @@ sudo cp single_header/catch/fakeit.hpp /usr/include/catch2/fakeit.hpp cd ../ && rm -rf FakeIt # Install RocksDB-Cloud -git clone https://github.com/monographdb/rocksdb-cloud.git +git clone https://github.com/eloqdata/rocksdb-cloud.git cd rocksdb-cloud -git checkout monographdb_main +git checkout main LIBNAME=librocksdb-cloud-aws USE_RTTI=1 USE_AWS=1 ROCKSDB_DISABLE_TCMALLOC=1 ROCKSDB_DISABLE_JEMALLOC=1 make shared_lib -j8 LIBNAME=librocksdb-cloud-aws PREFIX=$(pwd)/output make install-shared sudo mkdir -p /usr/local/include/rocksdb_cloud_header