eloqdata · JoJossd · Aug 18, 2025 · Aug 18, 2025 · Aug 18, 2025 · Aug 18, 2025
diff --git a/concourse/pipeline/build_tarball_open.yml b/concourse/pipeline/build_tarball_open.yml
@@ -0,0 +1,53 @@
+resources:
+  - name: eloqsql_src_main
+    type: git
+    source:
+      branch: main
+      uri: git@github.com:eloqdata/eloqsql.git
+      private_key: ((git-key))
+
+  - name: image_ubuntu2404_main
+    type: registry-image
+    source:
+      repository: ubuntu
+      tag: "24.04"
+
+  - name: logservice_src_main
+    type: git
+    source:
+      branch: main
+      private_key: ((git-key))
+      uri: git@github.com:eloqdata/log_service.git
+
+  - name: raft_host_manager_src_main
+    type: git
+    source:
+      branch: main
+      private_key: ((git-key))
+      uri: git@github.com:eloqdata/raft_host_manager.git
+
+jobs:
+  - name: ubuntu2404-debug-openlog
+    plan:
+      - in_parallel:
+          limit: 2
+          steps:
+            - get: eloqsql_src
+              resource: eloqsql_src_main
+            - get: logservice_src
+              resource: logservice_src_main
+            - get: raft_host_manager_src
+              resource: raft_host_manager_src_main
+            - get: image_ubuntu2404
+              resource: image_ubuntu2404_main
+
+      - task: build-debug-openlog
+        file: eloqsql_src/concourse/tasks/build_tarball_open.yml
+        image: image_ubuntu2404
+        params:
+          GIT_SSH_KEY: ((git-key))
+          BUILD_TYPE: Debug
+          OUT_NAME: debug-openlog
+
+    serial: true
+
diff --git a/concourse/scripts/build_tarball_open.bash b/concourse/scripts/build_tarball_open.bash
@@ -0,0 +1,152 @@
+#!/bin/bash
+set -exo pipefail
+
+# This script builds EloqSQL tarball with open_log_service enabled, using Debug build on Ubuntu 24.04
+# It follows the build configuration outlined in README.md, with packaging similar to build_tarball.bash
+
+export WORKSPACE=$PWD
+export AWS_PAGER=""
+export DEBIAN_FRONTEND=noninteractive
+export TZ=UTC
+
+apt-get update && apt-get install -y sudo openssh-client
+
+
+current_user=$(whoami)
+sudo chown -R "$current_user" "$PWD"
+
+# Setup SSH for private submodules if provided
+if [ -n "${GIT_SSH_KEY}" ]; then
+  mkdir -p ~/.ssh
+  echo "${GIT_SSH_KEY}" > ~/.ssh/id_rsa
+  chmod 600 ~/.ssh/id_rsa
+  ssh-keyscan github.com >> ~/.ssh/known_hosts 2>/dev/null || true
+fi
+
+# Defaults aligned with README.md
+: "${BUILD_TYPE:=Debug}"
+: "${ASAN:=OFF}"
+: "${DATA_STORE_TYPE:=ELOQDSS_ROCKSDB_CLOUD_S3}"
+: "${NCORE:=8}"
+
+DEST_DIR="${HOME}/EloqSQL"
+OUT_NAME="${OUT_NAME:-debug-openlog}"
+
+# Prepare workspace layout expected by scripts
+cd "$HOME"
+ln -sfn "${WORKSPACE}/eloqsql_src" eloqsql
+ln -sfn "${WORKSPACE}/logservice_src" logservice_src || true
+mkdir -p eloqsql/storage/eloq/tx_service
+ln -sfn "${WORKSPACE}/raft_host_manager_src" eloqsql/storage/eloq/tx_service/raft_host_manager || true
+
+ELOQSQL_SRC="${HOME}/eloqsql"
+
+# Install all dependencies using Ubuntu 24.04 script
+source /etc/os-release
+if [[ "$ID" == ubuntu* ]]; then
+  echo "Installing dependencies for Ubuntu ${VERSION_ID}..."
+  cd "$ELOQSQL_SRC"
+  bash scripts/install_dependency_ubuntu2404.sh
+
+  # Activate the Python virtual environment created by the dependency script
+  source $HOME/venv/bin/activate
+fi
+
+# Initialize submodules per README
+cd "$ELOQSQL_SRC"
+git submodule sync
+git submodule update --init --recursive
+
+# Also ensure log_service submodules
+if [ -d "storage/eloq/log_service" ]; then
+  pushd storage/eloq/log_service
+  git submodule sync
+  git submodule update --init --recursive
+  popd
+fi
+
+# Build EloqSQL per README with OPEN_LOG_SERVICE enabled
+mkdir build
+cd build
+
+cmake -DCMAKE_INSTALL_PREFIX="${DEST_DIR}" \
+      -DCMAKE_BUILD_TYPE="${BUILD_TYPE}" \
+      -DWITH_READLINE=1 \
+      -DPLUGIN_HANDLERSOCKET=NO \
+      -DPLUGIN_ROCKSDB=NO \
+      -DPLUGIN_ARIA=NO \
+      -DPLUGIN_ARCHIVE=NO \
+      -DPLUGIN_CVS=NO \
+      -DPLUGIN_FEDERATEDX=NO \
+      -DPLUGIN_TOKUDB=NO \
+      -DPLUGIN_MROONGA=NO \
+      -DPLUGIN_OQGRAPH=NO \
+      -DPLUGIN_CONNECT=NO \
+      -DPLUGIN_SPIDER=NO \
+      -DPLUGIN_SPHINX=NO \
+      -DPLUGIN_HEAP=NO \
+      -DPLUGIN_MYISAMMRG=NO \
+      -DPLUGIN_SEQUENCE=NO \
+      -DINSTALL_MYSQLTESTDIR= \
+      -DMYSQL_MAINTAINER_MODE=OFF \
+      -DWITH_SSL=system \
+      -DCOROUTINE_ENABLED=ON \
+      -DBRPC_WITH_GLOG=ON \
+      -DMARIA_WITH_GLOG=ON \
+      -DWITH_ASAN="${ASAN}" \
+      -DCMAKE_C_FLAGS_DEBUG="-O0 -g -DDBUG_ON -fno-omit-frame-pointer -fno-strict-aliasing" \
+      -DCMAKE_CXX_FLAGS_DEBUG="-O0 -g -DDBUG_ON -fno-omit-frame-pointer -fno-strict-aliasing -felide-constructors -Wno-error" \
+      -DWITH_DATA_STORE="${DATA_STORE_TYPE}" \
+      -DOPEN_LOG_SERVICE=ON \
+      ../
+
+cmake --build . --config "${BUILD_TYPE}" -j"${NCORE}"
+cmake --install . --config "${BUILD_TYPE}"
+
+# Helper to copy dependent libraries for portability
+copy_libraries() {
+  local executable="$1"
+  local path="$2"
+  libraries=$(ldd "$executable" | awk 'NF==4{print $(NF-1)}{}')
+  mkdir -p "$path"
+  for lib in $libraries; do
+    rsync -avL --ignore-existing "$lib" "$path/"
+  done
+}
+
+# Ensure runtime libs packaged
+mkdir -p "${DEST_DIR}/lib" "${DEST_DIR}/bin" "${DEST_DIR}/conf"
+copy_libraries "${DEST_DIR}/bin/mariadbd" "${DEST_DIR}/lib"
+copy_libraries "${DEST_DIR}/bin/mariadb" "${DEST_DIR}/lib"
+
+# Build and include dss_server component
+pushd "${ELOQSQL_SRC}/storage/eloq/store_handler/eloq_data_store_service"
+rm -rf bld
+mkdir bld && cd bld
+cmake -DCMAKE_BUILD_TYPE="${BUILD_TYPE}" -DWITH_DATA_STORE="${DATA_STORE_TYPE}" ../
+cmake --build . --config "${BUILD_TYPE}" -j"${NCORE}"
+copy_libraries dss_server "${DEST_DIR}/lib"
+mv dss_server "${DEST_DIR}/bin/"
+popd
+
+# Build and include log_service (launch_sv)
+pushd "${ELOQSQL_SRC}/storage/eloq/log_service"
+rm -rf bld
+mkdir bld && cd bld
+cmake -DCMAKE_BUILD_TYPE="${BUILD_TYPE}" -DUSE_ROCKSDB_LOG_STATE=ON ../
+cmake --build . --config "${BUILD_TYPE}" -j"${NCORE}"
+copy_libraries launch_sv "${DEST_DIR}/lib"
+mv launch_sv "${DEST_DIR}/bin/"
+popd
+
+# Create tarball
+cd "${HOME}"
+tar -czvf eloqsql.tar.gz -C "${HOME}" EloqSQL
+
+# Cleanup build directories
+cd "${ELOQSQL_SRC}"
+rm -rf build
+rm -rf storage/eloq/store_handler/eloq_data_store_service/bld
+rm -rf storage/eloq/log_service/bld
+
+echo "Build completed. Tarball created at: ${HOME}/eloqsql.tar.gz"
diff --git a/concourse/tasks/build_tarball_open.yml b/concourse/tasks/build_tarball_open.yml
@@ -0,0 +1,13 @@
+platform: linux
+image_resource:
+  type: registry-image
+inputs:
+  - name: eloqsql_src
+  - name: logservice_src
+  - name: raft_host_manager_src
+outputs:
+  - name: the-output
+run:
+  path: /bin/bash
+  args:
+    - eloqsql_src/concourse/scripts/build_tarball_open.bash
diff --git a/scripts/install_dependency_ubuntu2404.sh b/scripts/install_dependency_ubuntu2404.sh
@@ -2,8 +2,27 @@
 
 set -ex
 
+# Ensure noninteractive apt; keep TZ default
+export DEBIAN_FRONTEND=noninteractive
+export TZ=${TZ:-UTC}
+
+needs_tz_config=false
+if [ ! -f /etc/timezone ] || ! grep -qE '^(Etc/UTC|UTC)$' /etc/timezone; then
+  needs_tz_config=true
+fi
+if [ ! -L /etc/localtime ] || [ "$(readlink -f /etc/localtime)" != "/usr/share/zoneinfo/Etc/UTC" ]; then
+  needs_tz_config=true
+fi
+
+if $needs_tz_config; then
+  echo 'tzdata tzdata/Areas select Etc' | sudo debconf-set-selections || true
+  echo 'tzdata tzdata/Zones/Etc select UTC' | sudo debconf-set-selections || true
+  echo 'Etc/UTC' | sudo tee /etc/timezone >/dev/null
+  sudo ln -sf /usr/share/zoneinfo/Etc/UTC /etc/localtime
+fi
+
 # Install system packages
-sudo apt-get update
+DEBIAN_FRONTEND=noninteractive sudo apt-get update
 DEBIAN_FRONTEND=noninteractive sudo apt-get install -y --no-install-recommends \
     jq sudo vim wget curl apt-utils python3 python3-dev python3-pip python3-venv \
     python3-venv gdb libcurl4-openssl-dev build-essential libncurses5-dev \
@@ -49,7 +68,7 @@ sudo ldconfig
 cd ../ && rm -rf protobuf
 
 # Install glog
-git clone https://github.com/monographdb/glog.git glog
+git clone https://github.com/eloqdata/glog.git glog
 cd glog
 cmake -S . -B build -G "Unix Makefiles"
 cmake --build build -j6
@@ -65,7 +84,7 @@ make -j4 && sudo make install
 cd .. && rm -rf liburing
 
 # Install brpc
-git clone https://github.com/monographdb/brpc.git brpc
+git clone https://github.com/eloqdata/brpc.git brpc
 cd brpc
 mkdir build && cd build
 cmake .. \
@@ -78,7 +97,7 @@ sudo cp ./output/lib/* /usr/lib/
 cd ../../ && rm -rf brpc
 
 # Install braft
-git clone https://github.com/monographdb/braft.git braft
+git clone https://github.com/eloqdata/braft.git braft
 cd braft
 sed -i 's/libbrpc.a//g' CMakeLists.txt
 mkdir bld && cd bld
@@ -89,15 +108,15 @@ sudo cp ./output/lib/* /usr/lib/
 cd ../../ && rm -rf braft
 
 # Install mimalloc
-git clone https://github.com/monographdb/mimalloc.git mimalloc
+git clone https://github.com/eloqdata/mimalloc.git mimalloc
 cd mimalloc
 git checkout eloq-v2.1.2
 mkdir bld && cd bld
 cmake .. && make && sudo make install
 cd ../../ && rm -rf mimalloc
 
 # Install cuckoo filter
-git clone https://github.com/monographdb/cuckoofilter.git cuckoofilter
+git clone https://github.com/eloqdata/cuckoofilter.git cuckoofilter
 cd cuckoofilter
 sudo make install
 cd .. && rm -rf cuckoofilter
@@ -239,10 +258,10 @@ sudo cmake --build cmake-out --target install
 cd ../ && rm -rf google-cloud-cpp
 
 # Install Google Cloud CLI
-sudo apt-get install -y apt-transport-https ca-certificates gnupg curl sudo
+DEBIAN_FRONTEND=noninteractive sudo apt-get install -y apt-transport-https ca-certificates gnupg curl sudo
 echo "deb https://packages.cloud.google.com/apt cloud-sdk main" | sudo tee -a /etc/apt/sources.list.d/google-cloud-sdk.list
 curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -
-sudo apt-get update && sudo apt-get install -y google-cloud-cli
+DEBIAN_FRONTEND=noninteractive sudo apt-get update && DEBIAN_FRONTEND=noninteractive sudo apt-get install -y google-cloud-cli
 
 # Install FakeIt
 git clone https://github.com/eranpeer/FakeIt.git
@@ -251,9 +270,9 @@ sudo cp single_header/catch/fakeit.hpp /usr/include/catch2/fakeit.hpp
 cd ../ && rm -rf FakeIt
 
 # Install RocksDB-Cloud
-git clone https://github.com/monographdb/rocksdb-cloud.git
+git clone https://github.com/eloqdata/rocksdb-cloud.git
 cd rocksdb-cloud
-git checkout monographdb_main
+git checkout main
 LIBNAME=librocksdb-cloud-aws USE_RTTI=1 USE_AWS=1 ROCKSDB_DISABLE_TCMALLOC=1 ROCKSDB_DISABLE_JEMALLOC=1 make shared_lib -j8
 LIBNAME=librocksdb-cloud-aws PREFIX=$(pwd)/output make install-shared
 sudo mkdir -p /usr/local/include/rocksdb_cloud_header