diff --git a/concourse/pipeline/build_debug_tarball.yml b/concourse/pipeline/build_debug_tarball.yml index f91955386ed..bf1e45c9298 100644 --- a/concourse/pipeline/build_debug_tarball.yml +++ b/concourse/pipeline/build_debug_tarball.yml @@ -25,6 +25,13 @@ resources: private_key: ((git-key)) uri: git@github.com:eloqdata/raft_host_manager.git +- name: eloqstore_src_main + type: git + source: + branch: main + private_key: ((git-key)) + uri: git@github.com:eloqdata/eloqstore.git + jobs: - name: ubuntu2204-main plan: @@ -37,9 +44,27 @@ jobs: resource: logservice_src_main - get: raft_host_manager_src resource: raft_host_manager_src_main + - get: eloqstore_src + resource: eloqstore_src_main - get: image_ubuntu2204 resource: image_ubuntu2204_main + - task: build-eloqstore-type + file: eloqsql_src/concourse/tasks/build_debug_tarball.yml + image: image_ubuntu2204 + params: + ASAN: "ON" + TAGGED: "" + GIT_SSH_KEY: ((git-key)) + AWS_ACCESS_KEY_ID: ((aws_access_key_id)) + AWS_DEFAULT_REGION: ((aws-region)) + AWS_SECRET_ACCESS_KEY: ((aws_secret_access_key)) + BUILD_LOG_SRV: "true" + BUILD_TYPE: Debug + DATA_STORE_TYPE: ELOQDSS_ELOQSTORE + OUT_NAME: debug + CLOUDFRONT_DIST: ((cloudfront-dist)) + - task: build-rocksdb-cloud-s3-type file: eloqsql_src/concourse/tasks/build_debug_tarball.yml image: image_ubuntu2204 diff --git a/concourse/scripts/build_tarball.bash b/concourse/scripts/build_tarball.bash index 11a4ab104a7..ce4b75ec976 100755 --- a/concourse/scripts/build_tarball.bash +++ b/concourse/scripts/build_tarball.bash @@ -22,10 +22,28 @@ sudo chown -R $current_user $HOME/workspace 2>/dev/null || true cd $HOME ln -s ${WORKSPACE}/eloqsql_src eloqsql cd eloqsql -ln -s $WORKSPACE/logservice_src storage/eloq/eloq_log_service -pushd storage/eloq/tx_service +git submodule sync +git submodule update --init --recursive + +ln -s $WORKSPACE/logservice_src data_substrate/eloq_log_service +pushd data_substrate/eloq_log_service +git submodule sync +git submodule update --init --recursive +popd + +pushd data_substrate/tx_service ln -s $WORKSPACE/raft_host_manager_src raft_host_manager popd + +if [ "${DATA_STORE_TYPE}" = "ELOQDSS_ELOQSTORE" ]; then + pushd data_substrate/store_handler/eloq_data_store_service + ln -s $WORKSPACE/eloqstore_src eloqstore + cd eloqstore + git submodule sync + git submodule update --init --recursive + popd +fi + ELOQSQL_SRC=${PWD} export LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib64:$LD_LIBRARY_PATH @@ -106,8 +124,10 @@ elif [ "${DATA_STORE_TYPE}" = "ELOQDSS_ROCKSDB_CLOUD_GCS" ]; then DATA_STORE_ID="rocks_gcs" elif [ "${DATA_STORE_TYPE}" = "ELOQDSS_ROCKSDB" ]; then DATA_STORE_ID="eloqdss_rocksdb" + CMAKE_ARGS="${CMAKE_ARGS} -DWITH_LOG_STATE=ROCKSDB" elif [ "${DATA_STORE_TYPE}" = "ELOQDSS_ELOQSTORE" ]; then DATA_STORE_ID="eloqdss_eloqstore" + CMAKE_ARGS="${CMAKE_ARGS} -DWITH_LOG_STATE=ROCKSDB" else echo "Unsupported DATA_STORE_TYPE: ${DATA_STORE_TYPE}" exit 1 @@ -163,16 +183,6 @@ EOF echo "$LICENSE_CONTENT" >"${DEST_DIR}/LICENSE.txt" # build eloqsql -cd $ELOQSQL_SRC -git submodule sync -git submodule update --init --recursive - -# Init and sync submodules in required locations -cd storage/eloq/eloq_log_service -git submodule sync -git submodule update --init --recursive -cd $ELOQSQL_SRC - if [ ! -d "bld" ]; then mkdir bld; fi cd bld @@ -194,6 +204,7 @@ cmake -DCMAKE_INSTALL_PREFIX="${DEST_DIR}" \ -DIOURING_ENABLED=${IOURING_ENABLED} \ -DOPEN_LOG_SERVICE=OFF \ -DFORK_HM_PROCESS=ON \ + -DELOQ_MODULE_ENABLED=ON \ ../ cmake --build . --config ${BUILD_TYPE} -j4 @@ -211,9 +222,12 @@ else fi if [ -n "${DSS_TYPE}" ]; then - cd ${ELOQSQL_SRC}/storage/eloq/store_handler/eloq_data_store_service + cd ${ELOQSQL_SRC}/data_substrate/store_handler/eloq_data_store_service mkdir -p build && cd build - cmake .. -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DWITH_DATA_STORE=${DSS_TYPE} -DUSE_ONE_ELOQDSS_PARTITION_ENABLED=OFF + if [ "${DATA_STORE_TYPE}" = "ELOQDSS_ELOQSTORE" ]; then + DSS_CMAKE_ARGS="${DSS_CMAKE_ARGS} -DELOQ_MODULE_ENABLED=ON" + fi + cmake .. -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DWITH_DATA_STORE=${DSS_TYPE} -DUSE_ONE_ELOQDSS_PARTITION_ENABLED=OFF ${DSS_CMAKE_ARGS} cmake --build . --config ${BUILD_TYPE} -j4 copy_libraries dss_server ${DEST_DIR}/lib mv dss_server ${DEST_DIR}/bin/ @@ -221,7 +235,7 @@ if [ -n "${DSS_TYPE}" ]; then fi # Build and install log_server (launch_sv) -cd ${ELOQSQL_SRC}/storage/eloq/eloq_log_service +cd ${ELOQSQL_SRC}/data_substrate/eloq_log_service mkdir bld && cd bld cmake -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DCMAKE_EXPORT_COMPILE_COMMANDS=ON ${CMAKE_ARGS} ../ cmake --build . --config ${BUILD_TYPE} -j4 @@ -248,9 +262,9 @@ fi rm -rf eloqsql.tar.gz cd $ELOQSQL_SRC rm -rf bld -rm -rf storage/eloq/store_handler/eloq_data_store_service/bld -rm -rf storage/eloq/store_handler/eloq_data_store_service/build -rm -rf storage/eloq/eloq_log_service/bld +rm -rf data_substrate/store_handler/eloq_data_store_service/bld +rm -rf data_substrate/store_handler/eloq_data_store_service/build +rm -rf data_substrate/eloq_log_service/bld rm -rf ${DEST_DIR} build_upload_log_srv() { @@ -260,7 +274,7 @@ build_upload_log_srv() { fi local log_tarball=$1 local ds_type=$2 - log_sv_src=${ELOQSQL_SRC}/storage/eloq/eloq_log_service + log_sv_src=${ELOQSQL_SRC}/data_substrate/eloq_log_service cd ${log_sv_src} mkdir -p LogService/bin mkdir build && cd build diff --git a/concourse/tasks/build_debug_tarball.yml b/concourse/tasks/build_debug_tarball.yml index 0f668cf4348..f09886bfb9e 100644 --- a/concourse/tasks/build_debug_tarball.yml +++ b/concourse/tasks/build_debug_tarball.yml @@ -5,6 +5,7 @@ inputs: - name: eloqsql_src - name: logservice_src - name: raft_host_manager_src + - name: eloqstore_src outputs: - name: the-output run: diff --git a/data_substrate b/data_substrate index 213ff3a99fe..c5e25a81fcc 160000 --- a/data_substrate +++ b/data_substrate @@ -1 +1 @@ -Subproject commit 213ff3a99fe2074b137e58c4e8f4f13f25cf0b5e +Subproject commit c5e25a81fcc5d03e05a7f30976090c88e66802ac