From 701e17b538739ceaef696abf1a8b8d822376a171 Mon Sep 17 00:00:00 2001 From: yi-xmu Date: Thu, 4 Dec 2025 17:27:14 +0800 Subject: [PATCH 1/5] update concurse for eloqstore --- concourse/pipeline/build_debug_tarball.yml | 25 +++++++++++ concourse/scripts/build_tarball.bash | 51 ++++++++++++++-------- concourse/tasks/build_debug_tarball.yml | 1 + 3 files changed, 58 insertions(+), 19 deletions(-) diff --git a/concourse/pipeline/build_debug_tarball.yml b/concourse/pipeline/build_debug_tarball.yml index f91955386ed..bf1e45c9298 100644 --- a/concourse/pipeline/build_debug_tarball.yml +++ b/concourse/pipeline/build_debug_tarball.yml @@ -25,6 +25,13 @@ resources: private_key: ((git-key)) uri: git@github.com:eloqdata/raft_host_manager.git +- name: eloqstore_src_main + type: git + source: + branch: main + private_key: ((git-key)) + uri: git@github.com:eloqdata/eloqstore.git + jobs: - name: ubuntu2204-main plan: @@ -37,9 +44,27 @@ jobs: resource: logservice_src_main - get: raft_host_manager_src resource: raft_host_manager_src_main + - get: eloqstore_src + resource: eloqstore_src_main - get: image_ubuntu2204 resource: image_ubuntu2204_main + - task: build-eloqstore-type + file: eloqsql_src/concourse/tasks/build_debug_tarball.yml + image: image_ubuntu2204 + params: + ASAN: "ON" + TAGGED: "" + GIT_SSH_KEY: ((git-key)) + AWS_ACCESS_KEY_ID: ((aws_access_key_id)) + AWS_DEFAULT_REGION: ((aws-region)) + AWS_SECRET_ACCESS_KEY: ((aws_secret_access_key)) + BUILD_LOG_SRV: "true" + BUILD_TYPE: Debug + DATA_STORE_TYPE: ELOQDSS_ELOQSTORE + OUT_NAME: debug + CLOUDFRONT_DIST: ((cloudfront-dist)) + - task: build-rocksdb-cloud-s3-type file: eloqsql_src/concourse/tasks/build_debug_tarball.yml image: image_ubuntu2204 diff --git a/concourse/scripts/build_tarball.bash b/concourse/scripts/build_tarball.bash index 11a4ab104a7..c4973bc6637 100755 --- a/concourse/scripts/build_tarball.bash +++ b/concourse/scripts/build_tarball.bash @@ -22,10 +22,28 @@ sudo chown -R $current_user $HOME/workspace 2>/dev/null || true cd $HOME ln -s ${WORKSPACE}/eloqsql_src eloqsql cd eloqsql -ln -s $WORKSPACE/logservice_src storage/eloq/eloq_log_service -pushd storage/eloq/tx_service +git submodule sync +git submodule update --init --recursive + +ln -s $WORKSPACE/logservice_src data_substrate/eloq_log_service +pushd data_substrate/eloq_log_service +git submodule sync +git submodule update --init --recursive +popd + +pushd data_substrate/tx_service ln -s $WORKSPACE/raft_host_manager_src raft_host_manager popd + +if [ "${DATA_STORE_TYPE}" = "ELOQDSS_ELOQSTORE" ]; then + pushd data_substrate/store_handler/eloq_data_store_service + ln -s $WORKSPACE/eloqstore_src eloqstore + cd eloqstore + git submodule sync + git submodule update --init --recursive + popd +fi + ELOQSQL_SRC=${PWD} export LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib64:$LD_LIBRARY_PATH @@ -163,16 +181,6 @@ EOF echo "$LICENSE_CONTENT" >"${DEST_DIR}/LICENSE.txt" # build eloqsql -cd $ELOQSQL_SRC -git submodule sync -git submodule update --init --recursive - -# Init and sync submodules in required locations -cd storage/eloq/eloq_log_service -git submodule sync -git submodule update --init --recursive -cd $ELOQSQL_SRC - if [ ! -d "bld" ]; then mkdir bld; fi cd bld @@ -194,6 +202,8 @@ cmake -DCMAKE_INSTALL_PREFIX="${DEST_DIR}" \ -DIOURING_ENABLED=${IOURING_ENABLED} \ -DOPEN_LOG_SERVICE=OFF \ -DFORK_HM_PROCESS=ON \ + -DELOQ_MODULE_ENABLED=ON \ + -DWITH_LOG_STATE=ROCKSDB \ ../ cmake --build . --config ${BUILD_TYPE} -j4 @@ -211,9 +221,12 @@ else fi if [ -n "${DSS_TYPE}" ]; then - cd ${ELOQSQL_SRC}/storage/eloq/store_handler/eloq_data_store_service + cd ${ELOQSQL_SRC}/data_substrate/store_handler/eloq_data_store_service mkdir -p build && cd build - cmake .. -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DWITH_DATA_STORE=${DSS_TYPE} -DUSE_ONE_ELOQDSS_PARTITION_ENABLED=OFF + if [ "${DATA_STORE_TYPE}" = "ELOQDSS_ELOQSTORE" ]; then + DSS_CMAKE_ARGS="${DSS_CMAKE_ARGS} -DELOQ_MODULE_ENABLED=ON" + fi + cmake .. -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DWITH_DATA_STORE=${DSS_TYPE} -DUSE_ONE_ELOQDSS_PARTITION_ENABLED=OFF ${DSS_CMAKE_ARGS} cmake --build . --config ${BUILD_TYPE} -j4 copy_libraries dss_server ${DEST_DIR}/lib mv dss_server ${DEST_DIR}/bin/ @@ -221,7 +234,7 @@ if [ -n "${DSS_TYPE}" ]; then fi # Build and install log_server (launch_sv) -cd ${ELOQSQL_SRC}/storage/eloq/eloq_log_service +cd ${ELOQSQL_SRC}/data_substrate/eloq_log_service mkdir bld && cd bld cmake -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DCMAKE_EXPORT_COMPILE_COMMANDS=ON ${CMAKE_ARGS} ../ cmake --build . --config ${BUILD_TYPE} -j4 @@ -248,9 +261,9 @@ fi rm -rf eloqsql.tar.gz cd $ELOQSQL_SRC rm -rf bld -rm -rf storage/eloq/store_handler/eloq_data_store_service/bld -rm -rf storage/eloq/store_handler/eloq_data_store_service/build -rm -rf storage/eloq/eloq_log_service/bld +rm -rf data_substrate/store_handler/eloq_data_store_service/bld +rm -rf data_substrate/store_handler/eloq_data_store_service/build +rm -rf data_substrate/eloq_log_service/bld rm -rf ${DEST_DIR} build_upload_log_srv() { @@ -260,7 +273,7 @@ build_upload_log_srv() { fi local log_tarball=$1 local ds_type=$2 - log_sv_src=${ELOQSQL_SRC}/storage/eloq/eloq_log_service + log_sv_src=${ELOQSQL_SRC}/data_substrate/eloq_log_service cd ${log_sv_src} mkdir -p LogService/bin mkdir build && cd build diff --git a/concourse/tasks/build_debug_tarball.yml b/concourse/tasks/build_debug_tarball.yml index 0f668cf4348..f09886bfb9e 100644 --- a/concourse/tasks/build_debug_tarball.yml +++ b/concourse/tasks/build_debug_tarball.yml @@ -5,6 +5,7 @@ inputs: - name: eloqsql_src - name: logservice_src - name: raft_host_manager_src + - name: eloqstore_src outputs: - name: the-output run: From 90d103bfffe2b8f4977edbdf7b82b62a7be69c02 Mon Sep 17 00:00:00 2001 From: yi-xmu Date: Thu, 4 Dec 2025 17:27:25 +0800 Subject: [PATCH 2/5] update subm --- data_substrate | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data_substrate b/data_substrate index 213ff3a99fe..2f9da70fd5f 160000 --- a/data_substrate +++ b/data_substrate @@ -1 +1 @@ -Subproject commit 213ff3a99fe2074b137e58c4e8f4f13f25cf0b5e +Subproject commit 2f9da70fd5fa45aaa9a48d25889e5e2008e70e01 From 83aa0ce80e7024987383b9e6c22c1dd8d0634163 Mon Sep 17 00:00:00 2001 From: yi-xmu Date: Thu, 4 Dec 2025 17:57:29 +0800 Subject: [PATCH 3/5] update bash --- concourse/scripts/build_tarball.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/concourse/scripts/build_tarball.bash b/concourse/scripts/build_tarball.bash index c4973bc6637..17085d29b11 100755 --- a/concourse/scripts/build_tarball.bash +++ b/concourse/scripts/build_tarball.bash @@ -126,6 +126,7 @@ elif [ "${DATA_STORE_TYPE}" = "ELOQDSS_ROCKSDB" ]; then DATA_STORE_ID="eloqdss_rocksdb" elif [ "${DATA_STORE_TYPE}" = "ELOQDSS_ELOQSTORE" ]; then DATA_STORE_ID="eloqdss_eloqstore" + CMAKE_ARGS="${CMAKE_ARGS} -DWITH_LOG_STATE=ROCKSDB" else echo "Unsupported DATA_STORE_TYPE: ${DATA_STORE_TYPE}" exit 1 @@ -203,7 +204,6 @@ cmake -DCMAKE_INSTALL_PREFIX="${DEST_DIR}" \ -DOPEN_LOG_SERVICE=OFF \ -DFORK_HM_PROCESS=ON \ -DELOQ_MODULE_ENABLED=ON \ - -DWITH_LOG_STATE=ROCKSDB \ ../ cmake --build . --config ${BUILD_TYPE} -j4 From b010d3ddd9707ac9fcf7158cdd25e777a6314f4a Mon Sep 17 00:00:00 2001 From: yi-xmu Date: Thu, 4 Dec 2025 18:35:14 +0800 Subject: [PATCH 4/5] update bash --- concourse/scripts/build_tarball.bash | 1 + 1 file changed, 1 insertion(+) diff --git a/concourse/scripts/build_tarball.bash b/concourse/scripts/build_tarball.bash index 17085d29b11..ce4b75ec976 100755 --- a/concourse/scripts/build_tarball.bash +++ b/concourse/scripts/build_tarball.bash @@ -124,6 +124,7 @@ elif [ "${DATA_STORE_TYPE}" = "ELOQDSS_ROCKSDB_CLOUD_GCS" ]; then DATA_STORE_ID="rocks_gcs" elif [ "${DATA_STORE_TYPE}" = "ELOQDSS_ROCKSDB" ]; then DATA_STORE_ID="eloqdss_rocksdb" + CMAKE_ARGS="${CMAKE_ARGS} -DWITH_LOG_STATE=ROCKSDB" elif [ "${DATA_STORE_TYPE}" = "ELOQDSS_ELOQSTORE" ]; then DATA_STORE_ID="eloqdss_eloqstore" CMAKE_ARGS="${CMAKE_ARGS} -DWITH_LOG_STATE=ROCKSDB" From aafd341c79caf7c621de4887330476c7d63f2421 Mon Sep 17 00:00:00 2001 From: yi-xmu Date: Sat, 6 Dec 2025 15:11:47 +0800 Subject: [PATCH 5/5] update submodule for removing rclone --- data_substrate | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data_substrate b/data_substrate index 2f9da70fd5f..c5e25a81fcc 160000 --- a/data_substrate +++ b/data_substrate @@ -1 +1 @@ -Subproject commit 2f9da70fd5fa45aaa9a48d25889e5e2008e70e01 +Subproject commit c5e25a81fcc5d03e05a7f30976090c88e66802ac