From ec5b80603a5705e82f4d1bbb7f9847d075b3139a Mon Sep 17 00:00:00 2001 From: JoJo Date: Fri, 15 Aug 2025 16:21:28 +0800 Subject: [PATCH 1/6] fix pipeline --- concourse/scripts/build_tarball.bash | 7 ------- 1 file changed, 7 deletions(-) diff --git a/concourse/scripts/build_tarball.bash b/concourse/scripts/build_tarball.bash index ca2e3d000cc..e844da958c4 100755 --- a/concourse/scripts/build_tarball.bash +++ b/concourse/scripts/build_tarball.bash @@ -16,13 +16,6 @@ if [ -n "${GIT_SSH_KEY}" ]; then ssh-keyscan github.com >> ~/.ssh/known_hosts 2>/dev/null || true fi -# make coredump dir writable for debugging -if [ ! -d "/var/crash" ]; then sudo mkdir -p /var/crash; fi -sudo chmod 777 /var/crash - -ulimit -c unlimited -echo '/var/crash/core.%t.%e.%p' | sudo tee /proc/sys/kernel/core_pattern - # Ensure workspace ownership sudo chown -R $current_user $HOME/workspace 2>/dev/null || true From 39a8130ae802a2cdd56539ae1529093531153b68 Mon Sep 17 00:00:00 2001 From: JoJo Date: Fri, 15 Aug 2025 16:52:45 +0800 Subject: [PATCH 2/6] fix --- concourse/scripts/build_tarball.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/concourse/scripts/build_tarball.bash b/concourse/scripts/build_tarball.bash index e844da958c4..a966c74ea1e 100755 --- a/concourse/scripts/build_tarball.bash +++ b/concourse/scripts/build_tarball.bash @@ -224,7 +224,7 @@ tar -czvf eloqsql.tar.gz -C ${HOME} EloqSQL if [ -n "${TAGGED}" ]; then SQL_TARBALL="eloqsql-${TAGGED}-${OS_ID}-${ARCH}.tar.gz" eval ${INSTALL_PSQL} - SQL="INSERT INTO sql_release VALUES ('eloqsql', '${ARCH}', '${OS_ID}', '${DATA_STORE_ID}', $(echo ${TAGGED} | tr '.' ',')) ON CONFLICT DO NOTHING" + SQL="INSERT INTO tx_release VALUES ('eloqsql', '${ARCH}', '${OS_ID}', '${DATA_STORE_ID}', $(echo ${TAGGED} | tr '.' ',')) ON CONFLICT DO NOTHING" psql postgresql://${PG_CONN}/eloq_release?sslmode=require -c "${SQL}" else SQL_TARBALL="eloqsql-${OUT_NAME}-${OS_ID}-${ARCH}.tar.gz" From 24fc35c06db3252308ff59687b832e348eba060c Mon Sep 17 00:00:00 2001 From: JoJo Date: Fri, 15 Aug 2025 19:03:52 +0800 Subject: [PATCH 3/6] update --- concourse/pipeline/build_release_tarball.yml | 18 +++++++++++++++++- concourse/scripts/build_tarball.bash | 6 ++---- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/concourse/pipeline/build_release_tarball.yml b/concourse/pipeline/build_release_tarball.yml index fe9f96e98bf..d3eeb230dd4 100644 --- a/concourse/pipeline/build_release_tarball.yml +++ b/concourse/pipeline/build_release_tarball.yml @@ -2,7 +2,8 @@ resources: - name: eloqsql_src_main type: git source: - branch: eloq-10.6.10 + # TODO(ZX) + branch: rel_0_4_15_eloqsql uri: git@github.com:eloqdata/eloqsql.git private_key: ((git-key)) tag_regex: '^[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+$' @@ -70,6 +71,7 @@ jobs: BUILD_TYPE: RelWithDebInfo DATA_STORE_TYPE: ELOQDSS_ROCKSDB_CLOUD_S3 CLOUDFRONT_DIST: ((cloudfront-dist)) + PG_CONN: ((pg-conn)) privileged: true - task: build-eloqdss-rocksdb-type @@ -85,6 +87,7 @@ jobs: BUILD_TYPE: RelWithDebInfo DATA_STORE_TYPE: ELOQDSS_ROCKSDB CLOUDFRONT_DIST: ((cloudfront-dist)) + PG_CONN: ((pg-conn)) privileged: true - task: build-eloqstore-type @@ -100,8 +103,10 @@ jobs: BUILD_TYPE: RelWithDebInfo DATA_STORE_TYPE: ELOQDSS_ELOQSTORE CLOUDFRONT_DIST: ((cloudfront-dist)) + PG_CONN: ((pg-conn)) privileged: true + serial: true - name: ubuntu2204-main @@ -131,6 +136,7 @@ jobs: BUILD_TYPE: RelWithDebInfo DATA_STORE_TYPE: ELOQDSS_ROCKSDB_CLOUD_S3 CLOUDFRONT_DIST: ((cloudfront-dist)) + PG_CONN: ((pg-conn)) privileged: true - task: build-eloqdss-rocksdb-type @@ -146,6 +152,7 @@ jobs: BUILD_TYPE: RelWithDebInfo DATA_STORE_TYPE: ELOQDSS_ROCKSDB CLOUDFRONT_DIST: ((cloudfront-dist)) + PG_CONN: ((pg-conn)) privileged: true - task: build-eloqstore-type @@ -161,6 +168,7 @@ jobs: BUILD_TYPE: RelWithDebInfo DATA_STORE_TYPE: ELOQDSS_ELOQSTORE CLOUDFRONT_DIST: ((cloudfront-dist)) + PG_CONN: ((pg-conn)) privileged: true serial: true @@ -192,6 +200,7 @@ jobs: BUILD_TYPE: RelWithDebInfo DATA_STORE_TYPE: ELOQDSS_ROCKSDB_CLOUD_S3 CLOUDFRONT_DIST: ((cloudfront-dist)) + PG_CONN: ((pg-conn)) privileged: true - task: build-eloqdss-rocksdb-type @@ -207,6 +216,7 @@ jobs: BUILD_TYPE: RelWithDebInfo DATA_STORE_TYPE: ELOQDSS_ROCKSDB CLOUDFRONT_DIST: ((cloudfront-dist)) + PG_CONN: ((pg-conn)) privileged: true - task: build-eloqstore-type @@ -222,8 +232,10 @@ jobs: BUILD_TYPE: RelWithDebInfo DATA_STORE_TYPE: ELOQDSS_ELOQSTORE CLOUDFRONT_DIST: ((cloudfront-dist)) + PG_CONN: ((pg-conn)) privileged: true + serial: true - name: centos7-main @@ -253,6 +265,7 @@ jobs: BUILD_TYPE: RelWithDebInfo DATA_STORE_TYPE: ELOQDSS_ROCKSDB_CLOUD_S3 CLOUDFRONT_DIST: ((cloudfront-dist)) + PG_CONN: ((pg-conn)) privileged: true - task: build-eloqdss-rocksdb-type @@ -268,6 +281,7 @@ jobs: BUILD_TYPE: RelWithDebInfo DATA_STORE_TYPE: ELOQDSS_ROCKSDB CLOUDFRONT_DIST: ((cloudfront-dist)) + PG_CONN: ((pg-conn)) privileged: true - task: build-eloqstore-type @@ -283,6 +297,8 @@ jobs: BUILD_TYPE: RelWithDebInfo DATA_STORE_TYPE: ELOQDSS_ELOQSTORE CLOUDFRONT_DIST: ((cloudfront-dist)) + PG_CONN: ((pg-conn)) privileged: true + serial: true \ No newline at end of file diff --git a/concourse/scripts/build_tarball.bash b/concourse/scripts/build_tarball.bash index a966c74ea1e..42f87805d8d 100755 --- a/concourse/scripts/build_tarball.bash +++ b/concourse/scripts/build_tarball.bash @@ -98,11 +98,9 @@ S3_PREFIX="s3://${S3_BUCKET}/eloqsql" DATA_STORE_ID=$(echo ${DATA_STORE_TYPE} | tr '[:upper:]' '[:lower:]') # Normalize behavior for supported DATA_STORE_TYPE values -if [ "${DATA_STORE_TYPE}" = "ROCKSDB" ]; then - DATA_STORE_ID="rocksdb" -elif [ "${DATA_STORE_TYPE}" = "ELOQDSS_ROCKSDB_CLOUD_S3" ]; then +if [ "${DATA_STORE_TYPE}" = "ELOQDSS_ROCKSDB_CLOUD_S3" ]; then CMAKE_ARGS="${CMAKE_ARGS} -DUSE_ROCKSDB_LOG_STATE=ON -DWITH_ROCKSDB_CLOUD=S3 -DWITH_CLOUD_AZ_INFO=ON" - DATA_STORE_ID="eloqdss_s3" + DATA_STORE_ID="rocks_s3" elif [ "${DATA_STORE_TYPE}" = "ELOQDSS_ROCKSDB" ]; then DATA_STORE_ID="eloqdss_rocksdb" elif [ "${DATA_STORE_TYPE}" = "ELOQDSS_ELOQSTORE" ]; then From e438019aaa8d7d706fc925e2060b52736f284368 Mon Sep 17 00:00:00 2001 From: JoJo Date: Fri, 15 Aug 2025 19:15:09 +0800 Subject: [PATCH 4/6] update --- concourse/scripts/build_tarball.bash | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/concourse/scripts/build_tarball.bash b/concourse/scripts/build_tarball.bash index 42f87805d8d..575e2db0a1a 100755 --- a/concourse/scripts/build_tarball.bash +++ b/concourse/scripts/build_tarball.bash @@ -75,7 +75,7 @@ arm64 | aarch64) ARCH=arm64 ;; *) ARCH=$(uname -m) ;; esac -if [ -n "${TAGGED}" ]; then +if [ "${TAGGED}" = "true" ]; then TAGGED=$(git tag --sort=-v:refname | head -n 1) if [ -z "${TAGGED}" ]; then exit 1 @@ -219,7 +219,7 @@ mv launch_sv ${DEST_DIR}/bin/ cd ${HOME} tar -czvf eloqsql.tar.gz -C ${HOME} EloqSQL -if [ -n "${TAGGED}" ]; then +if [ "${TAGGED}" = "true" ]; then SQL_TARBALL="eloqsql-${TAGGED}-${OS_ID}-${ARCH}.tar.gz" eval ${INSTALL_PSQL} SQL="INSERT INTO tx_release VALUES ('eloqsql', '${ARCH}', '${OS_ID}', '${DATA_STORE_ID}', $(echo ${TAGGED} | tr '.' ',')) ON CONFLICT DO NOTHING" @@ -272,7 +272,7 @@ build_upload_log_srv() { if [ "${BUILD_LOG_SRV}" = true ]; then # make and build log_service - if [ -n "${TAGGED}" ]; then + if [ "${TAGGED}" = "true" ]; then LOG_TARBALL="log-service-${TAGGED}-${OS_ID}-${ARCH}.tar.gz" else LOG_TARBALL="log-service-${OUT_NAME}-${OS_ID}-${ARCH}.tar.gz" From 2d52d6f6f71ba4be5397a48fcf43c43e57ba3a89 Mon Sep 17 00:00:00 2001 From: Chen Zhao Date: Wed, 20 Aug 2025 18:48:06 +0800 Subject: [PATCH 5/6] add LD_LIBRARY_PATH for build release tarball --- concourse/tasks/build_release_tarball.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/concourse/tasks/build_release_tarball.yml b/concourse/tasks/build_release_tarball.yml index 0f668cf4348..9d78e42dc09 100644 --- a/concourse/tasks/build_release_tarball.yml +++ b/concourse/tasks/build_release_tarball.yml @@ -7,6 +7,8 @@ inputs: - name: raft_host_manager_src outputs: - name: the-output +params: + LD_LIBRARY_PATH: /usr/local/lib:/usr/local/lib64 run: path: /bin/bash args: From d58d9caa7bafd4cb00271958473a774cc340402d Mon Sep 17 00:00:00 2001 From: JoJo Date: Mon, 8 Sep 2025 12:03:17 +0800 Subject: [PATCH 6/6] update --- concourse/pipeline/build_release_tarball.yml | 108 ++++++++++++++++++- 1 file changed, 105 insertions(+), 3 deletions(-) diff --git a/concourse/pipeline/build_release_tarball.yml b/concourse/pipeline/build_release_tarball.yml index d3eeb230dd4..c6f345fa551 100644 --- a/concourse/pipeline/build_release_tarball.yml +++ b/concourse/pipeline/build_release_tarball.yml @@ -2,8 +2,7 @@ resources: - name: eloqsql_src_main type: git source: - # TODO(ZX) - branch: rel_0_4_15_eloqsql + branch: rel_x_x_x_eloqsql uri: git@github.com:eloqdata/eloqsql.git private_key: ((git-key)) tag_regex: '^[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+$' @@ -22,13 +21,22 @@ resources: type: registry-image source: repository: eloqdata/eloq-dev-ci-ubuntu2404 - tag: latest - name: image_centos7_main type: registry-image source: repository: eloqdata/eloq-dev-ci-centos7 +- name: image_centos8_main + type: registry-image + source: + repository: eloqdata/eloq-dev-ci-centos8 + +- name: image_rocky9_main + type: registry-image + source: + repository: eloqdata/eloq-dev-ci-rocky9 + - name: logservice_src_main type: git source: @@ -301,4 +309,98 @@ jobs: privileged: true + serial: true + +- name: centos8-main + plan: + - in_parallel: + limit: 2 + steps: + - get: eloqsql_src + resource: eloqsql_src_main + - get: logservice_src + resource: logservice_src_main + - get: raft_host_manager_src + resource: raft_host_manager_src_main + - get: image_centos8 + resource: image_centos8_main + + - task: build-rocksdb-cloud-s3-type + file: eloqsql_src/concourse/tasks/build_release_tarball.yml + image: image_centos8 + params: + TAGGED: "true" + GIT_SSH_KEY: ((git-key)) + AWS_ACCESS_KEY_ID: ((aws_access_key_id)) + AWS_DEFAULT_REGION: ((aws-region)) + AWS_SECRET_ACCESS_KEY: ((aws_secret_access_key)) + BUILD_LOG_SRV: "true" + BUILD_TYPE: RelWithDebInfo + DATA_STORE_TYPE: ELOQDSS_ROCKSDB_CLOUD_S3 + CLOUDFRONT_DIST: ((cloudfront-dist)) + PG_CONN: ((pg-conn)) + + - task: build-eloqdss-rocksdb-type + file: eloqsql_src/concourse/tasks/build_release_tarball.yml + image: image_centos8 + params: + TAGGED: "true" + GIT_SSH_KEY: ((git-key)) + AWS_ACCESS_KEY_ID: ((aws_access_key_id)) + AWS_DEFAULT_REGION: ((aws-region)) + AWS_SECRET_ACCESS_KEY: ((aws_secret_access_key)) + BUILD_LOG_SRV: "true" + BUILD_TYPE: RelWithDebInfo + DATA_STORE_TYPE: ELOQDSS_ROCKSDB + CLOUDFRONT_DIST: ((cloudfront-dist)) + PG_CONN: ((pg-conn)) + + + serial: true + +- name: rocky9-main + plan: + - in_parallel: + limit: 2 + steps: + - get: eloqsql_src + resource: eloqsql_src_main + - get: logservice_src + resource: logservice_src_main + - get: raft_host_manager_src + resource: raft_host_manager_src_main + - get: image_rocky9 + resource: image_rocky9_main + + - task: build-rocksdb-cloud-s3-type + file: eloqsql_src/concourse/tasks/build_release_tarball.yml + image: image_rocky9 + params: + TAGGED: "true" + GIT_SSH_KEY: ((git-key)) + AWS_ACCESS_KEY_ID: ((aws_access_key_id)) + AWS_DEFAULT_REGION: ((aws-region)) + AWS_SECRET_ACCESS_KEY: ((aws_secret_access_key)) + BUILD_LOG_SRV: "true" + BUILD_TYPE: RelWithDebInfo + DATA_STORE_TYPE: ELOQDSS_ROCKSDB_CLOUD_S3 + CLOUDFRONT_DIST: ((cloudfront-dist)) + PG_CONN: ((pg-conn)) + + - task: build-eloqdss-rocksdb-type + file: eloqsql_src/concourse/tasks/build_release_tarball.yml + image: image_rocky9 + params: + TAGGED: "true" + GIT_SSH_KEY: ((git-key)) + AWS_ACCESS_KEY_ID: ((aws_access_key_id)) + AWS_DEFAULT_REGION: ((aws-region)) + AWS_SECRET_ACCESS_KEY: ((aws_secret_access_key)) + BUILD_LOG_SRV: "true" + BUILD_TYPE: RelWithDebInfo + DATA_STORE_TYPE: ELOQDSS_ROCKSDB + CLOUDFRONT_DIST: ((cloudfront-dist)) + PG_CONN: ((pg-conn)) + + serial: true \ No newline at end of file