From 02fe14b001aeee9126d969bd77ec9290d644138e Mon Sep 17 00:00:00 2001 From: JoJo Date: Fri, 15 Aug 2025 16:21:28 +0800 Subject: [PATCH 1/6] fix pipeline --- concourse/scripts/build_tarball.bash | 7 ------- 1 file changed, 7 deletions(-) diff --git a/concourse/scripts/build_tarball.bash b/concourse/scripts/build_tarball.bash index 2fae7f1bb9d..bac3998c7d1 100755 --- a/concourse/scripts/build_tarball.bash +++ b/concourse/scripts/build_tarball.bash @@ -16,13 +16,6 @@ if [ -n "${GIT_SSH_KEY}" ]; then ssh-keyscan github.com >> ~/.ssh/known_hosts 2>/dev/null || true fi -# make coredump dir writable for debugging -if [ ! -d "/var/crash" ]; then sudo mkdir -p /var/crash; fi -sudo chmod 777 /var/crash - -ulimit -c unlimited -echo '/var/crash/core.%t.%e.%p' | sudo tee /proc/sys/kernel/core_pattern - # Ensure workspace ownership sudo chown -R $current_user $HOME/workspace 2>/dev/null || true From 3d5bed369d58d67f1d7ba89df02d4618e6bfc23b Mon Sep 17 00:00:00 2001 From: JoJo Date: Fri, 15 Aug 2025 16:52:45 +0800 Subject: [PATCH 2/6] fix --- concourse/scripts/build_tarball.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/concourse/scripts/build_tarball.bash b/concourse/scripts/build_tarball.bash index bac3998c7d1..618583cb4cb 100755 --- a/concourse/scripts/build_tarball.bash +++ b/concourse/scripts/build_tarball.bash @@ -215,7 +215,7 @@ tar -czvf eloqsql.tar.gz -C ${HOME} EloqSQL if [ -n "${TAGGED}" ]; then SQL_TARBALL="eloqsql-${TAGGED}-${OS_ID}-${ARCH}.tar.gz" eval ${INSTALL_PSQL} - SQL="INSERT INTO sql_release VALUES ('eloqsql', '${ARCH}', '${OS_ID}', '${DATA_STORE_ID}', $(echo ${TAGGED} | tr '.' ',')) ON CONFLICT DO NOTHING" + SQL="INSERT INTO tx_release VALUES ('eloqsql', '${ARCH}', '${OS_ID}', '${DATA_STORE_ID}', $(echo ${TAGGED} | tr '.' ',')) ON CONFLICT DO NOTHING" psql postgresql://${PG_CONN}/eloq_release?sslmode=require -c "${SQL}" else SQL_TARBALL="eloqsql-${OUT_NAME}-${OS_ID}-${ARCH}.tar.gz" From 83c78b7f0ae15ed90c26c8a157930cb7557d7a6d Mon Sep 17 00:00:00 2001 From: JoJo Date: Fri, 15 Aug 2025 19:03:52 +0800 Subject: [PATCH 3/6] update --- concourse/pipeline/build_release_tarball.yml | 64 +++----------------- concourse/scripts/build_tarball.bash | 6 +- 2 files changed, 12 insertions(+), 58 deletions(-) diff --git a/concourse/pipeline/build_release_tarball.yml b/concourse/pipeline/build_release_tarball.yml index d7f378fd21c..73d6054eaeb 100644 --- a/concourse/pipeline/build_release_tarball.yml +++ b/concourse/pipeline/build_release_tarball.yml @@ -2,7 +2,8 @@ resources: - name: eloqsql_src_main type: git source: - branch: eloq-10.6.10 + # TODO(ZX) + branch: rel_0_4_15_eloqsql uri: git@github.com:eloqdata/eloqsql.git private_key: ((git-key)) tag_regex: '^[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+$' @@ -70,6 +71,7 @@ jobs: BUILD_TYPE: RelWithDebInfo DATA_STORE_TYPE: ELOQDSS_ROCKSDB_CLOUD_S3 CLOUDFRONT_DIST: ((cloudfront-dist)) + PG_CONN: ((pg-conn)) - task: build-eloqdss-rocksdb-type file: eloqsql_src/concourse/tasks/build_release_tarball.yml @@ -84,20 +86,8 @@ jobs: BUILD_TYPE: RelWithDebInfo DATA_STORE_TYPE: ELOQDSS_ROCKSDB CLOUDFRONT_DIST: ((cloudfront-dist)) + PG_CONN: ((pg-conn)) - - task: build-eloqstore-type - file: eloqsql_src/concourse/tasks/build_release_tarball.yml - image: image_ubuntu2004 - params: - TAGGED: "true" - GIT_SSH_KEY: ((git-key)) - AWS_ACCESS_KEY_ID: ((aws_access_key_id)) - AWS_DEFAULT_REGION: ((aws-region)) - AWS_SECRET_ACCESS_KEY: ((aws_secret_access_key)) - BUILD_LOG_SRV: "true" - BUILD_TYPE: RelWithDebInfo - DATA_STORE_TYPE: ELOQDSS_ELOQSTORE - CLOUDFRONT_DIST: ((cloudfront-dist)) serial: true @@ -128,6 +118,7 @@ jobs: BUILD_TYPE: RelWithDebInfo DATA_STORE_TYPE: ELOQDSS_ROCKSDB_CLOUD_S3 CLOUDFRONT_DIST: ((cloudfront-dist)) + PG_CONN: ((pg-conn)) - task: build-eloqdss-rocksdb-type file: eloqsql_src/concourse/tasks/build_release_tarball.yml @@ -142,20 +133,7 @@ jobs: BUILD_TYPE: RelWithDebInfo DATA_STORE_TYPE: ELOQDSS_ROCKSDB CLOUDFRONT_DIST: ((cloudfront-dist)) - - - task: build-eloqstore-type - file: eloqsql_src/concourse/tasks/build_release_tarball.yml - image: image_ubuntu2204 - params: - TAGGED: "true" - GIT_SSH_KEY: ((git-key)) - AWS_ACCESS_KEY_ID: ((aws_access_key_id)) - AWS_DEFAULT_REGION: ((aws-region)) - AWS_SECRET_ACCESS_KEY: ((aws_secret_access_key)) - BUILD_LOG_SRV: "true" - BUILD_TYPE: RelWithDebInfo - DATA_STORE_TYPE: ELOQDSS_ELOQSTORE - CLOUDFRONT_DIST: ((cloudfront-dist)) + PG_CONN: ((pg-conn)) serial: true @@ -186,6 +164,7 @@ jobs: BUILD_TYPE: RelWithDebInfo DATA_STORE_TYPE: ELOQDSS_ROCKSDB_CLOUD_S3 CLOUDFRONT_DIST: ((cloudfront-dist)) + PG_CONN: ((pg-conn)) - task: build-eloqdss-rocksdb-type file: eloqsql_src/concourse/tasks/build_release_tarball.yml @@ -200,20 +179,8 @@ jobs: BUILD_TYPE: RelWithDebInfo DATA_STORE_TYPE: ELOQDSS_ROCKSDB CLOUDFRONT_DIST: ((cloudfront-dist)) + PG_CONN: ((pg-conn)) - - task: build-eloqstore-type - file: eloqsql_src/concourse/tasks/build_release_tarball.yml - image: image_ubuntu2404 - params: - TAGGED: "true" - GIT_SSH_KEY: ((git-key)) - AWS_ACCESS_KEY_ID: ((aws_access_key_id)) - AWS_DEFAULT_REGION: ((aws-region)) - AWS_SECRET_ACCESS_KEY: ((aws_secret_access_key)) - BUILD_LOG_SRV: "true" - BUILD_TYPE: RelWithDebInfo - DATA_STORE_TYPE: ELOQDSS_ELOQSTORE - CLOUDFRONT_DIST: ((cloudfront-dist)) serial: true @@ -244,6 +211,7 @@ jobs: BUILD_TYPE: RelWithDebInfo DATA_STORE_TYPE: ELOQDSS_ROCKSDB_CLOUD_S3 CLOUDFRONT_DIST: ((cloudfront-dist)) + PG_CONN: ((pg-conn)) - task: build-eloqdss-rocksdb-type file: eloqsql_src/concourse/tasks/build_release_tarball.yml @@ -258,19 +226,7 @@ jobs: BUILD_TYPE: RelWithDebInfo DATA_STORE_TYPE: ELOQDSS_ROCKSDB CLOUDFRONT_DIST: ((cloudfront-dist)) + PG_CONN: ((pg-conn)) - - task: build-eloqstore-type - file: eloqsql_src/concourse/tasks/build_release_tarball.yml - image: image_centos7 - params: - TAGGED: "true" - GIT_SSH_KEY: ((git-key)) - AWS_ACCESS_KEY_ID: ((aws_access_key_id)) - AWS_DEFAULT_REGION: ((aws-region)) - AWS_SECRET_ACCESS_KEY: ((aws_secret_access_key)) - BUILD_LOG_SRV: "true" - BUILD_TYPE: RelWithDebInfo - DATA_STORE_TYPE: ELOQDSS_ELOQSTORE - CLOUDFRONT_DIST: ((cloudfront-dist)) serial: true \ No newline at end of file diff --git a/concourse/scripts/build_tarball.bash b/concourse/scripts/build_tarball.bash index 618583cb4cb..83fcf9fb6cb 100755 --- a/concourse/scripts/build_tarball.bash +++ b/concourse/scripts/build_tarball.bash @@ -96,11 +96,9 @@ S3_PREFIX="s3://${S3_BUCKET}/eloqsql" DATA_STORE_ID=$(echo ${DATA_STORE_TYPE} | tr '[:upper:]' '[:lower:]') # Normalize behavior for supported DATA_STORE_TYPE values -if [ "${DATA_STORE_TYPE}" = "ROCKSDB" ]; then - DATA_STORE_ID="rocksdb" -elif [ "${DATA_STORE_TYPE}" = "ELOQDSS_ROCKSDB_CLOUD_S3" ]; then +if [ "${DATA_STORE_TYPE}" = "ELOQDSS_ROCKSDB_CLOUD_S3" ]; then CMAKE_ARGS="${CMAKE_ARGS} -DUSE_ROCKSDB_LOG_STATE=ON -DWITH_ROCKSDB_CLOUD=S3 -DWITH_CLOUD_AZ_INFO=ON" - DATA_STORE_ID="eloqdss_s3" + DATA_STORE_ID="rocks_s3" elif [ "${DATA_STORE_TYPE}" = "ELOQDSS_ROCKSDB" ]; then DATA_STORE_ID="eloqdss_rocksdb" elif [ "${DATA_STORE_TYPE}" = "ELOQDSS_ELOQSTORE" ]; then From 6e202dd431d6f542be510faf7c8aa2e3284fa7c5 Mon Sep 17 00:00:00 2001 From: JoJo Date: Fri, 15 Aug 2025 19:15:09 +0800 Subject: [PATCH 4/6] update --- concourse/scripts/build_tarball.bash | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/concourse/scripts/build_tarball.bash b/concourse/scripts/build_tarball.bash index 83fcf9fb6cb..0d8f0f7e980 100755 --- a/concourse/scripts/build_tarball.bash +++ b/concourse/scripts/build_tarball.bash @@ -73,7 +73,7 @@ arm64 | aarch64) ARCH=arm64 ;; *) ARCH=$(uname -m) ;; esac -if [ -n "${TAGGED}" ]; then +if [ "${TAGGED}" = "true" ]; then TAGGED=$(git tag --sort=-v:refname | head -n 1) if [ -z "${TAGGED}" ]; then exit 1 @@ -210,7 +210,7 @@ mv launch_sv ${DEST_DIR}/bin/ cd ${HOME} tar -czvf eloqsql.tar.gz -C ${HOME} EloqSQL -if [ -n "${TAGGED}" ]; then +if [ "${TAGGED}" = "true" ]; then SQL_TARBALL="eloqsql-${TAGGED}-${OS_ID}-${ARCH}.tar.gz" eval ${INSTALL_PSQL} SQL="INSERT INTO tx_release VALUES ('eloqsql', '${ARCH}', '${OS_ID}', '${DATA_STORE_ID}', $(echo ${TAGGED} | tr '.' ',')) ON CONFLICT DO NOTHING" @@ -263,7 +263,7 @@ build_upload_log_srv() { if [ "${BUILD_LOG_SRV}" = true ]; then # make and build log_service - if [ -n "${TAGGED}" ]; then + if [ "${TAGGED}" = "true" ]; then LOG_TARBALL="log-service-${TAGGED}-${OS_ID}-${ARCH}.tar.gz" else LOG_TARBALL="log-service-${OUT_NAME}-${OS_ID}-${ARCH}.tar.gz" From 78661c027f132dcda9c272d9390e3f8a2f67ae51 Mon Sep 17 00:00:00 2001 From: Chen Zhao Date: Wed, 20 Aug 2025 18:48:06 +0800 Subject: [PATCH 5/6] add LD_LIBRARY_PATH for build release tarball --- concourse/tasks/build_release_tarball.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/concourse/tasks/build_release_tarball.yml b/concourse/tasks/build_release_tarball.yml index 0f668cf4348..9d78e42dc09 100644 --- a/concourse/tasks/build_release_tarball.yml +++ b/concourse/tasks/build_release_tarball.yml @@ -7,6 +7,8 @@ inputs: - name: raft_host_manager_src outputs: - name: the-output +params: + LD_LIBRARY_PATH: /usr/local/lib:/usr/local/lib64 run: path: /bin/bash args: From 746ba2a920126603d0981eb07271e341fbd4cc12 Mon Sep 17 00:00:00 2001 From: JoJo Date: Mon, 8 Sep 2025 12:03:17 +0800 Subject: [PATCH 6/6] update --- concourse/pipeline/build_release_tarball.yml | 112 ++++++++++++++++++- 1 file changed, 107 insertions(+), 5 deletions(-) diff --git a/concourse/pipeline/build_release_tarball.yml b/concourse/pipeline/build_release_tarball.yml index 73d6054eaeb..556c9a7f8f4 100644 --- a/concourse/pipeline/build_release_tarball.yml +++ b/concourse/pipeline/build_release_tarball.yml @@ -2,8 +2,7 @@ resources: - name: eloqsql_src_main type: git source: - # TODO(ZX) - branch: rel_0_4_15_eloqsql + branch: rel_x_x_x_eloqsql uri: git@github.com:eloqdata/eloqsql.git private_key: ((git-key)) tag_regex: '^[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+$' @@ -21,13 +20,22 @@ resources: - name: image_ubuntu2404_main type: registry-image source: - repository: eloqdata/monograph-dev-ci-ubuntu2404 - tag: latest + repository: eloqdata/eloq-dev-ci-ubuntu2404 - name: image_centos7_main type: registry-image source: - repository: eloqdata/monograph-dev-ci-centos7 + repository: eloqdata/eloq-dev-ci-centos7 + +- name: image_centos8_main + type: registry-image + source: + repository: eloqdata/eloq-dev-ci-centos8 + +- name: image_rocky9_main + type: registry-image + source: + repository: eloqdata/eloq-dev-ci-rocky9 - name: logservice_src_main type: git @@ -229,4 +237,98 @@ jobs: PG_CONN: ((pg-conn)) + serial: true + +- name: centos8-main + plan: + - in_parallel: + limit: 2 + steps: + - get: eloqsql_src + resource: eloqsql_src_main + - get: logservice_src + resource: logservice_src_main + - get: raft_host_manager_src + resource: raft_host_manager_src_main + - get: image_centos8 + resource: image_centos8_main + + - task: build-rocksdb-cloud-s3-type + file: eloqsql_src/concourse/tasks/build_release_tarball.yml + image: image_centos8 + params: + TAGGED: "true" + GIT_SSH_KEY: ((git-key)) + AWS_ACCESS_KEY_ID: ((aws_access_key_id)) + AWS_DEFAULT_REGION: ((aws-region)) + AWS_SECRET_ACCESS_KEY: ((aws_secret_access_key)) + BUILD_LOG_SRV: "true" + BUILD_TYPE: RelWithDebInfo + DATA_STORE_TYPE: ELOQDSS_ROCKSDB_CLOUD_S3 + CLOUDFRONT_DIST: ((cloudfront-dist)) + PG_CONN: ((pg-conn)) + + - task: build-eloqdss-rocksdb-type + file: eloqsql_src/concourse/tasks/build_release_tarball.yml + image: image_centos8 + params: + TAGGED: "true" + GIT_SSH_KEY: ((git-key)) + AWS_ACCESS_KEY_ID: ((aws_access_key_id)) + AWS_DEFAULT_REGION: ((aws-region)) + AWS_SECRET_ACCESS_KEY: ((aws_secret_access_key)) + BUILD_LOG_SRV: "true" + BUILD_TYPE: RelWithDebInfo + DATA_STORE_TYPE: ELOQDSS_ROCKSDB + CLOUDFRONT_DIST: ((cloudfront-dist)) + PG_CONN: ((pg-conn)) + + + serial: true + +- name: rocky9-main + plan: + - in_parallel: + limit: 2 + steps: + - get: eloqsql_src + resource: eloqsql_src_main + - get: logservice_src + resource: logservice_src_main + - get: raft_host_manager_src + resource: raft_host_manager_src_main + - get: image_rocky9 + resource: image_rocky9_main + + - task: build-rocksdb-cloud-s3-type + file: eloqsql_src/concourse/tasks/build_release_tarball.yml + image: image_rocky9 + params: + TAGGED: "true" + GIT_SSH_KEY: ((git-key)) + AWS_ACCESS_KEY_ID: ((aws_access_key_id)) + AWS_DEFAULT_REGION: ((aws-region)) + AWS_SECRET_ACCESS_KEY: ((aws_secret_access_key)) + BUILD_LOG_SRV: "true" + BUILD_TYPE: RelWithDebInfo + DATA_STORE_TYPE: ELOQDSS_ROCKSDB_CLOUD_S3 + CLOUDFRONT_DIST: ((cloudfront-dist)) + PG_CONN: ((pg-conn)) + + - task: build-eloqdss-rocksdb-type + file: eloqsql_src/concourse/tasks/build_release_tarball.yml + image: image_rocky9 + params: + TAGGED: "true" + GIT_SSH_KEY: ((git-key)) + AWS_ACCESS_KEY_ID: ((aws_access_key_id)) + AWS_DEFAULT_REGION: ((aws-region)) + AWS_SECRET_ACCESS_KEY: ((aws_secret_access_key)) + BUILD_LOG_SRV: "true" + BUILD_TYPE: RelWithDebInfo + DATA_STORE_TYPE: ELOQDSS_ROCKSDB + CLOUDFRONT_DIST: ((cloudfront-dist)) + PG_CONN: ((pg-conn)) + + serial: true \ No newline at end of file