diff --git a/concourse/pipeline/build_release_tarball.yml b/concourse/pipeline/build_release_tarball.yml index fe9f96e98bf..c6f345fa551 100644 --- a/concourse/pipeline/build_release_tarball.yml +++ b/concourse/pipeline/build_release_tarball.yml @@ -2,7 +2,7 @@ resources: - name: eloqsql_src_main type: git source: - branch: eloq-10.6.10 + branch: rel_x_x_x_eloqsql uri: git@github.com:eloqdata/eloqsql.git private_key: ((git-key)) tag_regex: '^[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+$' @@ -21,13 +21,22 @@ resources: type: registry-image source: repository: eloqdata/eloq-dev-ci-ubuntu2404 - tag: latest - name: image_centos7_main type: registry-image source: repository: eloqdata/eloq-dev-ci-centos7 +- name: image_centos8_main + type: registry-image + source: + repository: eloqdata/eloq-dev-ci-centos8 + +- name: image_rocky9_main + type: registry-image + source: + repository: eloqdata/eloq-dev-ci-rocky9 + - name: logservice_src_main type: git source: @@ -70,6 +79,7 @@ jobs: BUILD_TYPE: RelWithDebInfo DATA_STORE_TYPE: ELOQDSS_ROCKSDB_CLOUD_S3 CLOUDFRONT_DIST: ((cloudfront-dist)) + PG_CONN: ((pg-conn)) privileged: true - task: build-eloqdss-rocksdb-type @@ -85,6 +95,7 @@ jobs: BUILD_TYPE: RelWithDebInfo DATA_STORE_TYPE: ELOQDSS_ROCKSDB CLOUDFRONT_DIST: ((cloudfront-dist)) + PG_CONN: ((pg-conn)) privileged: true - task: build-eloqstore-type @@ -100,8 +111,10 @@ jobs: BUILD_TYPE: RelWithDebInfo DATA_STORE_TYPE: ELOQDSS_ELOQSTORE CLOUDFRONT_DIST: ((cloudfront-dist)) + PG_CONN: ((pg-conn)) privileged: true + serial: true - name: ubuntu2204-main @@ -131,6 +144,7 @@ jobs: BUILD_TYPE: RelWithDebInfo DATA_STORE_TYPE: ELOQDSS_ROCKSDB_CLOUD_S3 CLOUDFRONT_DIST: ((cloudfront-dist)) + PG_CONN: ((pg-conn)) privileged: true - task: build-eloqdss-rocksdb-type @@ -146,6 +160,7 @@ jobs: BUILD_TYPE: RelWithDebInfo DATA_STORE_TYPE: ELOQDSS_ROCKSDB CLOUDFRONT_DIST: ((cloudfront-dist)) + PG_CONN: ((pg-conn)) privileged: true - task: build-eloqstore-type @@ -161,6 +176,7 @@ jobs: BUILD_TYPE: RelWithDebInfo DATA_STORE_TYPE: ELOQDSS_ELOQSTORE CLOUDFRONT_DIST: ((cloudfront-dist)) + PG_CONN: ((pg-conn)) privileged: true serial: true @@ -192,6 +208,7 @@ jobs: BUILD_TYPE: RelWithDebInfo DATA_STORE_TYPE: ELOQDSS_ROCKSDB_CLOUD_S3 CLOUDFRONT_DIST: ((cloudfront-dist)) + PG_CONN: ((pg-conn)) privileged: true - task: build-eloqdss-rocksdb-type @@ -207,6 +224,7 @@ jobs: BUILD_TYPE: RelWithDebInfo DATA_STORE_TYPE: ELOQDSS_ROCKSDB CLOUDFRONT_DIST: ((cloudfront-dist)) + PG_CONN: ((pg-conn)) privileged: true - task: build-eloqstore-type @@ -222,8 +240,10 @@ jobs: BUILD_TYPE: RelWithDebInfo DATA_STORE_TYPE: ELOQDSS_ELOQSTORE CLOUDFRONT_DIST: ((cloudfront-dist)) + PG_CONN: ((pg-conn)) privileged: true + serial: true - name: centos7-main @@ -253,6 +273,7 @@ jobs: BUILD_TYPE: RelWithDebInfo DATA_STORE_TYPE: ELOQDSS_ROCKSDB_CLOUD_S3 CLOUDFRONT_DIST: ((cloudfront-dist)) + PG_CONN: ((pg-conn)) privileged: true - task: build-eloqdss-rocksdb-type @@ -268,6 +289,7 @@ jobs: BUILD_TYPE: RelWithDebInfo DATA_STORE_TYPE: ELOQDSS_ROCKSDB CLOUDFRONT_DIST: ((cloudfront-dist)) + PG_CONN: ((pg-conn)) privileged: true - task: build-eloqstore-type @@ -283,6 +305,102 @@ jobs: BUILD_TYPE: RelWithDebInfo DATA_STORE_TYPE: ELOQDSS_ELOQSTORE CLOUDFRONT_DIST: ((cloudfront-dist)) + PG_CONN: ((pg-conn)) privileged: true + + serial: true + +- name: centos8-main + plan: + - in_parallel: + limit: 2 + steps: + - get: eloqsql_src + resource: eloqsql_src_main + - get: logservice_src + resource: logservice_src_main + - get: raft_host_manager_src + resource: raft_host_manager_src_main + - get: image_centos8 + resource: image_centos8_main + + - task: build-rocksdb-cloud-s3-type + file: eloqsql_src/concourse/tasks/build_release_tarball.yml + image: image_centos8 + params: + TAGGED: "true" + GIT_SSH_KEY: ((git-key)) + AWS_ACCESS_KEY_ID: ((aws_access_key_id)) + AWS_DEFAULT_REGION: ((aws-region)) + AWS_SECRET_ACCESS_KEY: ((aws_secret_access_key)) + BUILD_LOG_SRV: "true" + BUILD_TYPE: RelWithDebInfo + DATA_STORE_TYPE: ELOQDSS_ROCKSDB_CLOUD_S3 + CLOUDFRONT_DIST: ((cloudfront-dist)) + PG_CONN: ((pg-conn)) + + - task: build-eloqdss-rocksdb-type + file: eloqsql_src/concourse/tasks/build_release_tarball.yml + image: image_centos8 + params: + TAGGED: "true" + GIT_SSH_KEY: ((git-key)) + AWS_ACCESS_KEY_ID: ((aws_access_key_id)) + AWS_DEFAULT_REGION: ((aws-region)) + AWS_SECRET_ACCESS_KEY: ((aws_secret_access_key)) + BUILD_LOG_SRV: "true" + BUILD_TYPE: RelWithDebInfo + DATA_STORE_TYPE: ELOQDSS_ROCKSDB + CLOUDFRONT_DIST: ((cloudfront-dist)) + PG_CONN: ((pg-conn)) + + + serial: true + +- name: rocky9-main + plan: + - in_parallel: + limit: 2 + steps: + - get: eloqsql_src + resource: eloqsql_src_main + - get: logservice_src + resource: logservice_src_main + - get: raft_host_manager_src + resource: raft_host_manager_src_main + - get: image_rocky9 + resource: image_rocky9_main + + - task: build-rocksdb-cloud-s3-type + file: eloqsql_src/concourse/tasks/build_release_tarball.yml + image: image_rocky9 + params: + TAGGED: "true" + GIT_SSH_KEY: ((git-key)) + AWS_ACCESS_KEY_ID: ((aws_access_key_id)) + AWS_DEFAULT_REGION: ((aws-region)) + AWS_SECRET_ACCESS_KEY: ((aws_secret_access_key)) + BUILD_LOG_SRV: "true" + BUILD_TYPE: RelWithDebInfo + DATA_STORE_TYPE: ELOQDSS_ROCKSDB_CLOUD_S3 + CLOUDFRONT_DIST: ((cloudfront-dist)) + PG_CONN: ((pg-conn)) + + - task: build-eloqdss-rocksdb-type + file: eloqsql_src/concourse/tasks/build_release_tarball.yml + image: image_rocky9 + params: + TAGGED: "true" + GIT_SSH_KEY: ((git-key)) + AWS_ACCESS_KEY_ID: ((aws_access_key_id)) + AWS_DEFAULT_REGION: ((aws-region)) + AWS_SECRET_ACCESS_KEY: ((aws_secret_access_key)) + BUILD_LOG_SRV: "true" + BUILD_TYPE: RelWithDebInfo + DATA_STORE_TYPE: ELOQDSS_ROCKSDB + CLOUDFRONT_DIST: ((cloudfront-dist)) + PG_CONN: ((pg-conn)) + + serial: true \ No newline at end of file diff --git a/concourse/scripts/build_tarball.bash b/concourse/scripts/build_tarball.bash index ca2e3d000cc..575e2db0a1a 100755 --- a/concourse/scripts/build_tarball.bash +++ b/concourse/scripts/build_tarball.bash @@ -16,13 +16,6 @@ if [ -n "${GIT_SSH_KEY}" ]; then ssh-keyscan github.com >> ~/.ssh/known_hosts 2>/dev/null || true fi -# make coredump dir writable for debugging -if [ ! -d "/var/crash" ]; then sudo mkdir -p /var/crash; fi -sudo chmod 777 /var/crash - -ulimit -c unlimited -echo '/var/crash/core.%t.%e.%p' | sudo tee /proc/sys/kernel/core_pattern - # Ensure workspace ownership sudo chown -R $current_user $HOME/workspace 2>/dev/null || true @@ -82,7 +75,7 @@ arm64 | aarch64) ARCH=arm64 ;; *) ARCH=$(uname -m) ;; esac -if [ -n "${TAGGED}" ]; then +if [ "${TAGGED}" = "true" ]; then TAGGED=$(git tag --sort=-v:refname | head -n 1) if [ -z "${TAGGED}" ]; then exit 1 @@ -105,11 +98,9 @@ S3_PREFIX="s3://${S3_BUCKET}/eloqsql" DATA_STORE_ID=$(echo ${DATA_STORE_TYPE} | tr '[:upper:]' '[:lower:]') # Normalize behavior for supported DATA_STORE_TYPE values -if [ "${DATA_STORE_TYPE}" = "ROCKSDB" ]; then - DATA_STORE_ID="rocksdb" -elif [ "${DATA_STORE_TYPE}" = "ELOQDSS_ROCKSDB_CLOUD_S3" ]; then +if [ "${DATA_STORE_TYPE}" = "ELOQDSS_ROCKSDB_CLOUD_S3" ]; then CMAKE_ARGS="${CMAKE_ARGS} -DUSE_ROCKSDB_LOG_STATE=ON -DWITH_ROCKSDB_CLOUD=S3 -DWITH_CLOUD_AZ_INFO=ON" - DATA_STORE_ID="eloqdss_s3" + DATA_STORE_ID="rocks_s3" elif [ "${DATA_STORE_TYPE}" = "ELOQDSS_ROCKSDB" ]; then DATA_STORE_ID="eloqdss_rocksdb" elif [ "${DATA_STORE_TYPE}" = "ELOQDSS_ELOQSTORE" ]; then @@ -228,10 +219,10 @@ mv launch_sv ${DEST_DIR}/bin/ cd ${HOME} tar -czvf eloqsql.tar.gz -C ${HOME} EloqSQL -if [ -n "${TAGGED}" ]; then +if [ "${TAGGED}" = "true" ]; then SQL_TARBALL="eloqsql-${TAGGED}-${OS_ID}-${ARCH}.tar.gz" eval ${INSTALL_PSQL} - SQL="INSERT INTO sql_release VALUES ('eloqsql', '${ARCH}', '${OS_ID}', '${DATA_STORE_ID}', $(echo ${TAGGED} | tr '.' ',')) ON CONFLICT DO NOTHING" + SQL="INSERT INTO tx_release VALUES ('eloqsql', '${ARCH}', '${OS_ID}', '${DATA_STORE_ID}', $(echo ${TAGGED} | tr '.' ',')) ON CONFLICT DO NOTHING" psql postgresql://${PG_CONN}/eloq_release?sslmode=require -c "${SQL}" else SQL_TARBALL="eloqsql-${OUT_NAME}-${OS_ID}-${ARCH}.tar.gz" @@ -281,7 +272,7 @@ build_upload_log_srv() { if [ "${BUILD_LOG_SRV}" = true ]; then # make and build log_service - if [ -n "${TAGGED}" ]; then + if [ "${TAGGED}" = "true" ]; then LOG_TARBALL="log-service-${TAGGED}-${OS_ID}-${ARCH}.tar.gz" else LOG_TARBALL="log-service-${OUT_NAME}-${OS_ID}-${ARCH}.tar.gz" diff --git a/concourse/tasks/build_release_tarball.yml b/concourse/tasks/build_release_tarball.yml index 0f668cf4348..9d78e42dc09 100644 --- a/concourse/tasks/build_release_tarball.yml +++ b/concourse/tasks/build_release_tarball.yml @@ -7,6 +7,8 @@ inputs: - name: raft_host_manager_src outputs: - name: the-output +params: + LD_LIBRARY_PATH: /usr/local/lib:/usr/local/lib64 run: path: /bin/bash args: