diff --git a/concourse/pipeline/build_release_tarball.yml b/concourse/pipeline/build_release_tarball.yml index d7f378fd21c..556c9a7f8f4 100644 --- a/concourse/pipeline/build_release_tarball.yml +++ b/concourse/pipeline/build_release_tarball.yml @@ -2,7 +2,7 @@ resources: - name: eloqsql_src_main type: git source: - branch: eloq-10.6.10 + branch: rel_x_x_x_eloqsql uri: git@github.com:eloqdata/eloqsql.git private_key: ((git-key)) tag_regex: '^[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+$' @@ -20,13 +20,22 @@ resources: - name: image_ubuntu2404_main type: registry-image source: - repository: eloqdata/monograph-dev-ci-ubuntu2404 - tag: latest + repository: eloqdata/eloq-dev-ci-ubuntu2404 - name: image_centos7_main type: registry-image source: - repository: eloqdata/monograph-dev-ci-centos7 + repository: eloqdata/eloq-dev-ci-centos7 + +- name: image_centos8_main + type: registry-image + source: + repository: eloqdata/eloq-dev-ci-centos8 + +- name: image_rocky9_main + type: registry-image + source: + repository: eloqdata/eloq-dev-ci-rocky9 - name: logservice_src_main type: git @@ -70,6 +79,7 @@ jobs: BUILD_TYPE: RelWithDebInfo DATA_STORE_TYPE: ELOQDSS_ROCKSDB_CLOUD_S3 CLOUDFRONT_DIST: ((cloudfront-dist)) + PG_CONN: ((pg-conn)) - task: build-eloqdss-rocksdb-type file: eloqsql_src/concourse/tasks/build_release_tarball.yml @@ -84,20 +94,8 @@ jobs: BUILD_TYPE: RelWithDebInfo DATA_STORE_TYPE: ELOQDSS_ROCKSDB CLOUDFRONT_DIST: ((cloudfront-dist)) + PG_CONN: ((pg-conn)) - - task: build-eloqstore-type - file: eloqsql_src/concourse/tasks/build_release_tarball.yml - image: image_ubuntu2004 - params: - TAGGED: "true" - GIT_SSH_KEY: ((git-key)) - AWS_ACCESS_KEY_ID: ((aws_access_key_id)) - AWS_DEFAULT_REGION: ((aws-region)) - AWS_SECRET_ACCESS_KEY: ((aws_secret_access_key)) - BUILD_LOG_SRV: "true" - BUILD_TYPE: RelWithDebInfo - DATA_STORE_TYPE: ELOQDSS_ELOQSTORE - CLOUDFRONT_DIST: ((cloudfront-dist)) serial: true @@ -128,6 +126,7 @@ jobs: BUILD_TYPE: RelWithDebInfo DATA_STORE_TYPE: ELOQDSS_ROCKSDB_CLOUD_S3 CLOUDFRONT_DIST: ((cloudfront-dist)) + PG_CONN: ((pg-conn)) - task: build-eloqdss-rocksdb-type file: eloqsql_src/concourse/tasks/build_release_tarball.yml @@ -142,20 +141,7 @@ jobs: BUILD_TYPE: RelWithDebInfo DATA_STORE_TYPE: ELOQDSS_ROCKSDB CLOUDFRONT_DIST: ((cloudfront-dist)) - - - task: build-eloqstore-type - file: eloqsql_src/concourse/tasks/build_release_tarball.yml - image: image_ubuntu2204 - params: - TAGGED: "true" - GIT_SSH_KEY: ((git-key)) - AWS_ACCESS_KEY_ID: ((aws_access_key_id)) - AWS_DEFAULT_REGION: ((aws-region)) - AWS_SECRET_ACCESS_KEY: ((aws_secret_access_key)) - BUILD_LOG_SRV: "true" - BUILD_TYPE: RelWithDebInfo - DATA_STORE_TYPE: ELOQDSS_ELOQSTORE - CLOUDFRONT_DIST: ((cloudfront-dist)) + PG_CONN: ((pg-conn)) serial: true @@ -186,6 +172,7 @@ jobs: BUILD_TYPE: RelWithDebInfo DATA_STORE_TYPE: ELOQDSS_ROCKSDB_CLOUD_S3 CLOUDFRONT_DIST: ((cloudfront-dist)) + PG_CONN: ((pg-conn)) - task: build-eloqdss-rocksdb-type file: eloqsql_src/concourse/tasks/build_release_tarball.yml @@ -200,10 +187,28 @@ jobs: BUILD_TYPE: RelWithDebInfo DATA_STORE_TYPE: ELOQDSS_ROCKSDB CLOUDFRONT_DIST: ((cloudfront-dist)) + PG_CONN: ((pg-conn)) + - - task: build-eloqstore-type + serial: true + +- name: centos7-main + plan: + - in_parallel: + limit: 2 + steps: + - get: eloqsql_src + resource: eloqsql_src_main + - get: logservice_src + resource: logservice_src_main + - get: raft_host_manager_src + resource: raft_host_manager_src_main + - get: image_centos7 + resource: image_centos7_main + + - task: build-rocksdb-cloud-s3-type file: eloqsql_src/concourse/tasks/build_release_tarball.yml - image: image_ubuntu2404 + image: image_centos7 params: TAGGED: "true" GIT_SSH_KEY: ((git-key)) @@ -212,12 +217,29 @@ jobs: AWS_SECRET_ACCESS_KEY: ((aws_secret_access_key)) BUILD_LOG_SRV: "true" BUILD_TYPE: RelWithDebInfo - DATA_STORE_TYPE: ELOQDSS_ELOQSTORE + DATA_STORE_TYPE: ELOQDSS_ROCKSDB_CLOUD_S3 CLOUDFRONT_DIST: ((cloudfront-dist)) + PG_CONN: ((pg-conn)) + + - task: build-eloqdss-rocksdb-type + file: eloqsql_src/concourse/tasks/build_release_tarball.yml + image: image_centos7 + params: + TAGGED: "true" + GIT_SSH_KEY: ((git-key)) + AWS_ACCESS_KEY_ID: ((aws_access_key_id)) + AWS_DEFAULT_REGION: ((aws-region)) + AWS_SECRET_ACCESS_KEY: ((aws_secret_access_key)) + BUILD_LOG_SRV: "true" + BUILD_TYPE: RelWithDebInfo + DATA_STORE_TYPE: ELOQDSS_ROCKSDB + CLOUDFRONT_DIST: ((cloudfront-dist)) + PG_CONN: ((pg-conn)) + serial: true -- name: centos7-main +- name: centos8-main plan: - in_parallel: limit: 2 @@ -228,12 +250,12 @@ jobs: resource: logservice_src_main - get: raft_host_manager_src resource: raft_host_manager_src_main - - get: image_centos7 - resource: image_centos7_main + - get: image_centos8 + resource: image_centos8_main - task: build-rocksdb-cloud-s3-type file: eloqsql_src/concourse/tasks/build_release_tarball.yml - image: image_centos7 + image: image_centos8 params: TAGGED: "true" GIT_SSH_KEY: ((git-key)) @@ -244,10 +266,11 @@ jobs: BUILD_TYPE: RelWithDebInfo DATA_STORE_TYPE: ELOQDSS_ROCKSDB_CLOUD_S3 CLOUDFRONT_DIST: ((cloudfront-dist)) + PG_CONN: ((pg-conn)) - task: build-eloqdss-rocksdb-type file: eloqsql_src/concourse/tasks/build_release_tarball.yml - image: image_centos7 + image: image_centos8 params: TAGGED: "true" GIT_SSH_KEY: ((git-key)) @@ -258,10 +281,43 @@ jobs: BUILD_TYPE: RelWithDebInfo DATA_STORE_TYPE: ELOQDSS_ROCKSDB CLOUDFRONT_DIST: ((cloudfront-dist)) + PG_CONN: ((pg-conn)) + + + serial: true + +- name: rocky9-main + plan: + - in_parallel: + limit: 2 + steps: + - get: eloqsql_src + resource: eloqsql_src_main + - get: logservice_src + resource: logservice_src_main + - get: raft_host_manager_src + resource: raft_host_manager_src_main + - get: image_rocky9 + resource: image_rocky9_main + + - task: build-rocksdb-cloud-s3-type + file: eloqsql_src/concourse/tasks/build_release_tarball.yml + image: image_rocky9 + params: + TAGGED: "true" + GIT_SSH_KEY: ((git-key)) + AWS_ACCESS_KEY_ID: ((aws_access_key_id)) + AWS_DEFAULT_REGION: ((aws-region)) + AWS_SECRET_ACCESS_KEY: ((aws_secret_access_key)) + BUILD_LOG_SRV: "true" + BUILD_TYPE: RelWithDebInfo + DATA_STORE_TYPE: ELOQDSS_ROCKSDB_CLOUD_S3 + CLOUDFRONT_DIST: ((cloudfront-dist)) + PG_CONN: ((pg-conn)) - - task: build-eloqstore-type + - task: build-eloqdss-rocksdb-type file: eloqsql_src/concourse/tasks/build_release_tarball.yml - image: image_centos7 + image: image_rocky9 params: TAGGED: "true" GIT_SSH_KEY: ((git-key)) @@ -270,7 +326,9 @@ jobs: AWS_SECRET_ACCESS_KEY: ((aws_secret_access_key)) BUILD_LOG_SRV: "true" BUILD_TYPE: RelWithDebInfo - DATA_STORE_TYPE: ELOQDSS_ELOQSTORE + DATA_STORE_TYPE: ELOQDSS_ROCKSDB CLOUDFRONT_DIST: ((cloudfront-dist)) + PG_CONN: ((pg-conn)) + serial: true \ No newline at end of file diff --git a/concourse/scripts/build_tarball.bash b/concourse/scripts/build_tarball.bash index 2fae7f1bb9d..0d8f0f7e980 100755 --- a/concourse/scripts/build_tarball.bash +++ b/concourse/scripts/build_tarball.bash @@ -16,13 +16,6 @@ if [ -n "${GIT_SSH_KEY}" ]; then ssh-keyscan github.com >> ~/.ssh/known_hosts 2>/dev/null || true fi -# make coredump dir writable for debugging -if [ ! -d "/var/crash" ]; then sudo mkdir -p /var/crash; fi -sudo chmod 777 /var/crash - -ulimit -c unlimited -echo '/var/crash/core.%t.%e.%p' | sudo tee /proc/sys/kernel/core_pattern - # Ensure workspace ownership sudo chown -R $current_user $HOME/workspace 2>/dev/null || true @@ -80,7 +73,7 @@ arm64 | aarch64) ARCH=arm64 ;; *) ARCH=$(uname -m) ;; esac -if [ -n "${TAGGED}" ]; then +if [ "${TAGGED}" = "true" ]; then TAGGED=$(git tag --sort=-v:refname | head -n 1) if [ -z "${TAGGED}" ]; then exit 1 @@ -103,11 +96,9 @@ S3_PREFIX="s3://${S3_BUCKET}/eloqsql" DATA_STORE_ID=$(echo ${DATA_STORE_TYPE} | tr '[:upper:]' '[:lower:]') # Normalize behavior for supported DATA_STORE_TYPE values -if [ "${DATA_STORE_TYPE}" = "ROCKSDB" ]; then - DATA_STORE_ID="rocksdb" -elif [ "${DATA_STORE_TYPE}" = "ELOQDSS_ROCKSDB_CLOUD_S3" ]; then +if [ "${DATA_STORE_TYPE}" = "ELOQDSS_ROCKSDB_CLOUD_S3" ]; then CMAKE_ARGS="${CMAKE_ARGS} -DUSE_ROCKSDB_LOG_STATE=ON -DWITH_ROCKSDB_CLOUD=S3 -DWITH_CLOUD_AZ_INFO=ON" - DATA_STORE_ID="eloqdss_s3" + DATA_STORE_ID="rocks_s3" elif [ "${DATA_STORE_TYPE}" = "ELOQDSS_ROCKSDB" ]; then DATA_STORE_ID="eloqdss_rocksdb" elif [ "${DATA_STORE_TYPE}" = "ELOQDSS_ELOQSTORE" ]; then @@ -219,10 +210,10 @@ mv launch_sv ${DEST_DIR}/bin/ cd ${HOME} tar -czvf eloqsql.tar.gz -C ${HOME} EloqSQL -if [ -n "${TAGGED}" ]; then +if [ "${TAGGED}" = "true" ]; then SQL_TARBALL="eloqsql-${TAGGED}-${OS_ID}-${ARCH}.tar.gz" eval ${INSTALL_PSQL} - SQL="INSERT INTO sql_release VALUES ('eloqsql', '${ARCH}', '${OS_ID}', '${DATA_STORE_ID}', $(echo ${TAGGED} | tr '.' ',')) ON CONFLICT DO NOTHING" + SQL="INSERT INTO tx_release VALUES ('eloqsql', '${ARCH}', '${OS_ID}', '${DATA_STORE_ID}', $(echo ${TAGGED} | tr '.' ',')) ON CONFLICT DO NOTHING" psql postgresql://${PG_CONN}/eloq_release?sslmode=require -c "${SQL}" else SQL_TARBALL="eloqsql-${OUT_NAME}-${OS_ID}-${ARCH}.tar.gz" @@ -272,7 +263,7 @@ build_upload_log_srv() { if [ "${BUILD_LOG_SRV}" = true ]; then # make and build log_service - if [ -n "${TAGGED}" ]; then + if [ "${TAGGED}" = "true" ]; then LOG_TARBALL="log-service-${TAGGED}-${OS_ID}-${ARCH}.tar.gz" else LOG_TARBALL="log-service-${OUT_NAME}-${OS_ID}-${ARCH}.tar.gz" diff --git a/concourse/tasks/build_release_tarball.yml b/concourse/tasks/build_release_tarball.yml index 0f668cf4348..9d78e42dc09 100644 --- a/concourse/tasks/build_release_tarball.yml +++ b/concourse/tasks/build_release_tarball.yml @@ -7,6 +7,8 @@ inputs: - name: raft_host_manager_src outputs: - name: the-output +params: + LD_LIBRARY_PATH: /usr/local/lib:/usr/local/lib64 run: path: /bin/bash args: