From ef79bd9baf3cb089b580aef9ecec7adc21c259a9 Mon Sep 17 00:00:00 2001 From: githubzilla Date: Mon, 12 Jan 2026 15:52:50 +0800 Subject: [PATCH 1/5] Update for eloqstore --- concourse/scripts/common.sh | 75 ++++--------------- concourse/scripts/data_substrate.cnf | 17 +---- .../scripts/data_substrate.rocksdbcloud.cnf | 29 +++++++ 3 files changed, 46 insertions(+), 75 deletions(-) create mode 100644 concourse/scripts/data_substrate.rocksdbcloud.cnf diff --git a/concourse/scripts/common.sh b/concourse/scripts/common.sh index 6e83620323..1f4d5b9651 100644 --- a/concourse/scripts/common.sh +++ b/concourse/scripts/common.sh @@ -10,49 +10,6 @@ pip3 install minio # Setup Minio mc Client command mc alias set minio_server ${ELOQ_AWS_S3_ENDPOINT_URL} ${ELOQ_AWS_ACCESS_KEY_ID} ${ELOQ_AWS_SECRET_KEY} -# Helper function to update config template file with required settings -update_config_template() { - local config_file="$1" - if [ ! -f "$config_file" ]; then - echo "Warning: Config file $config_file does not exist" - return 1 - fi - - # ak/sk - sed -i "s/aws_access_key_id.*=.\+/aws_access_key_id=${ELOQ_AWS_ACCESS_KEY_ID}/g" "$config_file" - sed -i "s/aws_secret_key.*=.\+/aws_secret_key=${ELOQ_AWS_SECRET_KEY}/g" "$config_file" - # OSS settings - sed -i "s|rocksdb_cloud_s3_endpoint_url.*=.\+|rocksdb_cloud_s3_endpoint_url=${ELOQ_AWS_S3_ENDPOINT_URL}|g" "$config_file" - sed -i "s|txlog_rocksdb_cloud_s3_endpoint_url.*=.\+|txlog_rocksdb_cloud_s3_endpoint_url=${ELOQ_AWS_S3_ENDPOINT_URL}|g" "$config_file" - sed -i "s/rocksdb_cloud_bucket_name.*=.\+/rocksdb_cloud_bucket_name=${BUCKET_NAME}/g" "$config_file" - sed -i "s/txlog_rocksdb_cloud_bucket_name.*=.\+/txlog_rocksdb_cloud_bucket_name=${BUCKET_NAME}/g" "$config_file" - sed -i "s/rocksdb_cloud_region.*=.\+/rocksdb_cloud_region=${ELOQ_AWS_REGION}/g" "$config_file" - sed -i "s/txlog_rocksdb_cloud_region.*=.\+/txlog_rocksdb_cloud_region=${ELOQ_AWS_REGION}/g" "$config_file" - sed -i "s/rocksdb_cloud_bucket_prefix.*=.\+/rocksdb_cloud_bucket_prefix=${BUCKET_PREFIX}/g" "$config_file" - sed -i "s/txlog_rocksdb_cloud_bucket_prefix.*=.\+/txlog_rocksdb_cloud_bucket_prefix=${BUCKET_PREFIX}/g" "$config_file" - sed -i "s|eloq_dss_config_file_path.*=.\+|eloq_dss_config_file_path=${WORKSPACE}/eloqsql_src/concourse/scripts/dss_config.example.ini|g" "$config_file" -} - -if [[ -d "$WORKSPACE/eloqdoc_src/concourse" ]]; then - pushd "$WORKSPACE/eloqdoc_src/concourse" -elif [[ -d "$WORKSPACE/eloqdoc_pr/concourse" ]]; then - pushd "$WORKSPACE/eloqdoc_pr/concourse" -else - echo "No concourse directory found under \$WORKSPACE" >&2 - exit 1 -fi - -update_config_template ./scripts/data_substrate.cnf - -update_config_template ./artifact/ELOQDSS_ROCKSDB/data_substrate.cnf - -# Update all data_substrate*.cnf files in ELOQDSS_ROCKSDB_CLOUD_S3 -update_config_template ./artifact/ELOQDSS_ROCKSDB_CLOUD_S3/data_substrate.cnf -update_config_template ./artifact/ELOQDSS_ROCKSDB_CLOUD_S3/data_substrate_cluster_a.cnf -update_config_template ./artifact/ELOQDSS_ROCKSDB_CLOUD_S3/data_substrate_cluster_b.cnf -update_config_template ./artifact/ELOQDSS_ROCKSDB_CLOUD_S3/data_substrate_cluster_c.cnf -popd - # Make coredump dir writable. if [ ! -d "/var/crash" ]; then sudo mkdir -p /var/crash; fi sudo chmod 777 /var/crash @@ -156,7 +113,7 @@ compile_and_install_ent() { -DSTATISTICS=ON \ -DUSE_ASAN=OFF \ -DWITH_LOG_STATE=ROCKSDB_CLOUD_S3 \ - -DWITH_DATA_STORE=ELOQDSS_ROCKSDB_CLOUD_S3 \ + -DWITH_DATA_STORE=ELOQDSS_ELOQSTORE \ -DFORK_HM_PROCESS=ON \ -DOPEN_LOG_SERVICE=OFF @@ -168,7 +125,7 @@ compile_and_install_ent() { # Detect CPU cores for optimal parallel builds # CPU_CORE_SIZE=$(nproc 2>/dev/null || grep -c ^processor /proc/cpuinfo 2>/dev/null || echo 4) CPU_CORE_SIZE=4 - env OPEN_LOG_SERVICE=0 WITH_DATA_STORE=ELOQDSS_ROCKSDB_CLOUD_S3 WITH_LOG_STATE=ROCKSDB_CLOUD_S3 FORK_HM_PROCESS=1 \ + env OPEN_LOG_SERVICE=0 WITH_DATA_STORE=ELOQDSS_ELOQSTORE WITH_LOG_STATE=ROCKSDB_CLOUD_S3 FORK_HM_PROCESS=1 \ python2 scripts/buildscripts/scons.py MONGO_VERSION=4.0.3 \ VARIANT_DIR=Debug \ CXXFLAGS="-Wno-nonnull -Wno-class-memaccess -Wno-interference-size -Wno-redundant-move" \ @@ -197,15 +154,12 @@ launch_eloqdoc() { mkdir -p "$PREFIX/log" "$PREFIX/data" nohup $PREFIX/bin/eloqdoc \ --config=./concourse/scripts/eloqdoc.yaml \ - --data_substrate_config=./concourse/scripts/data_substrate.cnf \ - --rocksdb_cloud_bucket_name="$bucket_name" \ - --rocksdb_cloud_bucket_prefix="$bucket_prefix" \ - --rocksdb_cloud_object_path="dss" \ - --rocksdb_cloud_s3_endpoint_url=${ELOQ_AWS_S3_ENDPOINT_URL} \ - --txlog_rocksdb_cloud_bucket_name="$bucket_name" \ - --txlog_rocksdb_cloud_bucket_prefix="$bucket_prefix" \ - --txlog_rocksdb_cloud_object_path="txlog" \ - --txlog_rocksdb_cloud_s3_endpoint_url=${ELOQ_AWS_S3_ENDPOINT_URL} \ + --data_substrate_config=./concourse/scripts/data_substrate.cnf \ + --eloq_store_cloud_endpoint=${ELOQ_AWS_S3_ENDPOINT_URL} \ + --eloq_store_cloud_store_path="${bucket_prefix}${bucket_name}/eloqstore" + --txlog_rocksdb_cloud_object_store_service_url="${ELOQ_AWS_S3_ENDPOINT_URL}/${bucket_prefix}${bucket_name}/txlog" \ + --eloq_store_cloud_access_key=${MINIO_ACCESS_KEY_ID} \ + --eloq_store_cloud_secret_key=${MINIO_SECRECT_KEY} \ &>$PREFIX/log/eloqdoc.out & } @@ -222,14 +176,11 @@ launch_eloqdoc_fast() { nohup $PREFIX/bin/eloqdoc \ --config=./concourse/scripts/eloqdoc.yaml \ --data_substrate_config=./concourse/scripts/data_substrate.cnf \ - --rocksdb_cloud_bucket_name="$bucket_name" \ - --rocksdb_cloud_bucket_prefix="$bucket_prefix" \ - --rocksdb_cloud_object_path="dss" \ - --rocksdb_cloud_s3_endpoint_url=${ELOQ_AWS_S3_ENDPOINT_URL} \ - --txlog_rocksdb_cloud_bucket_name="$bucket_name" \ - --txlog_rocksdb_cloud_bucket_prefix="$bucket_prefix" \ - --txlog_rocksdb_cloud_object_path="txlog" \ - --txlog_rocksdb_cloud_s3_endpoint_url=${ELOQ_AWS_S3_ENDPOINT_URL} \ + --eloq_store_cloud_endpoint=${ELOQ_AWS_S3_ENDPOINT_URL} \ + --eloq_store_cloud_store_path="${bucket_prefix}${bucket_name}/eloqstore" + --txlog_rocksdb_cloud_object_store_service_url="${ELOQ_AWS_S3_ENDPOINT_URL}/${bucket_prefix}${bucket_name}/txlog" \ + --eloq_store_cloud_access_key=${MINIO_ACCESS_KEY_ID} \ + --eloq_store_cloud_secret_key=${MINIO_SECRECT_KEY} \ --enable_wal=false \ &>$PREFIX/log/eloqdoc.out & } diff --git a/concourse/scripts/data_substrate.cnf b/concourse/scripts/data_substrate.cnf index fca43f941a..37bf8eb284 100644 --- a/concourse/scripts/data_substrate.cnf +++ b/concourse/scripts/data_substrate.cnf @@ -9,21 +9,12 @@ event_dispatcher_num=1 #auto_redirect=true enable_mvcc=true logserver_snapshot_interval=60 -txlog_rocksdb_cloud_bucket_prefix=eloq- -txlog_rocksdb_cloud_bucket_name=test -txlog_rocksdb_cloud_object_path=txlog -txlog_rocksdb_cloud_s3_endpoint_url=http://127.0.0.1:9900 [cluster] ip_port_list=127.0.0.1:16379 -#txlog_service_list=127.0.0.1:9000 -#txlog_group_replica_num=1 [store] -# rocksdb_cloud store -rocksdb_cloud_bucket_prefix=eloq- -rocksdb_cloud_bucket_name=test -rocksdb_cloud_object_path=dss -rocksdb_cloud_s3_endpoint_url=http://127.0.0.1:9900 -aws_access_key_id=CHANGEME_ACCESS_KEY_ID -aws_secret_key=CHANGEME_SECRET_KEY +# Max number of open files used by eloqstore(default: 1024) +eloq_store_open_files_limit=1024 +eloq_store_data_path_list=install/eloqstore +eloq_store_buffer_pool_size=5MB diff --git a/concourse/scripts/data_substrate.rocksdbcloud.cnf b/concourse/scripts/data_substrate.rocksdbcloud.cnf new file mode 100644 index 0000000000..fca43f941a --- /dev/null +++ b/concourse/scripts/data_substrate.rocksdbcloud.cnf @@ -0,0 +1,29 @@ +[local] +core_number=2 +checkpoint_interval=10 +node_memory_limit_mb=4000 +enable_data_store=true +enable_wal=true +eloq_data_path=install/data +event_dispatcher_num=1 +#auto_redirect=true +enable_mvcc=true +logserver_snapshot_interval=60 +txlog_rocksdb_cloud_bucket_prefix=eloq- +txlog_rocksdb_cloud_bucket_name=test +txlog_rocksdb_cloud_object_path=txlog +txlog_rocksdb_cloud_s3_endpoint_url=http://127.0.0.1:9900 + +[cluster] +ip_port_list=127.0.0.1:16379 +#txlog_service_list=127.0.0.1:9000 +#txlog_group_replica_num=1 + +[store] +# rocksdb_cloud store +rocksdb_cloud_bucket_prefix=eloq- +rocksdb_cloud_bucket_name=test +rocksdb_cloud_object_path=dss +rocksdb_cloud_s3_endpoint_url=http://127.0.0.1:9900 +aws_access_key_id=CHANGEME_ACCESS_KEY_ID +aws_secret_key=CHANGEME_SECRET_KEY From 7a8920ea6fe660bb7a784dda3b503563a3955fdf Mon Sep 17 00:00:00 2001 From: githubzilla Date: Mon, 12 Jan 2026 17:01:29 +0800 Subject: [PATCH 2/5] Fix crash --- concourse/scripts/data_substrate.cnf | 1 - 1 file changed, 1 deletion(-) diff --git a/concourse/scripts/data_substrate.cnf b/concourse/scripts/data_substrate.cnf index 37bf8eb284..5bd6517ef5 100644 --- a/concourse/scripts/data_substrate.cnf +++ b/concourse/scripts/data_substrate.cnf @@ -16,5 +16,4 @@ ip_port_list=127.0.0.1:16379 [store] # Max number of open files used by eloqstore(default: 1024) eloq_store_open_files_limit=1024 -eloq_store_data_path_list=install/eloqstore eloq_store_buffer_pool_size=5MB From d7242ea2ea94578b98eaee0416ef72164a544f63 Mon Sep 17 00:00:00 2001 From: githubzilla Date: Mon, 12 Jan 2026 17:23:45 +0800 Subject: [PATCH 3/5] Update data_substrate --- src/mongo/db/modules/eloq/data_substrate | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mongo/db/modules/eloq/data_substrate b/src/mongo/db/modules/eloq/data_substrate index a43c2d2b91..1210b03579 160000 --- a/src/mongo/db/modules/eloq/data_substrate +++ b/src/mongo/db/modules/eloq/data_substrate @@ -1 +1 @@ -Subproject commit a43c2d2b911debf4967d90c3a4e5d893e2ac3c0d +Subproject commit 1210b0357993fdb5d982fe1c4e7deacc8f144fbd From 287a184ac046267051dd109ffbc6341ca69927ca Mon Sep 17 00:00:00 2001 From: githubzilla Date: Mon, 12 Jan 2026 17:26:00 +0800 Subject: [PATCH 4/5] Fix launch_eloqdoc --- concourse/scripts/common.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/concourse/scripts/common.sh b/concourse/scripts/common.sh index 1f4d5b9651..c372f4de0e 100644 --- a/concourse/scripts/common.sh +++ b/concourse/scripts/common.sh @@ -157,9 +157,11 @@ launch_eloqdoc() { --data_substrate_config=./concourse/scripts/data_substrate.cnf \ --eloq_store_cloud_endpoint=${ELOQ_AWS_S3_ENDPOINT_URL} \ --eloq_store_cloud_store_path="${bucket_prefix}${bucket_name}/eloqstore" + --eloq_store_cloud_access_key=${ELOQ_AWS_ACCESS_KEY_ID} \ + --eloq_store_cloud_secret_key=${ELOQ_AWS_SECRET_KEY} \ --txlog_rocksdb_cloud_object_store_service_url="${ELOQ_AWS_S3_ENDPOINT_URL}/${bucket_prefix}${bucket_name}/txlog" \ - --eloq_store_cloud_access_key=${MINIO_ACCESS_KEY_ID} \ - --eloq_store_cloud_secret_key=${MINIO_SECRECT_KEY} \ + --aws_access_key_id=${ELOQ_AWS_ACCESS_KEY_ID} \ + --aws_secret_key=${ELOQ_AWS_SECRET_KEY} \ &>$PREFIX/log/eloqdoc.out & } @@ -178,9 +180,11 @@ launch_eloqdoc_fast() { --data_substrate_config=./concourse/scripts/data_substrate.cnf \ --eloq_store_cloud_endpoint=${ELOQ_AWS_S3_ENDPOINT_URL} \ --eloq_store_cloud_store_path="${bucket_prefix}${bucket_name}/eloqstore" + --eloq_store_cloud_access_key=${ELOQ_AWS_ACCESS_KEY_ID} \ + --eloq_store_cloud_secret_key=${ELOQ_AWS_SECRET_KEY} \ --txlog_rocksdb_cloud_object_store_service_url="${ELOQ_AWS_S3_ENDPOINT_URL}/${bucket_prefix}${bucket_name}/txlog" \ - --eloq_store_cloud_access_key=${MINIO_ACCESS_KEY_ID} \ - --eloq_store_cloud_secret_key=${MINIO_SECRECT_KEY} \ + --aws_access_key_id=${ELOQ_AWS_ACCESS_KEY_ID} \ + --aws_secret_key=${ELOQ_AWS_SECRET_KEY} \ --enable_wal=false \ &>$PREFIX/log/eloqdoc.out & } From 9dfab4b965a06a397d83dcc3df5608407bd10349 Mon Sep 17 00:00:00 2001 From: githubzilla Date: Mon, 12 Jan 2026 18:00:39 +0800 Subject: [PATCH 5/5] Fix launch_eloqdoc failed --- concourse/scripts/common.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/concourse/scripts/common.sh b/concourse/scripts/common.sh index c372f4de0e..e0875e25af 100644 --- a/concourse/scripts/common.sh +++ b/concourse/scripts/common.sh @@ -156,7 +156,7 @@ launch_eloqdoc() { --config=./concourse/scripts/eloqdoc.yaml \ --data_substrate_config=./concourse/scripts/data_substrate.cnf \ --eloq_store_cloud_endpoint=${ELOQ_AWS_S3_ENDPOINT_URL} \ - --eloq_store_cloud_store_path="${bucket_prefix}${bucket_name}/eloqstore" + --eloq_store_cloud_store_path="${bucket_prefix}${bucket_name}/eloqstore" \ --eloq_store_cloud_access_key=${ELOQ_AWS_ACCESS_KEY_ID} \ --eloq_store_cloud_secret_key=${ELOQ_AWS_SECRET_KEY} \ --txlog_rocksdb_cloud_object_store_service_url="${ELOQ_AWS_S3_ENDPOINT_URL}/${bucket_prefix}${bucket_name}/txlog" \ @@ -179,7 +179,7 @@ launch_eloqdoc_fast() { --config=./concourse/scripts/eloqdoc.yaml \ --data_substrate_config=./concourse/scripts/data_substrate.cnf \ --eloq_store_cloud_endpoint=${ELOQ_AWS_S3_ENDPOINT_URL} \ - --eloq_store_cloud_store_path="${bucket_prefix}${bucket_name}/eloqstore" + --eloq_store_cloud_store_path="${bucket_prefix}${bucket_name}/eloqstore" \ --eloq_store_cloud_access_key=${ELOQ_AWS_ACCESS_KEY_ID} \ --eloq_store_cloud_secret_key=${ELOQ_AWS_SECRET_KEY} \ --txlog_rocksdb_cloud_object_store_service_url="${ELOQ_AWS_S3_ENDPOINT_URL}/${bucket_prefix}${bucket_name}/txlog" \