diff --git a/concourse/scripts/common.sh b/concourse/scripts/common.sh index 6e83620323..e0875e25af 100644 --- a/concourse/scripts/common.sh +++ b/concourse/scripts/common.sh @@ -10,49 +10,6 @@ pip3 install minio # Setup Minio mc Client command mc alias set minio_server ${ELOQ_AWS_S3_ENDPOINT_URL} ${ELOQ_AWS_ACCESS_KEY_ID} ${ELOQ_AWS_SECRET_KEY} -# Helper function to update config template file with required settings -update_config_template() { - local config_file="$1" - if [ ! -f "$config_file" ]; then - echo "Warning: Config file $config_file does not exist" - return 1 - fi - - # ak/sk - sed -i "s/aws_access_key_id.*=.\+/aws_access_key_id=${ELOQ_AWS_ACCESS_KEY_ID}/g" "$config_file" - sed -i "s/aws_secret_key.*=.\+/aws_secret_key=${ELOQ_AWS_SECRET_KEY}/g" "$config_file" - # OSS settings - sed -i "s|rocksdb_cloud_s3_endpoint_url.*=.\+|rocksdb_cloud_s3_endpoint_url=${ELOQ_AWS_S3_ENDPOINT_URL}|g" "$config_file" - sed -i "s|txlog_rocksdb_cloud_s3_endpoint_url.*=.\+|txlog_rocksdb_cloud_s3_endpoint_url=${ELOQ_AWS_S3_ENDPOINT_URL}|g" "$config_file" - sed -i "s/rocksdb_cloud_bucket_name.*=.\+/rocksdb_cloud_bucket_name=${BUCKET_NAME}/g" "$config_file" - sed -i "s/txlog_rocksdb_cloud_bucket_name.*=.\+/txlog_rocksdb_cloud_bucket_name=${BUCKET_NAME}/g" "$config_file" - sed -i "s/rocksdb_cloud_region.*=.\+/rocksdb_cloud_region=${ELOQ_AWS_REGION}/g" "$config_file" - sed -i "s/txlog_rocksdb_cloud_region.*=.\+/txlog_rocksdb_cloud_region=${ELOQ_AWS_REGION}/g" "$config_file" - sed -i "s/rocksdb_cloud_bucket_prefix.*=.\+/rocksdb_cloud_bucket_prefix=${BUCKET_PREFIX}/g" "$config_file" - sed -i "s/txlog_rocksdb_cloud_bucket_prefix.*=.\+/txlog_rocksdb_cloud_bucket_prefix=${BUCKET_PREFIX}/g" "$config_file" - sed -i "s|eloq_dss_config_file_path.*=.\+|eloq_dss_config_file_path=${WORKSPACE}/eloqsql_src/concourse/scripts/dss_config.example.ini|g" "$config_file" -} - -if [[ -d "$WORKSPACE/eloqdoc_src/concourse" ]]; then - pushd "$WORKSPACE/eloqdoc_src/concourse" -elif [[ -d "$WORKSPACE/eloqdoc_pr/concourse" ]]; then - pushd "$WORKSPACE/eloqdoc_pr/concourse" -else - echo "No concourse directory found under \$WORKSPACE" >&2 - exit 1 -fi - -update_config_template ./scripts/data_substrate.cnf - -update_config_template ./artifact/ELOQDSS_ROCKSDB/data_substrate.cnf - -# Update all data_substrate*.cnf files in ELOQDSS_ROCKSDB_CLOUD_S3 -update_config_template ./artifact/ELOQDSS_ROCKSDB_CLOUD_S3/data_substrate.cnf -update_config_template ./artifact/ELOQDSS_ROCKSDB_CLOUD_S3/data_substrate_cluster_a.cnf -update_config_template ./artifact/ELOQDSS_ROCKSDB_CLOUD_S3/data_substrate_cluster_b.cnf -update_config_template ./artifact/ELOQDSS_ROCKSDB_CLOUD_S3/data_substrate_cluster_c.cnf -popd - # Make coredump dir writable. if [ ! -d "/var/crash" ]; then sudo mkdir -p /var/crash; fi sudo chmod 777 /var/crash @@ -156,7 +113,7 @@ compile_and_install_ent() { -DSTATISTICS=ON \ -DUSE_ASAN=OFF \ -DWITH_LOG_STATE=ROCKSDB_CLOUD_S3 \ - -DWITH_DATA_STORE=ELOQDSS_ROCKSDB_CLOUD_S3 \ + -DWITH_DATA_STORE=ELOQDSS_ELOQSTORE \ -DFORK_HM_PROCESS=ON \ -DOPEN_LOG_SERVICE=OFF @@ -168,7 +125,7 @@ compile_and_install_ent() { # Detect CPU cores for optimal parallel builds # CPU_CORE_SIZE=$(nproc 2>/dev/null || grep -c ^processor /proc/cpuinfo 2>/dev/null || echo 4) CPU_CORE_SIZE=4 - env OPEN_LOG_SERVICE=0 WITH_DATA_STORE=ELOQDSS_ROCKSDB_CLOUD_S3 WITH_LOG_STATE=ROCKSDB_CLOUD_S3 FORK_HM_PROCESS=1 \ + env OPEN_LOG_SERVICE=0 WITH_DATA_STORE=ELOQDSS_ELOQSTORE WITH_LOG_STATE=ROCKSDB_CLOUD_S3 FORK_HM_PROCESS=1 \ python2 scripts/buildscripts/scons.py MONGO_VERSION=4.0.3 \ VARIANT_DIR=Debug \ CXXFLAGS="-Wno-nonnull -Wno-class-memaccess -Wno-interference-size -Wno-redundant-move" \ @@ -197,15 +154,14 @@ launch_eloqdoc() { mkdir -p "$PREFIX/log" "$PREFIX/data" nohup $PREFIX/bin/eloqdoc \ --config=./concourse/scripts/eloqdoc.yaml \ - --data_substrate_config=./concourse/scripts/data_substrate.cnf \ - --rocksdb_cloud_bucket_name="$bucket_name" \ - --rocksdb_cloud_bucket_prefix="$bucket_prefix" \ - --rocksdb_cloud_object_path="dss" \ - --rocksdb_cloud_s3_endpoint_url=${ELOQ_AWS_S3_ENDPOINT_URL} \ - --txlog_rocksdb_cloud_bucket_name="$bucket_name" \ - --txlog_rocksdb_cloud_bucket_prefix="$bucket_prefix" \ - --txlog_rocksdb_cloud_object_path="txlog" \ - --txlog_rocksdb_cloud_s3_endpoint_url=${ELOQ_AWS_S3_ENDPOINT_URL} \ + --data_substrate_config=./concourse/scripts/data_substrate.cnf \ + --eloq_store_cloud_endpoint=${ELOQ_AWS_S3_ENDPOINT_URL} \ + --eloq_store_cloud_store_path="${bucket_prefix}${bucket_name}/eloqstore" \ + --eloq_store_cloud_access_key=${ELOQ_AWS_ACCESS_KEY_ID} \ + --eloq_store_cloud_secret_key=${ELOQ_AWS_SECRET_KEY} \ + --txlog_rocksdb_cloud_object_store_service_url="${ELOQ_AWS_S3_ENDPOINT_URL}/${bucket_prefix}${bucket_name}/txlog" \ + --aws_access_key_id=${ELOQ_AWS_ACCESS_KEY_ID} \ + --aws_secret_key=${ELOQ_AWS_SECRET_KEY} \ &>$PREFIX/log/eloqdoc.out & } @@ -222,14 +178,13 @@ launch_eloqdoc_fast() { nohup $PREFIX/bin/eloqdoc \ --config=./concourse/scripts/eloqdoc.yaml \ --data_substrate_config=./concourse/scripts/data_substrate.cnf \ - --rocksdb_cloud_bucket_name="$bucket_name" \ - --rocksdb_cloud_bucket_prefix="$bucket_prefix" \ - --rocksdb_cloud_object_path="dss" \ - --rocksdb_cloud_s3_endpoint_url=${ELOQ_AWS_S3_ENDPOINT_URL} \ - --txlog_rocksdb_cloud_bucket_name="$bucket_name" \ - --txlog_rocksdb_cloud_bucket_prefix="$bucket_prefix" \ - --txlog_rocksdb_cloud_object_path="txlog" \ - --txlog_rocksdb_cloud_s3_endpoint_url=${ELOQ_AWS_S3_ENDPOINT_URL} \ + --eloq_store_cloud_endpoint=${ELOQ_AWS_S3_ENDPOINT_URL} \ + --eloq_store_cloud_store_path="${bucket_prefix}${bucket_name}/eloqstore" \ + --eloq_store_cloud_access_key=${ELOQ_AWS_ACCESS_KEY_ID} \ + --eloq_store_cloud_secret_key=${ELOQ_AWS_SECRET_KEY} \ + --txlog_rocksdb_cloud_object_store_service_url="${ELOQ_AWS_S3_ENDPOINT_URL}/${bucket_prefix}${bucket_name}/txlog" \ + --aws_access_key_id=${ELOQ_AWS_ACCESS_KEY_ID} \ + --aws_secret_key=${ELOQ_AWS_SECRET_KEY} \ --enable_wal=false \ &>$PREFIX/log/eloqdoc.out & } diff --git a/concourse/scripts/data_substrate.cnf b/concourse/scripts/data_substrate.cnf index fca43f941a..5bd6517ef5 100644 --- a/concourse/scripts/data_substrate.cnf +++ b/concourse/scripts/data_substrate.cnf @@ -9,21 +9,11 @@ event_dispatcher_num=1 #auto_redirect=true enable_mvcc=true logserver_snapshot_interval=60 -txlog_rocksdb_cloud_bucket_prefix=eloq- -txlog_rocksdb_cloud_bucket_name=test -txlog_rocksdb_cloud_object_path=txlog -txlog_rocksdb_cloud_s3_endpoint_url=http://127.0.0.1:9900 [cluster] ip_port_list=127.0.0.1:16379 -#txlog_service_list=127.0.0.1:9000 -#txlog_group_replica_num=1 [store] -# rocksdb_cloud store -rocksdb_cloud_bucket_prefix=eloq- -rocksdb_cloud_bucket_name=test -rocksdb_cloud_object_path=dss -rocksdb_cloud_s3_endpoint_url=http://127.0.0.1:9900 -aws_access_key_id=CHANGEME_ACCESS_KEY_ID -aws_secret_key=CHANGEME_SECRET_KEY +# Max number of open files used by eloqstore(default: 1024) +eloq_store_open_files_limit=1024 +eloq_store_buffer_pool_size=5MB diff --git a/concourse/scripts/data_substrate.rocksdbcloud.cnf b/concourse/scripts/data_substrate.rocksdbcloud.cnf new file mode 100644 index 0000000000..fca43f941a --- /dev/null +++ b/concourse/scripts/data_substrate.rocksdbcloud.cnf @@ -0,0 +1,29 @@ +[local] +core_number=2 +checkpoint_interval=10 +node_memory_limit_mb=4000 +enable_data_store=true +enable_wal=true +eloq_data_path=install/data +event_dispatcher_num=1 +#auto_redirect=true +enable_mvcc=true +logserver_snapshot_interval=60 +txlog_rocksdb_cloud_bucket_prefix=eloq- +txlog_rocksdb_cloud_bucket_name=test +txlog_rocksdb_cloud_object_path=txlog +txlog_rocksdb_cloud_s3_endpoint_url=http://127.0.0.1:9900 + +[cluster] +ip_port_list=127.0.0.1:16379 +#txlog_service_list=127.0.0.1:9000 +#txlog_group_replica_num=1 + +[store] +# rocksdb_cloud store +rocksdb_cloud_bucket_prefix=eloq- +rocksdb_cloud_bucket_name=test +rocksdb_cloud_object_path=dss +rocksdb_cloud_s3_endpoint_url=http://127.0.0.1:9900 +aws_access_key_id=CHANGEME_ACCESS_KEY_ID +aws_secret_key=CHANGEME_SECRET_KEY diff --git a/src/mongo/db/modules/eloq/data_substrate b/src/mongo/db/modules/eloq/data_substrate index a43c2d2b91..1210b03579 160000 --- a/src/mongo/db/modules/eloq/data_substrate +++ b/src/mongo/db/modules/eloq/data_substrate @@ -1 +1 @@ -Subproject commit a43c2d2b911debf4967d90c3a4e5d893e2ac3c0d +Subproject commit 1210b0357993fdb5d982fe1c4e7deacc8f144fbd