From 136ff33987e2a61d755eb2775e5f9100e8827976 Mon Sep 17 00:00:00 2001 From: Chen Zhao Date: Wed, 4 Mar 2026 07:26:18 +0000 Subject: [PATCH 1/7] feat: adapt to eloqstore local standby feature fix wip fix wip update config update prolang timeout update test fix update update update update update update --- concourse/scripts/common.sh | 138 ++++++++++++++++++++++++++++++------ data_substrate | 2 +- eloqkv.ini | 7 +- 3 files changed, 123 insertions(+), 24 deletions(-) diff --git a/concourse/scripts/common.sh b/concourse/scripts/common.sh index 00cf6f41..7da263dd 100644 --- a/concourse/scripts/common.sh +++ b/concourse/scripts/common.sh @@ -46,7 +46,7 @@ function wait_until_ready() { # Function to wait until server is finished function wait_until_finished() { - local timeout=120 + local timeout=300 local elapsed=0 local interval=1 @@ -54,7 +54,7 @@ function wait_until_finished() { sleep $interval elapsed=$((elapsed + interval)) if [ $elapsed -ge $timeout ]; then - echo "Timeout: Process still running after 20 seconds." + echo "Timeout: Process still running after $timeout seconds." # list eloqkv still alived ps aux | grep eloqkv | grep -v grep | grep -v launch_sv | grep -v dss_server return 1 @@ -63,6 +63,83 @@ function wait_until_finished() { return 0 } +function setup_passwordless_ssh_for_eloq_test() { + local user_home="/home/$current_user" + local ssh_dir="${user_home}/.ssh" + local private_key="${ssh_dir}/id_ed25519" + local public_key="${private_key}.pub" + local authorized_keys="${ssh_dir}/authorized_keys" + local sshd_config="/etc/ssh/sshd_config" + local user_group + + mkdir -p "${ssh_dir}" + chmod 700 "${ssh_dir}" + + if [ ! -f "${private_key}" ]; then + ssh-keygen -q -t ed25519 -N "" -f "${private_key}" + elif [ ! -f "${public_key}" ]; then + ssh-keygen -y -f "${private_key}" > "${public_key}" + fi + + touch "${authorized_keys}" + chmod 600 "${authorized_keys}" + if ! grep -qxF "$(cat "${public_key}")" "${authorized_keys}"; then + cat "${public_key}" >> "${authorized_keys}" + echo >> "${authorized_keys}" + fi + + user_group=$(id -gn "${current_user}" 2>/dev/null || true) + if [ -n "${user_group}" ]; then + chown -R "${current_user}:${user_group}" "${ssh_dir}" + else + chown -R "${current_user}" "${ssh_dir}" + fi + + if [ "$(id -u)" -eq 0 ]; then + if grep -qE '^[#[:space:]]*MaxStartups[[:space:]]+' "${sshd_config}"; then + sed -i 's/^[#[:space:]]*MaxStartups[[:space:]].*/MaxStartups 200/' "${sshd_config}" + else + printf '\nMaxStartups 200\n' >> "${sshd_config}" + fi + + if grep -qE '^[#[:space:]]*PubkeyAuthentication[[:space:]]+' "${sshd_config}"; then + sed -i 's/^[#[:space:]]*PubkeyAuthentication[[:space:]].*/PubkeyAuthentication yes/' "${sshd_config}" + else + printf 'PubkeyAuthentication yes\n' >> "${sshd_config}" + fi + else + if sudo -n test -f "${sshd_config}"; then + if sudo -n grep -qE '^[#[:space:]]*MaxStartups[[:space:]]+' "${sshd_config}"; then + sudo -n sed -i 's/^[#[:space:]]*MaxStartups[[:space:]].*/MaxStartups 200/' "${sshd_config}" + else + printf '\nMaxStartups 200\n' | sudo -n tee -a "${sshd_config}" >/dev/null + fi + + if sudo -n grep -qE '^[#[:space:]]*PubkeyAuthentication[[:space:]]+' "${sshd_config}"; then + sudo -n sed -i 's/^[#[:space:]]*PubkeyAuthentication[[:space:]].*/PubkeyAuthentication yes/' "${sshd_config}" + else + printf 'PubkeyAuthentication yes\n' | sudo -n tee -a "${sshd_config}" >/dev/null + fi + else + echo "sudo access is required to update ${sshd_config} and restart ssh." + return 1 + fi + fi + + if [ "$(id -u)" -eq 0 ]; then + systemctl restart ssh || service ssh restart + else + sudo -n systemctl restart ssh || sudo -n service ssh restart + fi + + if [ $? -ne 0 ]; then + echo "Failed to restart ssh service." + return 1 + fi + + return 0 +} + function run_tcl_tests() { local test_to_run=$1 local is_cluster=${3:-false} @@ -2194,6 +2271,8 @@ function run_eloq_test() { echo "/home/$current_user/workspace/eloq_test/ not exists, exit !!!" fi + setup_passwordless_ssh_for_eloq_test + cd /home/$current_user/workspace/eloq_test ./setup @@ -2277,7 +2356,7 @@ function run_eloq_test() { local rocksdb_cloud_object_path=${ROCKSDB_CLOUD_OBJECT_PATH} local txlog_rocksdb_cloud_object_path=${TXLOG_ROCKSDB_CLOUD_OBJECT_PATH} local eloqstore_cloud_store_path=${ELOQSTORE_BUCKET_NAME} - + # rocksdb cloud s3 config for txlog echo "rocksdb_cloud_s3_endpoint_url: ${rocksdb_cloud_s3_endpoint_url}" sed -i "s/rocksdb_cloud_s3_endpoint_url.*=.\+/rocksdb_cloud_s3_endpoint_url=${rocksdb_cloud_s3_endpoint_url_escape}/g" ./storage.cnf @@ -2285,14 +2364,23 @@ function run_eloq_test() { sed -i "s/aws_secret_key.*=.\+/aws_secret_key=${rocksdb_cloud_aws_secret_access_key}/g" ./storage.cnf sed -i "s/rocksdb_cloud_bucket_name.*=.\+/rocksdb_cloud_bucket_name=${rocksdb_cloud_bucket_name}/g" ./storage.cnf - sed -i "s/rocksdb_cloud_s3_endpoint_url.*=.\+/rocksdb_cloud_s3_endpoint_url=${rocksdb_cloud_s3_endpoint_url_escape}/g" ./bootstrap_cnf/*_eloqdss_eloqstore.cnf - sed -i "s/txlog_rocksdb_cloud_s3_endpoint_url.*=.\+/txlog_rocksdb_cloud_s3_endpoint_url=${rocksdb_cloud_s3_endpoint_url_escape}/g" ./bootstrap_cnf/*_eloqdss_eloqstore.cnf - sed -i "s/aws_access_key_id.*=.\+/aws_access_key_id=${rocksdb_cloud_aws_access_key_id}/g" ./bootstrap_cnf/*_eloqdss_eloqstore.cnf - sed -i "s/aws_secret_key.*=.\+/aws_secret_key=${rocksdb_cloud_aws_secret_access_key}/g" ./bootstrap_cnf/*_eloqdss_eloqstore.cnf - sed -i "s/rocksdb_cloud_bucket_name.*=.\+/rocksdb_cloud_bucket_name=${rocksdb_cloud_bucket_name}/g" ./bootstrap_cnf/*_eloqdss_eloqstore.cnf - sed -i "s/txlog_rocksdb_cloud_bucket_name.*=.\+/txlog_rocksdb_cloud_bucket_name=${rocksdb_cloud_bucket_name}/g" ./bootstrap_cnf/*_eloqdss_eloqstore.cnf - sed -i "s/rocksdb_cloud_object_path.*=.\+/rocksdb_cloud_object_path=${rocksdb_cloud_object_path}/g" ./bootstrap_cnf/*_eloqdss_eloqstore.cnf - sed -i "s/txlog_rocksdb_cloud_object_path.*=.\+/txlog_rocksdb_cloud_object_path=${txlog_rocksdb_cloud_object_path}/g" ./bootstrap_cnf/*_eloqdss_eloqstore.cnf + sed -i "s/rocksdb_cloud_s3_endpoint_url.*=.\+/rocksdb_cloud_s3_endpoint_url=${rocksdb_cloud_s3_endpoint_url_escape}/g" ./bootstrap_cnf/*_eloqdss_eloqstore_local.cnf + sed -i "s/txlog_rocksdb_cloud_s3_endpoint_url.*=.\+/txlog_rocksdb_cloud_s3_endpoint_url=${rocksdb_cloud_s3_endpoint_url_escape}/g" ./bootstrap_cnf/*_eloqdss_eloqstore_local.cnf + sed -i "s/aws_access_key_id.*=.\+/aws_access_key_id=${rocksdb_cloud_aws_access_key_id}/g" ./bootstrap_cnf/*_eloqdss_eloqstore_local.cnf + sed -i "s/aws_secret_key.*=.\+/aws_secret_key=${rocksdb_cloud_aws_secret_access_key}/g" ./bootstrap_cnf/*_eloqdss_eloqstore_local.cnf + sed -i "s/rocksdb_cloud_bucket_name.*=.\+/rocksdb_cloud_bucket_name=${rocksdb_cloud_bucket_name}/g" ./bootstrap_cnf/*_eloqdss_eloqstore_local.cnf + sed -i "s/txlog_rocksdb_cloud_bucket_name.*=.\+/txlog_rocksdb_cloud_bucket_name=${rocksdb_cloud_bucket_name}/g" ./bootstrap_cnf/*_eloqdss_eloqstore_local.cnf + sed -i "s/rocksdb_cloud_object_path.*=.\+/rocksdb_cloud_object_path=${rocksdb_cloud_object_path}/g" ./bootstrap_cnf/*_eloqdss_eloqstore_local.cnf + sed -i "s/txlog_rocksdb_cloud_object_path.*=.\+/txlog_rocksdb_cloud_object_path=${txlog_rocksdb_cloud_object_path}/g" ./bootstrap_cnf/*_eloqdss_eloqstore_local.cnf + + sed -i "s/rocksdb_cloud_s3_endpoint_url.*=.\+/rocksdb_cloud_s3_endpoint_url=${rocksdb_cloud_s3_endpoint_url_escape}/g" ./bootstrap_cnf/*_eloqdss_eloqstore_cloud.cnf + sed -i "s/txlog_rocksdb_cloud_s3_endpoint_url.*=.\+/txlog_rocksdb_cloud_s3_endpoint_url=${rocksdb_cloud_s3_endpoint_url_escape}/g" ./bootstrap_cnf/*_eloqdss_eloqstore_cloud.cnf + sed -i "s/aws_access_key_id.*=.\+/aws_access_key_id=${rocksdb_cloud_aws_access_key_id}/g" ./bootstrap_cnf/*_eloqdss_eloqstore_cloud.cnf + sed -i "s/aws_secret_key.*=.\+/aws_secret_key=${rocksdb_cloud_aws_secret_access_key}/g" ./bootstrap_cnf/*_eloqdss_eloqstore_cloud.cnf + sed -i "s/rocksdb_cloud_bucket_name.*=.\+/rocksdb_cloud_bucket_name=${rocksdb_cloud_bucket_name}/g" ./bootstrap_cnf/*_eloqdss_eloqstore_cloud.cnf + sed -i "s/txlog_rocksdb_cloud_bucket_name.*=.\+/txlog_rocksdb_cloud_bucket_name=${rocksdb_cloud_bucket_name}/g" ./bootstrap_cnf/*_eloqdss_eloqstore_cloud.cnf + sed -i "s/rocksdb_cloud_object_path.*=.\+/rocksdb_cloud_object_path=${rocksdb_cloud_object_path}/g" ./bootstrap_cnf/*_eloqdss_eloqstore_cloud.cnf + sed -i "s/txlog_rocksdb_cloud_object_path.*=.\+/txlog_rocksdb_cloud_object_path=${txlog_rocksdb_cloud_object_path}/g" ./bootstrap_cnf/*_eloqdss_eloqstore_cloud.cnf sed -i "s/rocksdb_cloud_s3_endpoint_url.*=.\+/rocksdb_cloud_s3_endpoint_url=${rocksdb_cloud_s3_endpoint_url_escape}/g" ./bootstrap_cnf/eloqdss_server.cnf sed -i "s/txlog_rocksdb_cloud_s3_endpoint_url.*=.\+/txlog_rocksdb_cloud_s3_endpoint_url=${rocksdb_cloud_s3_endpoint_url_escape}/g" ./bootstrap_cnf/eloqdss_server.cnf @@ -2301,32 +2389,38 @@ function run_eloq_test() { sed -i "s/rocksdb_cloud_bucket_name.*=.\+/rocksdb_cloud_bucket_name=${rocksdb_cloud_bucket_name}/g" ./bootstrap_cnf/eloqdss_server.cnf sed -i "s/txlog_rocksdb_cloud_bucket_name.*=.\+/txlog_rocksdb_cloud_bucket_name=${rocksdb_cloud_bucket_name}/g" ./bootstrap_cnf/eloqdss_server.cnf echo "rocksdb_cloud_s3_endpoint_url: ${rocksdb_cloud_s3_endpoint_url}" - + # eloqstore config sed -i "s/eloq_store_cloud_endpoint.*=.\+/eloq_store_cloud_endpoint=${rocksdb_cloud_s3_endpoint_url_escape}/g" ./storage.cnf sed -i "s/eloq_store_cloud_access_key.*=.\+/eloq_store_cloud_access_key=${rocksdb_cloud_aws_access_key_id}/g" ./storage.cnf sed -i "s/eloq_store_cloud_secret_key.*=.\+/eloq_store_cloud_secret_key=${rocksdb_cloud_aws_secret_access_key}/g" ./storage.cnf sed -i "s/eloq_store_cloud_store_path.*=.\+/eloq_store_cloud_store_path=${eloqstore_cloud_store_path}/g" ./storage.cnf - - sed -i "s/eloq_store_cloud_endpoint.*=.\+/eloq_store_cloud_endpoint=${rocksdb_cloud_s3_endpoint_url_escape}/g" ./bootstrap_cnf/*_eloqdss_eloqstore.cnf - sed -i "s/eloq_store_cloud_access_key.*=.\+/eloq_store_cloud_access_key=${rocksdb_cloud_aws_access_key_id}/g" ./bootstrap_cnf/*_eloqdss_eloqstore.cnf - sed -i "s/eloq_store_cloud_secret_key.*=.\+/eloq_store_cloud_secret_key=${rocksdb_cloud_aws_secret_access_key}/g" ./bootstrap_cnf/*_eloqdss_eloqstore.cnf - sed -i "s/eloq_store_cloud_store_path.*=.\+/eloq_store_cloud_store_path=${eloqstore_cloud_store_path}/g" ./bootstrap_cnf/*_eloqdss_eloqstore.cnf + + sed -i "s/eloq_store_cloud_endpoint.*=.\+/eloq_store_cloud_endpoint=${rocksdb_cloud_s3_endpoint_url_escape}/g" ./bootstrap_cnf/*_eloqdss_eloqstore_local.cnf + sed -i "s/eloq_store_cloud_access_key.*=.\+/eloq_store_cloud_access_key=${rocksdb_cloud_aws_access_key_id}/g" ./bootstrap_cnf/*_eloqdss_eloqstore_local.cnf + sed -i "s/eloq_store_cloud_secret_key.*=.\+/eloq_store_cloud_secret_key=${rocksdb_cloud_aws_secret_access_key}/g" ./bootstrap_cnf/*_eloqdss_eloqstore_local.cnf + + sed -i "s/eloq_store_cloud_endpoint.*=.\+/eloq_store_cloud_endpoint=${rocksdb_cloud_s3_endpoint_url_escape}/g" ./bootstrap_cnf/*_eloqdss_eloqstore_cloud.cnf + sed -i "s/eloq_store_cloud_access_key.*=.\+/eloq_store_cloud_access_key=${rocksdb_cloud_aws_access_key_id}/g" ./bootstrap_cnf/*_eloqdss_eloqstore_cloud.cnf + sed -i "s/eloq_store_cloud_secret_key.*=.\+/eloq_store_cloud_secret_key=${rocksdb_cloud_aws_secret_access_key}/g" ./bootstrap_cnf/*_eloqdss_eloqstore_cloud.cnf + sed -i "s/eloq_store_cloud_store_path.*=.\+/eloq_store_cloud_store_path=${eloqstore_cloud_store_path}/g" ./bootstrap_cnf/*_eloqdss_eloqstore_cloud.cnf # run single/multi test rm -rf runtime/* - python3 redis_test/multi_test/smoke_test.py --dbtype redis --storage eloqdss-eloqstore --install_path ${eloqkv_install_path} --bootstrap true + python3 redis_test/multi_test/smoke_test.py --dbtype redis --storage eloqdss-eloqstore-cloud --install_path ${eloqkv_install_path} --bootstrap true - python3 redis_test/multi_test/cluster_rolling_upgrade.py --dbtype redis --storage eloqdss-eloqstore --install_path ${eloqkv_install_path} --bootstrap true - python3 redis_test/multi_test/cluster_scale_test.py --dbtype redis --storage eloqdss-eloqstore --install_path ${eloqkv_install_path} --bootstrap true + python3 redis_test/multi_test/cluster_rolling_upgrade.py --dbtype redis --storage eloqdss-eloqstore-cloud --install_path ${eloqkv_install_path} --bootstrap true + python3 redis_test/multi_test/cluster_scale_test.py --dbtype redis --storage eloqdss-eloqstore-cloud --install_path ${eloqkv_install_path} --bootstrap true # run log service scale test rm -rf runtime/* - python3 redis_test/log_service_test/log_service_scale_test.py --dbtype redis --storage eloqdss-eloqstore --install_path ${eloqkv_install_path} + python3 redis_test/log_service_test/log_service_scale_test.py --dbtype redis --storage eloqdss-eloqstore-cloud --install_path ${eloqkv_install_path} # run standby test rm -rf runtime/* - python3 run_tests.py --dbtype redis --group standby --storage eloqdss-eloqstore --install_path ${eloqkv_install_path} --bootstrap true + python3 run_tests.py --dbtype redis --group standby --storage eloqdss-eloqstore-local --install_path ${eloqkv_install_path} --bootstrap true + rm -rf runtime/* + python3 run_tests.py --dbtype redis --group standby --storage eloqdss-eloqstore-cloud --install_path ${eloqkv_install_path} --bootstrap true rm -rf runtime/* # rm -rf runtime/* # python3 redis_test/standby_test/test_with_kv.py --dbtype redis --storage eloqdss-eloqstore --install_path ${eloqkv_install_path} diff --git a/data_substrate b/data_substrate index 500ffa32..3daf3499 160000 --- a/data_substrate +++ b/data_substrate @@ -1 +1 @@ -Subproject commit 500ffa32730595e07631e43eff0ff128b3f389c5 +Subproject commit 3daf3499f8fa9e36a4f9cffaa87bde0c4b754a1e diff --git a/eloqkv.ini b/eloqkv.ini index 6bb3dd8c..b40dbaad 100644 --- a/eloqkv.ini +++ b/eloqkv.ini @@ -89,7 +89,9 @@ node_group_replica_num = 1 # auto_redirect = false [store] -# Shard data directories separated by ',' when multiple. By default under eloq_data_path. +# Shard data directories. +# Format: path1,path2,...[,pathN][:weight1,weight2,...,weightN] +# Weights are optional. When omitted, disk-capacity-based weighting is used. # eloq_store_data_path_list= # Maximum number of open files allowed. @@ -171,6 +173,9 @@ node_group_replica_num = 1 # [CLOUD OPTIONAL] Reuse existing local files when restoring from cloud. # eloq_store_reuse_local_files=false +# Maximum number of concurrent standby rsync/ssh jobs managed by EloqStore. +# eloq_store_standby_max_concurrency=100 + # size of each data page. # eloq_store_data_page_size=4096 From e0f3e6de60bbeaf2851a7f4f29263f10b588d57d Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Tue, 17 Mar 2026 01:53:54 +0000 Subject: [PATCH 2/7] fix --- data_substrate | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data_substrate b/data_substrate index 3daf3499..9a440711 160000 --- a/data_substrate +++ b/data_substrate @@ -1 +1 @@ -Subproject commit 3daf3499f8fa9e36a4f9cffaa87bde0c4b754a1e +Subproject commit 9a440711b22ad78e0e06f0d0c023bc7ff6211d91 From e1b9b60ca9c63318c93b4521e66bd673d98c7008 Mon Sep 17 00:00:00 2001 From: Chen Zhao Date: Tue, 17 Mar 2026 03:53:40 +0000 Subject: [PATCH 3/7] update --- data_substrate | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data_substrate b/data_substrate index 9a440711..8cca93e1 160000 --- a/data_substrate +++ b/data_substrate @@ -1 +1 @@ -Subproject commit 9a440711b22ad78e0e06f0d0c023bc7ff6211d91 +Subproject commit 8cca93e17b3362f7bd0198a88affe34a0fdb36c3 From d35f5494a42647e8bc8a2da6158a3da2f23bf0cc Mon Sep 17 00:00:00 2001 From: Chen Zhao Date: Tue, 17 Mar 2026 04:19:21 +0000 Subject: [PATCH 4/7] update --- data_substrate | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data_substrate b/data_substrate index 8cca93e1..cd09b839 160000 --- a/data_substrate +++ b/data_substrate @@ -1 +1 @@ -Subproject commit 8cca93e17b3362f7bd0198a88affe34a0fdb36c3 +Subproject commit cd09b8390ea4ea6551a7f7b5ba258507756b01cb From 7082efcfa93409c0cb37ddd3fb871014d494106d Mon Sep 17 00:00:00 2001 From: Chen Zhao Date: Tue, 17 Mar 2026 05:08:44 +0000 Subject: [PATCH 5/7] update --- data_substrate | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data_substrate b/data_substrate index cd09b839..da4a4728 160000 --- a/data_substrate +++ b/data_substrate @@ -1 +1 @@ -Subproject commit cd09b8390ea4ea6551a7f7b5ba258507756b01cb +Subproject commit da4a47280c05b63461e7d6f93b9676000e59b065 From 7eccbb2dc3f594f0b13bae1623b209338a09e1b2 Mon Sep 17 00:00:00 2001 From: Chen Zhao Date: Tue, 17 Mar 2026 17:23:10 +0800 Subject: [PATCH 6/7] fix --- data_substrate | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data_substrate b/data_substrate index da4a4728..1dc28b7f 160000 --- a/data_substrate +++ b/data_substrate @@ -1 +1 @@ -Subproject commit da4a47280c05b63461e7d6f93b9676000e59b065 +Subproject commit 1dc28b7fde7b0f83a4b8056e5207eba179a66709 From 0330294dbbc52c9a634b123e684b96831022bea1 Mon Sep 17 00:00:00 2001 From: Chen Zhao Date: Tue, 17 Mar 2026 18:58:43 +0800 Subject: [PATCH 7/7] update --- data_substrate | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data_substrate b/data_substrate index 1dc28b7f..1a4729b7 160000 --- a/data_substrate +++ b/data_substrate @@ -1 +1 @@ -Subproject commit 1dc28b7fde7b0f83a4b8056e5207eba179a66709 +Subproject commit 1a4729b71e08b72f0461bd3fbe107fd728592516