Skip to content

feat: adapt to eloqstore local standby feature #451

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
thweetkomputer merged 7 commits into from
Mar 17, 2026
Merged

feat: adapt to eloqstore local standby feature #451

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
136ff33
feat: adapt to eloqstore local standby feature
thweetkomputer Mar 4, 2026
e0f3e6d
fix
thweetkomputer Mar 17, 2026
e1b9b60
update
thweetkomputer Mar 17, 2026
d35f549
update
thweetkomputer Mar 17, 2026
7082efc
update
thweetkomputer Mar 17, 2026
7eccbb2
fix
thweetkomputer Mar 17, 2026
0330294
update
thweetkomputer Mar 17, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
138 changes: 116 additions & 22 deletions concourse/scripts/common.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,15 +46,15 @@ function wait_until_ready() {

# Function to wait until server is finished
function wait_until_finished() {
local timeout=120
local timeout=300
local elapsed=0
local interval=1

while [ $(ps aux | grep eloqkv | grep -v grep | grep -v launch_sv | grep -v dss_server | wc -l) -gt 0 ]; do
sleep $interval
elapsed=$((elapsed + interval))
if [ $elapsed -ge $timeout ]; then
echo "Timeout: Process still running after 20 seconds."
echo "Timeout: Process still running after $timeout seconds."
# list eloqkv still alived
ps aux | grep eloqkv | grep -v grep | grep -v launch_sv | grep -v dss_server
return 1
Expand All @@ -63,6 +63,83 @@ function wait_until_finished() {
return 0
}

function setup_passwordless_ssh_for_eloq_test() {
local user_home="/home/$current_user"
local ssh_dir="${user_home}/.ssh"
local private_key="${ssh_dir}/id_ed25519"
local public_key="${private_key}.pub"
local authorized_keys="${ssh_dir}/authorized_keys"
local sshd_config="/etc/ssh/sshd_config"
local user_group

mkdir -p "${ssh_dir}"
chmod 700 "${ssh_dir}"

if [ ! -f "${private_key}" ]; then
ssh-keygen -q -t ed25519 -N "" -f "${private_key}"
elif [ ! -f "${public_key}" ]; then
ssh-keygen -y -f "${private_key}" > "${public_key}"
fi

touch "${authorized_keys}"
chmod 600 "${authorized_keys}"
if ! grep -qxF "$(cat "${public_key}")" "${authorized_keys}"; then
cat "${public_key}" >> "${authorized_keys}"
echo >> "${authorized_keys}"
fi

user_group=$(id -gn "${current_user}" 2>/dev/null || true)
if [ -n "${user_group}" ]; then
chown -R "${current_user}:${user_group}" "${ssh_dir}"
else
chown -R "${current_user}" "${ssh_dir}"
fi

if [ "$(id -u)" -eq 0 ]; then
if grep -qE '^[#[:space:]]*MaxStartups[[:space:]]+' "${sshd_config}"; then
sed -i 's/^[#[:space:]]*MaxStartups[[:space:]].*/MaxStartups 200/' "${sshd_config}"
else
printf '\nMaxStartups 200\n' >> "${sshd_config}"
fi

if grep -qE '^[#[:space:]]*PubkeyAuthentication[[:space:]]+' "${sshd_config}"; then
sed -i 's/^[#[:space:]]*PubkeyAuthentication[[:space:]].*/PubkeyAuthentication yes/' "${sshd_config}"
else
printf 'PubkeyAuthentication yes\n' >> "${sshd_config}"
fi
else
if sudo -n test -f "${sshd_config}"; then
if sudo -n grep -qE '^[#[:space:]]*MaxStartups[[:space:]]+' "${sshd_config}"; then
sudo -n sed -i 's/^[#[:space:]]*MaxStartups[[:space:]].*/MaxStartups 200/' "${sshd_config}"
else
printf '\nMaxStartups 200\n' | sudo -n tee -a "${sshd_config}" >/dev/null
fi

if sudo -n grep -qE '^[#[:space:]]*PubkeyAuthentication[[:space:]]+' "${sshd_config}"; then
sudo -n sed -i 's/^[#[:space:]]*PubkeyAuthentication[[:space:]].*/PubkeyAuthentication yes/' "${sshd_config}"
else
printf 'PubkeyAuthentication yes\n' | sudo -n tee -a "${sshd_config}" >/dev/null
fi
else
echo "sudo access is required to update ${sshd_config} and restart ssh."
return 1
fi
fi

if [ "$(id -u)" -eq 0 ]; then
systemctl restart ssh || service ssh restart
else
sudo -n systemctl restart ssh || sudo -n service ssh restart
fi

if [ $? -ne 0 ]; then
echo "Failed to restart ssh service."
return 1
fi

return 0
}

function run_tcl_tests() {
local test_to_run=$1
local is_cluster=${3:-false}
Expand Down Expand Up @@ -2194,6 +2271,8 @@ function run_eloq_test() {
echo "/home/$current_user/workspace/eloq_test/ not exists, exit !!!"
fi

setup_passwordless_ssh_for_eloq_test

cd /home/$current_user/workspace/eloq_test
./setup

Expand Down Expand Up @@ -2277,22 +2356,31 @@ function run_eloq_test() {
local rocksdb_cloud_object_path=${ROCKSDB_CLOUD_OBJECT_PATH}
local txlog_rocksdb_cloud_object_path=${TXLOG_ROCKSDB_CLOUD_OBJECT_PATH}
local eloqstore_cloud_store_path=${ELOQSTORE_BUCKET_NAME}

# rocksdb cloud s3 config for txlog
echo "rocksdb_cloud_s3_endpoint_url: ${rocksdb_cloud_s3_endpoint_url}"
sed -i "s/rocksdb_cloud_s3_endpoint_url.*=.\+/rocksdb_cloud_s3_endpoint_url=${rocksdb_cloud_s3_endpoint_url_escape}/g" ./storage.cnf
sed -i "s/aws_access_key_id.*=.\+/aws_access_key_id=${rocksdb_cloud_aws_access_key_id}/g" ./storage.cnf
sed -i "s/aws_secret_key.*=.\+/aws_secret_key=${rocksdb_cloud_aws_secret_access_key}/g" ./storage.cnf
sed -i "s/rocksdb_cloud_bucket_name.*=.\+/rocksdb_cloud_bucket_name=${rocksdb_cloud_bucket_name}/g" ./storage.cnf

sed -i "s/rocksdb_cloud_s3_endpoint_url.*=.\+/rocksdb_cloud_s3_endpoint_url=${rocksdb_cloud_s3_endpoint_url_escape}/g" ./bootstrap_cnf/*_eloqdss_eloqstore.cnf
sed -i "s/txlog_rocksdb_cloud_s3_endpoint_url.*=.\+/txlog_rocksdb_cloud_s3_endpoint_url=${rocksdb_cloud_s3_endpoint_url_escape}/g" ./bootstrap_cnf/*_eloqdss_eloqstore.cnf
sed -i "s/aws_access_key_id.*=.\+/aws_access_key_id=${rocksdb_cloud_aws_access_key_id}/g" ./bootstrap_cnf/*_eloqdss_eloqstore.cnf
sed -i "s/aws_secret_key.*=.\+/aws_secret_key=${rocksdb_cloud_aws_secret_access_key}/g" ./bootstrap_cnf/*_eloqdss_eloqstore.cnf
sed -i "s/rocksdb_cloud_bucket_name.*=.\+/rocksdb_cloud_bucket_name=${rocksdb_cloud_bucket_name}/g" ./bootstrap_cnf/*_eloqdss_eloqstore.cnf
sed -i "s/txlog_rocksdb_cloud_bucket_name.*=.\+/txlog_rocksdb_cloud_bucket_name=${rocksdb_cloud_bucket_name}/g" ./bootstrap_cnf/*_eloqdss_eloqstore.cnf
sed -i "s/rocksdb_cloud_object_path.*=.\+/rocksdb_cloud_object_path=${rocksdb_cloud_object_path}/g" ./bootstrap_cnf/*_eloqdss_eloqstore.cnf
sed -i "s/txlog_rocksdb_cloud_object_path.*=.\+/txlog_rocksdb_cloud_object_path=${txlog_rocksdb_cloud_object_path}/g" ./bootstrap_cnf/*_eloqdss_eloqstore.cnf
sed -i "s/rocksdb_cloud_s3_endpoint_url.*=.\+/rocksdb_cloud_s3_endpoint_url=${rocksdb_cloud_s3_endpoint_url_escape}/g" ./bootstrap_cnf/*_eloqdss_eloqstore_local.cnf
sed -i "s/txlog_rocksdb_cloud_s3_endpoint_url.*=.\+/txlog_rocksdb_cloud_s3_endpoint_url=${rocksdb_cloud_s3_endpoint_url_escape}/g" ./bootstrap_cnf/*_eloqdss_eloqstore_local.cnf
sed -i "s/aws_access_key_id.*=.\+/aws_access_key_id=${rocksdb_cloud_aws_access_key_id}/g" ./bootstrap_cnf/*_eloqdss_eloqstore_local.cnf
sed -i "s/aws_secret_key.*=.\+/aws_secret_key=${rocksdb_cloud_aws_secret_access_key}/g" ./bootstrap_cnf/*_eloqdss_eloqstore_local.cnf
sed -i "s/rocksdb_cloud_bucket_name.*=.\+/rocksdb_cloud_bucket_name=${rocksdb_cloud_bucket_name}/g" ./bootstrap_cnf/*_eloqdss_eloqstore_local.cnf
sed -i "s/txlog_rocksdb_cloud_bucket_name.*=.\+/txlog_rocksdb_cloud_bucket_name=${rocksdb_cloud_bucket_name}/g" ./bootstrap_cnf/*_eloqdss_eloqstore_local.cnf
sed -i "s/rocksdb_cloud_object_path.*=.\+/rocksdb_cloud_object_path=${rocksdb_cloud_object_path}/g" ./bootstrap_cnf/*_eloqdss_eloqstore_local.cnf
sed -i "s/txlog_rocksdb_cloud_object_path.*=.\+/txlog_rocksdb_cloud_object_path=${txlog_rocksdb_cloud_object_path}/g" ./bootstrap_cnf/*_eloqdss_eloqstore_local.cnf

sed -i "s/rocksdb_cloud_s3_endpoint_url.*=.\+/rocksdb_cloud_s3_endpoint_url=${rocksdb_cloud_s3_endpoint_url_escape}/g" ./bootstrap_cnf/*_eloqdss_eloqstore_cloud.cnf
sed -i "s/txlog_rocksdb_cloud_s3_endpoint_url.*=.\+/txlog_rocksdb_cloud_s3_endpoint_url=${rocksdb_cloud_s3_endpoint_url_escape}/g" ./bootstrap_cnf/*_eloqdss_eloqstore_cloud.cnf
sed -i "s/aws_access_key_id.*=.\+/aws_access_key_id=${rocksdb_cloud_aws_access_key_id}/g" ./bootstrap_cnf/*_eloqdss_eloqstore_cloud.cnf
sed -i "s/aws_secret_key.*=.\+/aws_secret_key=${rocksdb_cloud_aws_secret_access_key}/g" ./bootstrap_cnf/*_eloqdss_eloqstore_cloud.cnf
sed -i "s/rocksdb_cloud_bucket_name.*=.\+/rocksdb_cloud_bucket_name=${rocksdb_cloud_bucket_name}/g" ./bootstrap_cnf/*_eloqdss_eloqstore_cloud.cnf
sed -i "s/txlog_rocksdb_cloud_bucket_name.*=.\+/txlog_rocksdb_cloud_bucket_name=${rocksdb_cloud_bucket_name}/g" ./bootstrap_cnf/*_eloqdss_eloqstore_cloud.cnf
sed -i "s/rocksdb_cloud_object_path.*=.\+/rocksdb_cloud_object_path=${rocksdb_cloud_object_path}/g" ./bootstrap_cnf/*_eloqdss_eloqstore_cloud.cnf
sed -i "s/txlog_rocksdb_cloud_object_path.*=.\+/txlog_rocksdb_cloud_object_path=${txlog_rocksdb_cloud_object_path}/g" ./bootstrap_cnf/*_eloqdss_eloqstore_cloud.cnf

sed -i "s/rocksdb_cloud_s3_endpoint_url.*=.\+/rocksdb_cloud_s3_endpoint_url=${rocksdb_cloud_s3_endpoint_url_escape}/g" ./bootstrap_cnf/eloqdss_server.cnf
sed -i "s/txlog_rocksdb_cloud_s3_endpoint_url.*=.\+/txlog_rocksdb_cloud_s3_endpoint_url=${rocksdb_cloud_s3_endpoint_url_escape}/g" ./bootstrap_cnf/eloqdss_server.cnf
Expand All @@ -2301,32 +2389,38 @@ function run_eloq_test() {
sed -i "s/rocksdb_cloud_bucket_name.*=.\+/rocksdb_cloud_bucket_name=${rocksdb_cloud_bucket_name}/g" ./bootstrap_cnf/eloqdss_server.cnf
sed -i "s/txlog_rocksdb_cloud_bucket_name.*=.\+/txlog_rocksdb_cloud_bucket_name=${rocksdb_cloud_bucket_name}/g" ./bootstrap_cnf/eloqdss_server.cnf
echo "rocksdb_cloud_s3_endpoint_url: ${rocksdb_cloud_s3_endpoint_url}"

# eloqstore config
sed -i "s/eloq_store_cloud_endpoint.*=.\+/eloq_store_cloud_endpoint=${rocksdb_cloud_s3_endpoint_url_escape}/g" ./storage.cnf
sed -i "s/eloq_store_cloud_access_key.*=.\+/eloq_store_cloud_access_key=${rocksdb_cloud_aws_access_key_id}/g" ./storage.cnf
sed -i "s/eloq_store_cloud_secret_key.*=.\+/eloq_store_cloud_secret_key=${rocksdb_cloud_aws_secret_access_key}/g" ./storage.cnf
sed -i "s/eloq_store_cloud_store_path.*=.\+/eloq_store_cloud_store_path=${eloqstore_cloud_store_path}/g" ./storage.cnf

sed -i "s/eloq_store_cloud_endpoint.*=.\+/eloq_store_cloud_endpoint=${rocksdb_cloud_s3_endpoint_url_escape}/g" ./bootstrap_cnf/*_eloqdss_eloqstore.cnf
sed -i "s/eloq_store_cloud_access_key.*=.\+/eloq_store_cloud_access_key=${rocksdb_cloud_aws_access_key_id}/g" ./bootstrap_cnf/*_eloqdss_eloqstore.cnf
sed -i "s/eloq_store_cloud_secret_key.*=.\+/eloq_store_cloud_secret_key=${rocksdb_cloud_aws_secret_access_key}/g" ./bootstrap_cnf/*_eloqdss_eloqstore.cnf
sed -i "s/eloq_store_cloud_store_path.*=.\+/eloq_store_cloud_store_path=${eloqstore_cloud_store_path}/g" ./bootstrap_cnf/*_eloqdss_eloqstore.cnf

sed -i "s/eloq_store_cloud_endpoint.*=.\+/eloq_store_cloud_endpoint=${rocksdb_cloud_s3_endpoint_url_escape}/g" ./bootstrap_cnf/*_eloqdss_eloqstore_local.cnf
sed -i "s/eloq_store_cloud_access_key.*=.\+/eloq_store_cloud_access_key=${rocksdb_cloud_aws_access_key_id}/g" ./bootstrap_cnf/*_eloqdss_eloqstore_local.cnf
sed -i "s/eloq_store_cloud_secret_key.*=.\+/eloq_store_cloud_secret_key=${rocksdb_cloud_aws_secret_access_key}/g" ./bootstrap_cnf/*_eloqdss_eloqstore_local.cnf

sed -i "s/eloq_store_cloud_endpoint.*=.\+/eloq_store_cloud_endpoint=${rocksdb_cloud_s3_endpoint_url_escape}/g" ./bootstrap_cnf/*_eloqdss_eloqstore_cloud.cnf
sed -i "s/eloq_store_cloud_access_key.*=.\+/eloq_store_cloud_access_key=${rocksdb_cloud_aws_access_key_id}/g" ./bootstrap_cnf/*_eloqdss_eloqstore_cloud.cnf
sed -i "s/eloq_store_cloud_secret_key.*=.\+/eloq_store_cloud_secret_key=${rocksdb_cloud_aws_secret_access_key}/g" ./bootstrap_cnf/*_eloqdss_eloqstore_cloud.cnf
sed -i "s/eloq_store_cloud_store_path.*=.\+/eloq_store_cloud_store_path=${eloqstore_cloud_store_path}/g" ./bootstrap_cnf/*_eloqdss_eloqstore_cloud.cnf

# run single/multi test
rm -rf runtime/*
python3 redis_test/multi_test/smoke_test.py --dbtype redis --storage eloqdss-eloqstore --install_path ${eloqkv_install_path} --bootstrap true
python3 redis_test/multi_test/smoke_test.py --dbtype redis --storage eloqdss-eloqstore-cloud --install_path ${eloqkv_install_path} --bootstrap true

python3 redis_test/multi_test/cluster_rolling_upgrade.py --dbtype redis --storage eloqdss-eloqstore --install_path ${eloqkv_install_path} --bootstrap true
python3 redis_test/multi_test/cluster_scale_test.py --dbtype redis --storage eloqdss-eloqstore --install_path ${eloqkv_install_path} --bootstrap true
python3 redis_test/multi_test/cluster_rolling_upgrade.py --dbtype redis --storage eloqdss-eloqstore-cloud --install_path ${eloqkv_install_path} --bootstrap true
python3 redis_test/multi_test/cluster_scale_test.py --dbtype redis --storage eloqdss-eloqstore-cloud --install_path ${eloqkv_install_path} --bootstrap true

# run log service scale test
rm -rf runtime/*
python3 redis_test/log_service_test/log_service_scale_test.py --dbtype redis --storage eloqdss-eloqstore --install_path ${eloqkv_install_path}
python3 redis_test/log_service_test/log_service_scale_test.py --dbtype redis --storage eloqdss-eloqstore-cloud --install_path ${eloqkv_install_path}

# run standby test
rm -rf runtime/*
python3 run_tests.py --dbtype redis --group standby --storage eloqdss-eloqstore --install_path ${eloqkv_install_path} --bootstrap true
python3 run_tests.py --dbtype redis --group standby --storage eloqdss-eloqstore-local --install_path ${eloqkv_install_path} --bootstrap true
rm -rf runtime/*
python3 run_tests.py --dbtype redis --group standby --storage eloqdss-eloqstore-cloud --install_path ${eloqkv_install_path} --bootstrap true
rm -rf runtime/*
# rm -rf runtime/*
# python3 redis_test/standby_test/test_with_kv.py --dbtype redis --storage eloqdss-eloqstore --install_path ${eloqkv_install_path}
Expand Down
2 changes: 1 addition & 1 deletion data_substrate
Open in desktop
Submodule data_substrate updated 69 files
+64 −0 .cursor_knowledge/standby_snapshot_subscription_barrier_design.md
+101 −0 .cursor_knowledge/standby_snapshot_subscription_barrier_implementation.md
+1 −1 log_service
+7 −0 store_handler/bigtable_handler.cpp
+3 −0 store_handler/bigtable_handler.h
+229 −24 store_handler/data_store_service_client.cpp
+42 −7 store_handler/data_store_service_client.h
+45 −4 store_handler/data_store_service_client_closure.cpp
+8 −0 store_handler/data_store_service_client_closure.h
+6 −0 store_handler/dynamo_handler.cpp
+1 −0 store_handler/dynamo_handler.h
+2 −1 store_handler/eloq_data_store_service/build_eloq_store.cmake
+31 −2 store_handler/eloq_data_store_service/data_store.h
+313 −45 store_handler/eloq_data_store_service/data_store_service.cpp
+26 −4 store_handler/eloq_data_store_service/data_store_service.h
+45 −26 store_handler/eloq_data_store_service/eloq_store_config.cpp
+0 −3 store_handler/eloq_data_store_service/eloq_store_config.h
+110 −47 store_handler/eloq_data_store_service/eloq_store_data_store.cpp
+17 −4 store_handler/eloq_data_store_service/eloq_store_data_store.h
+22 −0 store_handler/eloq_data_store_service/eloq_store_data_store_factory.h
+1 −1 store_handler/eloq_data_store_service/eloqstore
+5 −3 store_handler/eloq_data_store_service/rocksdb_data_store_common.h
+22 −4 store_handler/rocksdb_handler.cpp
+9 −4 store_handler/rocksdb_handler.h
+3 −1 tx_service/include/cc/cc_handler.h
+25 −0 tx_service/include/cc/cc_map.h
+1 −2 tx_service/include/cc/cc_page_clean_guard.h
+52 −31 tx_service/include/cc/cc_req_misc.h
+415 −546 tx_service/include/cc/cc_request.h
+46 −0 tx_service/include/cc/cc_shard.h
+21 −368 tx_service/include/cc/ccm_scanner.h
+3 −1 tx_service/include/cc/local_cc_handler.h
+10 −5 tx_service/include/cc/local_cc_shards.h
+22 −22 tx_service/include/cc/object_cc_map.h
+102 −0 tx_service/include/cc/range_cc_map.h
+56 −75 tx_service/include/cc/range_slice.h
+546 −911 tx_service/include/cc/template_cc_map.h
+11 −2 tx_service/include/data_sync_task.h
+31 −0 tx_service/include/fault/log_replay_service.h
+24 −4 tx_service/include/proto/cc_request.proto
+13 −2 tx_service/include/read_write_entry.h
+42 −0 tx_service/include/remote/cc_node_service.h
+3 −1 tx_service/include/remote/remote_cc_handler.h
+2 −2 tx_service/include/remote/remote_cc_request.h
+17 −12 tx_service/include/sk_generator.h
+53 −3 tx_service/include/store/data_store_handler.h
+87 −3 tx_service/include/store/snapshot_manager.h
+24 −94 tx_service/include/tx_operation_result.h
+7 −0 tx_service/include/type.h
+54 −0 tx_service/src/cc/cc_map.cpp
+130 −117 tx_service/src/cc/cc_req_misc.cpp
+43 −0 tx_service/src/cc/cc_shard.cpp
+30 −34 tx_service/src/cc/local_cc_handler.cpp
+319 −370 tx_service/src/cc/local_cc_shards.cpp
+14 −31 tx_service/src/cc/range_slice.cpp
+15 −3 tx_service/src/checkpointer.cpp
+36 −2 tx_service/src/data_sync_task.cpp
+104 −2 tx_service/src/fault/cc_node.cpp
+117 −27 tx_service/src/fault/log_replay_service.cpp
+353 −17 tx_service/src/remote/cc_node_service.cpp
+13 −47 tx_service/src/remote/cc_stream_receiver.cpp
+10 −11 tx_service/src/remote/remote_cc_handler.cpp
+56 −98 tx_service/src/remote/remote_cc_request.cpp
+27 −0 tx_service/src/sharder.cpp
+131 −124 tx_service/src/sk_generator.cpp
+118 −13 tx_service/src/standby.cpp
+817 −81 tx_service/src/store/snapshot_manager.cpp
+28 −14 tx_service/src/tx_execution.cpp
+70 −19 tx_service/src/tx_operation.cpp
7 changes: 6 additions & 1 deletion eloqkv.ini
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,9 @@ node_group_replica_num = 1
# auto_redirect = false

[store]
# Shard data directories separated by ',' when multiple. By default under eloq_data_path.
# Shard data directories.
# Format: path1,path2,...[,pathN][:weight1,weight2,...,weightN]
# Weights are optional. When omitted, disk-capacity-based weighting is used.
# eloq_store_data_path_list=

# Maximum number of open files allowed.
Expand Down Expand Up @@ -171,6 +173,9 @@ node_group_replica_num = 1
# [CLOUD OPTIONAL] Reuse existing local files when restoring from cloud.
# eloq_store_reuse_local_files=false

# Maximum number of concurrent standby rsync/ssh jobs managed by EloqStore.
# eloq_store_standby_max_concurrency=100

# size of each data page.
# eloq_store_data_page_size=4096

Expand Down
Loading