From bf028d7d90ad4bc87d9d67161bba75f938a666ac Mon Sep 17 00:00:00 2001 From: liunyl Date: Sat, 18 Oct 2025 08:09:17 +0000 Subject: [PATCH 01/10] converge restructure --- .gitignore | 1 + .gitmodules | 16 +- CMakeLists.txt | 6 +- client/CMakeLists.txt | 4 +- data_substrate/CMakeLists.txt | 285 +++ .../build_eloq_log_service.cmake | 3 +- .../build_eloq_metrics.cmake | 2 + .../build_log_service.cmake | 2 +- .../build_tx_service.cmake | 169 +- data_substrate/core/CONFIGURATION.md | 140 ++ data_substrate/core/CONFIG_MAPPING.md | 209 ++ .../core/config/sample_data_substrate.ini | 41 + data_substrate/core/include/data_substrate.h | 257 +++ data_substrate/core/include/mysql_metrics.h | 81 + data_substrate/core/src/data_substrate.cpp | 528 +++++ data_substrate/core/src/log_init.cpp | 541 +++++ data_substrate/core/src/metrics_init.cpp | 163 ++ data_substrate/core/src/storage_init.cpp | 365 ++++ data_substrate/core/src/tx_service_init.cpp | 253 +++ {storage/eloq => data_substrate}/eloq_metrics | 0 {storage/eloq => data_substrate}/log_service | 0 data_substrate/store_handler | 1 + data_substrate/tx_service | 1 + scripts/mysql_install_db.sh | 27 +- sql/CMakeLists.txt | 4 +- sql/main.cc | 25 + sql/mdl.h | 2 +- sql/mysql_metrics.h | 4 +- sql/mysqld.cc | 38 +- sql/sql_class.h | 2 +- sql/sql_parse.cc | 2 +- sql/sql_select.cc | 2 +- sql/threadpool_generic.h | 2 +- storage/eloq/CMakeLists.txt | 494 ++--- storage/eloq/eloq_catalog_factory.h | 2 +- storage/eloq/eloq_i_s.cc | 2 +- storage/eloq/ha_eloq.cc | 1849 +---------------- storage/eloq/ha_eloq.h | 2 +- storage/eloq/metrics_registry_impl.cpp | 80 - storage/eloq/metrics_registry_impl.h | 67 - storage/eloq/store_handler | 1 - storage/eloq/tx_service | 1 - 42 files changed, 3361 insertions(+), 2313 deletions(-) create mode 100644 data_substrate/CMakeLists.txt rename {storage/eloq => data_substrate}/build_eloq_log_service.cmake (99%) rename {storage/eloq => data_substrate}/build_eloq_metrics.cmake (93%) rename {storage/eloq => data_substrate}/build_log_service.cmake (99%) rename {storage/eloq => data_substrate}/build_tx_service.cmake (64%) create mode 100644 data_substrate/core/CONFIGURATION.md create mode 100644 data_substrate/core/CONFIG_MAPPING.md create mode 100644 data_substrate/core/config/sample_data_substrate.ini create mode 100644 data_substrate/core/include/data_substrate.h create mode 100644 data_substrate/core/include/mysql_metrics.h create mode 100644 data_substrate/core/src/data_substrate.cpp create mode 100644 data_substrate/core/src/log_init.cpp create mode 100644 data_substrate/core/src/metrics_init.cpp create mode 100644 data_substrate/core/src/storage_init.cpp create mode 100644 data_substrate/core/src/tx_service_init.cpp rename {storage/eloq => data_substrate}/eloq_metrics (100%) rename {storage/eloq => data_substrate}/log_service (100%) create mode 160000 data_substrate/store_handler create mode 160000 data_substrate/tx_service delete mode 100644 storage/eloq/metrics_registry_impl.cpp delete mode 100644 storage/eloq/metrics_registry_impl.h delete mode 160000 storage/eloq/store_handler delete mode 160000 storage/eloq/tx_service diff --git a/.gitignore b/.gitignore index 5233e6382a6..b1921f1e071 100644 --- a/.gitignore +++ b/.gitignore @@ -544,6 +544,7 @@ compile_commands.json # Visual Studio Code workspace .vscode/ +.cursor/ # Clion && other JetBrains ides /.idea/ diff --git a/.gitmodules b/.gitmodules index f680171f7ba..975797de598 100644 --- a/.gitmodules +++ b/.gitmodules @@ -17,15 +17,15 @@ [submodule "storage/columnstore/columnstore"] path = storage/columnstore/columnstore url = https://github.com/mariadb-corporation/mariadb-columnstore-engine.git -[submodule "storage/eloq/eloq_metrics"] - path = storage/eloq/eloq_metrics +[submodule "data_substrate/eloq_metrics"] + path = data_substrate/eloq_metrics url = https://github.com/eloqdata/eloq-metrics.git -[submodule "storage/eloq/tx_service"] - path = storage/eloq/tx_service +[submodule "data_substrate/tx_service"] + path = data_substrate/tx_service url = https://github.com/eloqdata/tx_service.git -[submodule "storage/eloq/store_handler"] - path = storage/eloq/store_handler +[submodule "data_substrate/store_handler"] + path = data_substrate/store_handler url = https://github.com/eloqdata/store_handler.git -[submodule "storage/eloq/log_service"] - path = storage/eloq/log_service +[submodule "data_substrate/log_service"] + path = data_substrate/log_service url = https://github.com/eloqdata/log_service.git diff --git a/CMakeLists.txt b/CMakeLists.txt index 84ac54ce155..4f5dbbf4389 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -450,10 +450,14 @@ INCLUDE(cpack_deb) SET(PYTHON_SHEBANG "/usr/bin/env python3" CACHE STRING "python shebang") MARK_AS_ADVANCED(PYTHON_SHEBANG) - # Add storage engines and plugins. CONFIGURE_PLUGINS() + +# Build data substrate library first (before sql/ and storage/) +IF(ELOQ_MODULE_ENABLED) + ADD_SUBDIRECTORY(data_substrate) +ENDIF() ADD_SUBDIRECTORY(include) ADD_SUBDIRECTORY(dbug) ADD_SUBDIRECTORY(strings) diff --git a/client/CMakeLists.txt b/client/CMakeLists.txt index 27fc76e6321..ac718ea8e26 100644 --- a/client/CMakeLists.txt +++ b/client/CMakeLists.txt @@ -44,9 +44,9 @@ IF(UNIX) SET_TARGET_PROPERTIES(mariadb PROPERTIES ENABLE_EXPORTS TRUE) ENDIF(UNIX) -IF(PLUGIN_ELOQ) +# IF(PLUGIN_ELOQ) ADD_DEFINITIONS(-DWITH_ELOQ_STORAGE_ENGINE) -ENDIF() +# ENDIF() MYSQL_ADD_EXECUTABLE(mariadb-test mysqltest.cc ${CMAKE_SOURCE_DIR}/sql/sql_string.cc COMPONENT Test) SET_SOURCE_FILES_PROPERTIES(mysqltest.cc PROPERTIES COMPILE_FLAGS "-DTHREADS ${PCRE2_DEBIAN_HACK}") diff --git a/data_substrate/CMakeLists.txt b/data_substrate/CMakeLists.txt new file mode 100644 index 00000000000..e6c7dcfc5b7 --- /dev/null +++ b/data_substrate/CMakeLists.txt @@ -0,0 +1,285 @@ +cmake_minimum_required(VERSION 3.8) +project(data_substrate) + +# Set library name +set(DATA_SUBSTRATE_LIB "data_substrate" CACHE STRING "Data substrate library name") +set(METRICS_LIB "eloq-metrics" CACHE STRING "Metrics library name") + +# Options from original eloq CMakeLists +set(WITH_DATA_STORE "ELOQDSS_ROCKSDB" CACHE STRING "Which key-value storage to use") +set_property(CACHE WITH_DATA_STORE PROPERTY STRINGS "DYNAMODB" "BIGTABLE" "ELOQDSS_ROCKSDB_CLOUD_S3" "ELOQDSS_ELOQSTORE" "ELOQDSS_ROCKSDB") + +option(WITH_LOG_SERVICE "Compile with built-in log service." ON) +option(OPEN_LOG_SERVICE "Compile with open log service." ON) +option(SMALL_RANGE "Whether enable small range" OFF) +option(STATISTICS "Whether enable table statistics" ON) +option(EXT_TX_PROC_ENABLED "Allows external threads to move forward the tx service." ON) +option(ELOQ_MODULE_ENABLED "Register the tx service as an ELOQ module." OFF) + + +# TODO: Remove this after testing +add_compile_definitions(ELOQ_MODULE_ELOQSQL) + +message(NOTICE "WITH_DATA_STORE: ${WITH_DATA_STORE}") +message(NOTICE "WITH_LOG_SERVICE: ${WITH_LOG_SERVICE}") +message(NOTICE "OPEN_LOG_SERVICE: ${OPEN_LOG_SERVICE}") + +# Add compile definitions based on options +if(WITH_DATA_STORE STREQUAL "DYNAMODB") + set(KV_STORAGE_VAL 1 CACHE STRING "dynamodb" FORCE) + add_compile_definitions(DATA_STORE_TYPE_DYNAMODB) + find_package(AWSSDK REQUIRED COMPONENTS dynamodb) +elseif(WITH_DATA_STORE STREQUAL "BIGTABLE") + set(KV_STORAGE_VAL 2 CACHE STRING "big table" FORCE) + add_compile_definitions(DATA_STORE_TYPE_BIGTABLE) + find_package(google_cloud_cpp_bigtable REQUIRED) +elseif(WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB_CLOUD_S3") + set(KV_STORAGE_VAL 3 CACHE STRING "eloq_ds" FORCE) + add_compile_definitions(DATA_STORE_TYPE_ELOQDSS_ROCKSDB_CLOUD_S3) +elseif(WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB") + set(KV_STORAGE_VAL 3 CACHE STRING "eloq_ds" FORCE) + add_compile_definitions(DATA_STORE_TYPE_ELOQDSS_ROCKSDB) +elseif(WITH_DATA_STORE STREQUAL "ELOQDSS_ELOQSTORE") + set(KV_STORAGE_VAL 3 CACHE STRING "eloq_ds" FORCE) + add_compile_definitions(DATA_STORE_TYPE_ELOQDSS_ELOQSTORE) + option(WITH_TXSERVICE "Whether compile eloqstore with txservice" ON) + set(ELOQSTORE_PARENT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/store_handler/eloq_data_store_service CACHE PATH "EloqStore parent directory") + INCLUDE(${CMAKE_CURRENT_SOURCE_DIR}/store_handler/eloq_data_store_service/build_eloq_store.cmake) +else() + message(FATAL_ERROR "Unset WITH_DATA_STORE") +endif() + +# Find RocksDB if needed +if(WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB") + find_path(ROCKSDB_INCLUDE_PATH NAMES rocksdb/db.h) + if(NOT ROCKSDB_INCLUDE_PATH) + message(FATAL_ERROR "Fail to find RocksDB include path") + endif() + message(STATUS "ROCKSDB_INCLUDE_PATH: ${ROCKSDB_INCLUDE_PATH}") + include_directories(${ROCKSDB_INCLUDE_PATH}) + + find_library(ROCKSDB_LIB NAMES rocksdb) + if(NOT ROCKSDB_LIB) + message(FATAL_ERROR "Fail to find RocksDB lib path") + endif() + message(STATUS "ROCKSDB_LIB: ${ROCKSDB_LIB}") + set(ROCKSDB_LIBRARIES ${ROCKSDB_LIBRARIES} ${ROCKSDB_LIB}) +endif() + +# Handle RocksDB Cloud S3/GCS configurations +if((WITH_DATA_STORE STREQUAL "ROCKSDB_CLOUD_S3") OR (WITH_DATA_STORE STREQUAL "ROCKSDB_CLOUD_GCS") + OR (WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB_CLOUD_S3") OR (WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB_CLOUD_GCS")) + if((WITH_DATA_STORE STREQUAL "ROCKSDB_CLOUD_S3") OR (WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB_CLOUD_S3")) + find_path(AWS_CORE_INCLUDE_PATH aws/core/Aws.h) + if((NOT AWS_CORE_INCLUDE_PATH)) + message(FATAL_ERROR "Fail to find aws/core include path") + endif() + message(STATUS "aws/core include path: ${AWS_CORE_INCLUDE_PATH}") + + find_library(AWS_CORE_LIB aws-cpp-sdk-core) + if((NOT AWS_CORE_LIB )) + message(FATAL_ERROR "Fail to find aws-cpp-sdk-core lib") + endif() + message(STATUS "aws-cpp-sdk-core library: ${AWS_CORE_LIB}") + + find_path(AWS_S3_INCLUDE_PATH aws/s3/S3Client.h) + if((NOT AWS_S3_INCLUDE_PATH)) + message(FATAL_ERROR "Fail to find aws/s3 include path") + endif() + message(STATUS "aws/s3 include path: ${AWS_S3_INCLUDE_PATH}") + + find_library(AWS_S3_LIB aws-cpp-sdk-s3) + if((NOT AWS_S3_LIB )) + message(FATAL_ERROR "Fail to find aws-cpp-sdk-s3 lib") + endif() + message(STATUS "aws-cpp-sdk-s3 library: ${AWS_S3_LIB}") + + set(ROCKSDB_INCLUDE_PATH ${ROCKSDB_INCLUDE_PATH} ${AWS_CORE_INCLUDE_PATH}) + set(ROCKSDB_INCLUDE_PATH ${ROCKSDB_INCLUDE_PATH} ${AWS_S3_INCLUDE_PATH}) + + set(ROCKSDB_LIBRARIES ${ROCKSDB_LIBRARIES} ${AWS_CORE_LIB}) + set(ROCKSDB_LIBRARIES ${ROCKSDB_LIBRARIES} ${AWS_S3_LIB}) + + find_library(ROCKSDB_CLOUD_LIB NAMES rocksdb-cloud-aws) + endif() + + # Common RocksDB Cloud setup + find_path(ROCKSDB_CLOUD_INCLUDE_PATH NAMES rocksdb/db.h PATH_SUFFIXES "rocksdb_cloud_header") + if(NOT ROCKSDB_CLOUD_INCLUDE_PATH) + message(FATAL_ERROR "Fail to find RocksDB Cloud include path") + endif() + message(STATUS "ROCKSDB_CLOUD_INCLUDE_PATH: ${ROCKSDB_CLOUD_INCLUDE_PATH}") + set(ROCKSDB_INCLUDE_PATH ${ROCKSDB_INCLUDE_PATH} ${ROCKSDB_CLOUD_INCLUDE_PATH}) + include_directories(${ROCKSDB_INCLUDE_PATH}) + + if(NOT ROCKSDB_CLOUD_LIB) + message(FATAL_ERROR "Fail to find RocksDB Cloud lib path") + endif() + message(STATUS "ROCKSDB_CLOUD_LIB: ${ROCKSDB_CLOUD_LIB}") + set(ROCKSDB_LIBRARIES ${ROCKSDB_LIBRARIES} ${ROCKSDB_CLOUD_LIB}) +endif() + +# Compile proto file for data store service +if((WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB_CLOUD_S3") OR (WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB_CLOUD_GCS") OR (WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB") OR (WITH_DATA_STORE STREQUAL "ELOQDSS_ELOQSTORE")) + set(DS_PROTO_DIR ${CMAKE_CURRENT_SOURCE_DIR}/store_handler/eloq_data_store_service) + message(NOTICE "data store service proto dir: ${DS_PROTO_DIR}") + set(PROTO_SRC ${DS_PROTO_DIR}) + set(PROTO_NAME ds_request) + execute_process( + COMMAND protoc ./${PROTO_NAME}.proto --cpp_out=./ + WORKING_DIRECTORY ${PROTO_SRC} + ) +endif() + +# Add compile definitions based on options +if(WITH_LOG_SERVICE) + add_compile_definitions(WITH_LOG_SERVICE) +endif() + +if(OPEN_LOG_SERVICE) + add_compile_definitions(OPEN_LOG_SERVICE=1) +endif() + +if(EXT_TX_PROC_ENABLED) + add_compile_definitions(EXT_TX_PROC_ENABLED) +endif() + +if(ELOQ_MODULE_ENABLED) + add_compile_definitions(ELOQ_MODULE_ENABLED) +endif() + +if(SMALL_RANGE) + add_compile_definitions(SMALL_RANGE) +endif() + +if(STATISTICS) + add_compile_definitions(STATISTICS) +endif() + +add_compile_definitions(ELOQ_MODULE_ELOQSQL) + +# Include build files for subcomponents +INCLUDE(build_tx_service.cmake) +INCLUDE(build_eloq_metrics.cmake) + +if(WITH_LOG_SERVICE) + if(OPEN_LOG_SERVICE) + INCLUDE(build_log_service.cmake) + else() + INCLUDE(build_eloq_log_service.cmake) + endif() +endif() + +# Find mimalloc +find_package(MIMALLOC REQUIRED) +include_directories(${MIMALLOC_INCLUDE_DIR}) + +# Include directories +include_directories( + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_SOURCE_DIR}/core/include + ${CMAKE_CURRENT_SOURCE_DIR}/tx_service/include + ${CMAKE_CURRENT_SOURCE_DIR}/tx_service/include/cc + ${CMAKE_CURRENT_SOURCE_DIR}/tx_service/include/remote + ${CMAKE_CURRENT_SOURCE_DIR}/tx_service/include/fault + ${CMAKE_CURRENT_SOURCE_DIR}/tx_service/include/store + ${CMAKE_CURRENT_SOURCE_DIR}/tx_service/tx-log-protos + ${CMAKE_CURRENT_SOURCE_DIR}/eloq_metrics/include + ${CMAKE_CURRENT_SOURCE_DIR}/store_handler +) + +if(WITH_LOG_SERVICE) + if(OPEN_LOG_SERVICE) + include_directories(${CMAKE_CURRENT_SOURCE_DIR}/log_service/include) + else() + include_directories(${CMAKE_CURRENT_SOURCE_DIR}/eloq_log_service/include) + endif() +endif() + +# Collect all source files for the library +SET(DATA_SUBSTRATE_SOURCES + # Core data substrate files + core/src/data_substrate.cpp + core/src/log_init.cpp + core/src/metrics_init.cpp + core/src/storage_init.cpp + core/src/tx_service_init.cpp +) + +# Add store handler sources based on data store type +if(WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB" OR WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB_CLOUD_S3") + SET(DATA_SUBSTRATE_SOURCES ${DATA_SUBSTRATE_SOURCES} + store_handler/data_store_service_client.cpp + store_handler/data_store_service_client_closure.cpp + store_handler/data_store_service_scanner.cpp + store_handler/store_util.cpp + store_handler/eloq_data_store_service/ini.c + store_handler/eloq_data_store_service/INIReader.cpp + store_handler/eloq_data_store_service/thread_worker_pool.cpp + store_handler/eloq_data_store_service/data_store_service.cpp + store_handler/eloq_data_store_service/data_store_fault_inject.cpp + store_handler/eloq_data_store_service/data_store_service_config.cpp + store_handler/eloq_data_store_service/ds_request.pb.cc + store_handler/eloq_data_store_service/rocksdb_config.cpp + store_handler/eloq_data_store_service/rocksdb_data_store_common.cpp + ) + + if(WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB") + SET(DATA_SUBSTRATE_SOURCES ${DATA_SUBSTRATE_SOURCES} + store_handler/eloq_data_store_service/rocksdb_data_store.cpp + ) + elseif(WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB_CLOUD_S3") + SET(DATA_SUBSTRATE_SOURCES ${DATA_SUBSTRATE_SOURCES} + store_handler/eloq_data_store_service/rocksdb_cloud_data_store.cpp + store_handler/eloq_data_store_service/purger_event_listener.cpp + store_handler/eloq_data_store_service/purger_sliding_window.cpp + ) + endif() +elseif(WITH_DATA_STORE STREQUAL "ELOQDSS_ELOQSTORE") + SET(DATA_SUBSTRATE_SOURCES ${DATA_SUBSTRATE_SOURCES} + store_handler/data_store_service_client.cpp + store_handler/data_store_service_client_closure.cpp + store_handler/data_store_service_scanner.cpp + store_handler/store_util.cpp + store_handler/eloq_data_store_service/thread_worker_pool.cpp + store_handler/eloq_data_store_service/data_store_service.cpp + store_handler/eloq_data_store_service/data_store_fault_inject.cpp + store_handler/eloq_data_store_service/data_store_service_config.cpp + store_handler/eloq_data_store_service/eloq_store_data_store.cpp + store_handler/eloq_data_store_service/ds_request.pb.cc + store_handler/eloq_data_store_service/eloq_store_config.cpp + ) +endif() + +# Build the static library +ADD_LIBRARY(${DATA_SUBSTRATE_LIB} STATIC ${DATA_SUBSTRATE_SOURCES}) + +# Link with dependencies +SET(DATA_SUBSTRATE_LINK_LIBS + txservice + ${METRICS_LIB} +) + +if(WITH_LOG_SERVICE) + SET(DATA_SUBSTRATE_LINK_LIBS ${DATA_SUBSTRATE_LINK_LIBS} logservice) +endif() + +if(WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB" OR WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB_CLOUD_S3") + SET(DATA_SUBSTRATE_LINK_LIBS ${DATA_SUBSTRATE_LINK_LIBS} ${ROCKSDB_LIBRARIES}) +elseif(WITH_DATA_STORE STREQUAL "DYNAMODB") + SET(DATA_SUBSTRATE_LINK_LIBS ${DATA_SUBSTRATE_LINK_LIBS} ${AWSSDK_LINK_LIBRARIES}) +elseif(WITH_DATA_STORE STREQUAL "BIGTABLE") + SET(DATA_SUBSTRATE_LINK_LIBS ${DATA_SUBSTRATE_LINK_LIBS} google-cloud-cpp::bigtable) +elseif(WITH_DATA_STORE STREQUAL "ELOQDSS_ELOQSTORE") + SET(DATA_SUBSTRATE_LINK_LIBS ${DATA_SUBSTRATE_LINK_LIBS} eloqstore) +endif() + +TARGET_LINK_LIBRARIES(${DATA_SUBSTRATE_LIB} ${DATA_SUBSTRATE_LINK_LIBS}) + +# Export include directories for other targets +TARGET_INCLUDE_DIRECTORIES(${DATA_SUBSTRATE_LIB} PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/core/include + ${CMAKE_CURRENT_SOURCE_DIR}/tx_service/include + ${CMAKE_CURRENT_SOURCE_DIR}/eloq_metrics/include + ${CMAKE_CURRENT_SOURCE_DIR}/store_handler +) diff --git a/storage/eloq/build_eloq_log_service.cmake b/data_substrate/build_eloq_log_service.cmake similarity index 99% rename from storage/eloq/build_eloq_log_service.cmake rename to data_substrate/build_eloq_log_service.cmake index 9aa66165f5e..5412da2d33e 100644 --- a/storage/eloq/build_eloq_log_service.cmake +++ b/data_substrate/build_eloq_log_service.cmake @@ -14,6 +14,7 @@ if(WITH_LOG_STATE STREQUAL "MEMORY") add_compile_definitions(LOG_STATE_TYPE_MEM) elseif(WITH_LOG_STATE STREQUAL "ROCKSDB") add_compile_definitions(LOG_STATE_TYPE_RKDB) + add_compile_definitions(LOG_STATE_TYPE_RKDB_ALL) elseif(WITH_LOG_STATE STREQUAL "ROCKSDB_CLOUD_S3") add_compile_definitions(LOG_STATE_TYPE_RKDB_S3) elseif(WITH_LOG_STATE STREQUAL "ROCKSDB_CLOUD_GCS") @@ -265,7 +266,7 @@ set(LOG_LIB find_package(Protobuf REQUIRED) -ADD_CONVENIENCE_LIBRARY(logservice STATIC +ADD_LIBRARY(logservice STATIC ${LOG_SOURCE_DIR}/src/log_instance.cpp ${LOG_SOURCE_DIR}/src/log_server.cpp ${LOG_SOURCE_DIR}/src/log_state_rocksdb_impl.cpp diff --git a/storage/eloq/build_eloq_metrics.cmake b/data_substrate/build_eloq_metrics.cmake similarity index 93% rename from storage/eloq/build_eloq_metrics.cmake rename to data_substrate/build_eloq_metrics.cmake index 9fe05735256..18df1d3c0b9 100644 --- a/storage/eloq/build_eloq_metrics.cmake +++ b/data_substrate/build_eloq_metrics.cmake @@ -31,9 +31,11 @@ set(ELOQ_METRICS_TARGET_SOURCE_LIST ${ELOQ_METRICS_INCLUDE_DIR}/metrics_collector.h ${ELOQ_METRICS_INCLUDE_DIR}/prometheus_collector.h ${ELOQ_METRICS_INCLUDE_DIR}/metrics_manager.h + ${ELOQ_METRICS_INCLUDE_DIR}/metrics_registry_impl.h ${ELOQ_METRICS_SRC_DIR}/metrics.cc ${ELOQ_METRICS_SRC_DIR}/prometheus_collector.cc ${ELOQ_METRICS_SRC_DIR}/metrics_manager.cc + ${ELOQ_METRICS_SRC_DIR}/metrics_registry_impl.cpp ) add_library(${METRICS_LIB} STATIC ${ELOQ_METRICS_TARGET_SOURCE_LIST}) diff --git a/storage/eloq/build_log_service.cmake b/data_substrate/build_log_service.cmake similarity index 99% rename from storage/eloq/build_log_service.cmake rename to data_substrate/build_log_service.cmake index 86a86c4f740..21cc010b861 100644 --- a/storage/eloq/build_log_service.cmake +++ b/data_substrate/build_log_service.cmake @@ -143,7 +143,7 @@ set(LOG_LIB find_package(Protobuf REQUIRED) -ADD_CONVENIENCE_LIBRARY(logservice STATIC +ADD_LIBRARY(logservice STATIC ${LOG_SOURCE_DIR}/src/log_server.cpp ${LOG_SOURCE_DIR}/src/log_state_rocksdb_impl.cpp ${LOG_SOURCE_DIR}/src/fault_inject.cpp diff --git a/storage/eloq/build_tx_service.cmake b/data_substrate/build_tx_service.cmake similarity index 64% rename from storage/eloq/build_tx_service.cmake rename to data_substrate/build_tx_service.cmake index a4fd72b3fe2..be16ca79f24 100644 --- a/storage/eloq/build_tx_service.cmake +++ b/data_substrate/build_tx_service.cmake @@ -196,65 +196,119 @@ SET(ELOQ_SOURCES set(INCLUDE_DIR ${INCLUDE_DIR} ${PROTO_SRC}) set(ELOQ_SOURCES ${ELOQ_SOURCES} ${PROTO_CC_FILES}) -ADD_CONVENIENCE_LIBRARY(txservice STATIC +ADD_LIBRARY(txservice STATIC ${ELOQ_SOURCES}) target_include_directories(txservice PUBLIC ${INCLUDE_DIR}) target_link_libraries(txservice PUBLIC ${LINK_LIB} ${PROTOBUF_LIBRARIES}) -if(FORK_HM_PROCESS) - include(FetchContent) - # Import yaml-cpp library used by host manager - FetchContent_Declare( - yaml-cpp - GIT_REPOSITORY https://github.com/jbeder/yaml-cpp.git - GIT_TAG yaml-cpp-0.7.0 # Can be a tag (yaml-cpp-x.x.x), a commit hash, or a branch name (master) - ) - FetchContent_MakeAvailable(yaml-cpp) - - SET(HOST_MANAGER_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/tx_service/raft_host_manager) +# if(FORK_HM_PROCESS) +# include(FetchContent) +# # Import yaml-cpp library used by host manager +# FetchContent_Declare( +# yaml-cpp +# GIT_REPOSITORY https://github.com/jbeder/yaml-cpp.git +# GIT_TAG yaml-cpp-0.7.0 # Can be a tag (yaml-cpp-x.x.x), a commit hash, or a branch name (master) +# ) +# FetchContent_MakeAvailable(yaml-cpp) + +# SET(HOST_MANAGER_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/tx_service/raft_host_manager) +# set(HOST_MANAGER_INCLUDE_DIR +# ${HOST_MANAGER_SOURCE_DIR}/include +# ${TX_SERVICE_SOURCE_DIR}/tx-log-protos +# ${OPENSSL_INCLUDE_DIR} +# ${LOG_PROTO_SRC} +# ${PROTO_SRC}) + +# if(CMAKE_COMPILER_IS_GNUCC) +# set(HOST_MANAGER_INCLUDE_DIR ${HOST_MANAGER_INCLUDE_DIR} +# ${BRPC_INCLUDE_PATH} +# ${BRAFT_INCLUDE_PATH} +# ${GLOG_INCLUDE_PATH} +# ${GFLAGS_INCLUDE_PATH}) +# endif() + +# set(HOST_MANAGER_LINK_LIB ${HOST_MANAGER_LINK_LIB} ${PROTOBUF_LIBRARIES}) + +# if(CMAKE_COMPILER_IS_GNUCC) +# find_path(BRAFT_INCLUDE_PATH NAMES braft/raft.h) +# find_library(BRAFT_LIB NAMES braft) + +# if((NOT BRAFT_INCLUDE_PATH) OR(NOT BRAFT_LIB)) +# message(FATAL_ERROR "Fail to find braft") +# endif() + +# set(HOST_MANAGER_LINK_LIB ${HOST_MANAGER_LINK_LIB} +# ${GFLAGS_LIBRARY} +# ${LEVELDB_LIB} +# ${BRAFT_LIB} +# ${BRPC_LIB} +# ${OPENSSL_LIB}) +# find_path(GLOG_INCLUDE_PATH NAMES glog/logging.h) +# find_library(GLOG_LIB NAMES glog VERSION ">=0.6.0" REQUIRED) + +# if((NOT GLOG_INCLUDE_PATH) OR(NOT GLOG_LIB)) +# message(FATAL_ERROR "Fail to find glog") +# endif() + +# include_directories(${GLOG_INCLUDE_PATH}) +# set(HOST_MANAGER_LINK_LIB ${HOST_MANAGER_LINK_LIB} ${GLOG_LIB}) +# endif() + +# SET(RaftHM_SOURCES +# ${HOST_MANAGER_SOURCE_DIR}/src/main.cpp +# ${HOST_MANAGER_SOURCE_DIR}/src/raft_host_manager_service.cpp +# ${HOST_MANAGER_SOURCE_DIR}/src/raft_host_manager.cpp +# ${HOST_MANAGER_SOURCE_DIR}/src/ini.c +# ${HOST_MANAGER_SOURCE_DIR}/src/INIReader.cpp +# ${LOG_PROTO_SRC}/log_agent.cpp +# ${LOG_PROTO_SRC}/${LOG_PROTO_NAME}.pb.cc +# ${PROTO_CC_FILES} +# ) + +# include_directories(${HOST_MANAGER_INCLUDE_DIR}) +# MYSQL_ADD_EXECUTABLE(host_manager ${RaftHM_SOURCES} DESTINATION ${INSTALL_SBINDIR} COMPONENT Server) +# target_link_libraries(host_manager ${HOST_MANAGER_LINK_LIB} yaml-cpp::yaml-cpp) + +# endif() + +if (FORK_HM_PROCESS) + SET (HOST_MANAGER_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/tx_service/raft_host_manager) set(HOST_MANAGER_INCLUDE_DIR ${HOST_MANAGER_SOURCE_DIR}/include ${TX_SERVICE_SOURCE_DIR}/tx-log-protos ${OPENSSL_INCLUDE_DIR} - ${LOG_PROTO_SRC} - ${PROTO_SRC}) - - if(CMAKE_COMPILER_IS_GNUCC) - set(HOST_MANAGER_INCLUDE_DIR ${HOST_MANAGER_INCLUDE_DIR} - ${BRPC_INCLUDE_PATH} - ${BRAFT_INCLUDE_PATH} - ${GLOG_INCLUDE_PATH} - ${GFLAGS_INCLUDE_PATH}) - endif() + ${PROTO_SRC} + ${LOG_PROTO_SRC}) - set(HOST_MANAGER_LINK_LIB ${HOST_MANAGER_LINK_LIB} ${PROTOBUF_LIBRARIES}) - - if(CMAKE_COMPILER_IS_GNUCC) - find_path(BRAFT_INCLUDE_PATH NAMES braft/raft.h) - find_library(BRAFT_LIB NAMES braft) - - if((NOT BRAFT_INCLUDE_PATH) OR(NOT BRAFT_LIB)) - message(FATAL_ERROR "Fail to find braft") - endif() - - set(HOST_MANAGER_LINK_LIB ${HOST_MANAGER_LINK_LIB} - ${GFLAGS_LIBRARY} - ${LEVELDB_LIB} - ${BRAFT_LIB} - ${BRPC_LIB} - ${OPENSSL_LIB}) - find_path(GLOG_INCLUDE_PATH NAMES glog/logging.h) - find_library(GLOG_LIB NAMES glog VERSION ">=0.6.0" REQUIRED) + set(HOST_MANAGER_INCLUDE_DIR ${HOST_MANAGER_INCLUDE_DIR} + ${BRPC_INCLUDE_PATH} + ${BRAFT_INCLUDE_PATH} + ${GLOG_INCLUDE_PATH} + ${GFLAGS_INCLUDE_PATH}) - if((NOT GLOG_INCLUDE_PATH) OR(NOT GLOG_LIB)) - message(FATAL_ERROR "Fail to find glog") - endif() + set(HOST_MANAGER_LINK_LIB ${HOST_MANAGER_LINK_LIB} ${PROTOBUF_LIBRARIES}) - include_directories(${GLOG_INCLUDE_PATH}) - set(HOST_MANAGER_LINK_LIB ${HOST_MANAGER_LINK_LIB} ${GLOG_LIB}) + find_path(BRAFT_INCLUDE_PATH NAMES braft/raft.h) + find_library(BRAFT_LIB NAMES braft) + if ((NOT BRAFT_INCLUDE_PATH) OR (NOT BRAFT_LIB)) + message (FATAL_ERROR "Fail to find braft") + endif() + set(HOST_MANAGER_LINK_LIB ${HOST_MANAGER_LINK_LIB} + ${GFLAGS_LIBRARY} + ${GPERFTOOLS_LIBRARIES} + ${LEVELDB_LIB} + ${BRAFT_LIB} + ${BRPC_LIB} + ${OPENSSL_LIB}) + find_path(GLOG_INCLUDE_PATH NAMES glog/logging.h) + find_library(GLOG_LIB NAMES glog VERSION ">=0.6.0" REQUIRED) + if((NOT GLOG_INCLUDE_PATH) OR (NOT GLOG_LIB)) + message(FATAL_ERROR "Fail to find glog") endif() + include_directories(${GLOG_INCLUDE_PATH}) + set(HOST_MANAGER_LINK_LIB ${HOST_MANAGER_LINK_LIB} ${GLOG_LIB}) SET(RaftHM_SOURCES ${HOST_MANAGER_SOURCE_DIR}/src/main.cpp @@ -262,12 +316,31 @@ if(FORK_HM_PROCESS) ${HOST_MANAGER_SOURCE_DIR}/src/raft_host_manager.cpp ${HOST_MANAGER_SOURCE_DIR}/src/ini.c ${HOST_MANAGER_SOURCE_DIR}/src/INIReader.cpp + ${PROTO_SRC}/${PROTO_NAME}.pb.cc ${LOG_PROTO_SRC}/log_agent.cpp ${LOG_PROTO_SRC}/${LOG_PROTO_NAME}.pb.cc - ${PROTO_CC_FILES} + ) + + include(FetchContent) + + # Import yaml-cpp library used by host manager + FetchContent_Declare( + yaml-cpp + GIT_REPOSITORY https://github.com/jbeder/yaml-cpp.git + GIT_TAG yaml-cpp-0.7.0 # Can be a tag (yaml-cpp-x.x.x), a commit hash, or a branch name (master) ) + FetchContent_MakeAvailable(yaml-cpp) + set(HOST_MANAGER_LINK_LIB ${HOST_MANAGER_LINK_LIB} yaml-cpp::yaml-cpp) include_directories(${HOST_MANAGER_INCLUDE_DIR}) - MYSQL_ADD_EXECUTABLE(host_manager ${RaftHM_SOURCES} DESTINATION ${INSTALL_SBINDIR} COMPONENT Server) - target_link_libraries(host_manager ${HOST_MANAGER_LINK_LIB} yaml-cpp::yaml-cpp) + add_executable(host_manager ${RaftHM_SOURCES}) + target_link_libraries(host_manager PRIVATE ${HOST_MANAGER_LINK_LIB} ) + + set_target_properties(host_manager PROPERTIES + BUILD_RPATH "$ORIGIN/../lib" + INSTALL_RPATH "$ORIGIN/../lib" + INSTALL_RPATH_USE_LINK_PATH TRUE) + + install(TARGETS host_manager RUNTIME DESTINATION bin) endif() + diff --git a/data_substrate/core/CONFIGURATION.md b/data_substrate/core/CONFIGURATION.md new file mode 100644 index 00000000000..dfb5cc6a0d3 --- /dev/null +++ b/data_substrate/core/CONFIGURATION.md @@ -0,0 +1,140 @@ +# Data Substrate Configuration System + +## Overview + +The Data Substrate supports configuration through both INI configuration files and command line flags (gflags). Command line flags take precedence over configuration file values, allowing for flexible deployment and testing scenarios. + +## Configuration Sources (Priority Order) + +1. **Command Line Flags** (highest priority) +2. **Configuration File** (medium priority) +3. **Default Values** (lowest priority) + +## Configuration File Format + +The configuration file uses INI format with the following sections: + +### [data_substrate] +Core data substrate configuration parameters. + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `eloq_data_path` | string | "eloq_data" | Path for cc_ng and tx_log | +| `core_num` | int | 8 | Number of cores for transaction service | +| `checkpointer_interval` | int | 10 | Interval time(seconds) of checkpoint | +| `node_memory_limit_mb` | int | 8192 | txservice node_memory_limit_mb | +| `checkpointer_delay_seconds` | int | 5 | Checkpointer delay in seconds | +| `collect_active_tx_ts_interval_seconds` | int | 2 | Collect active tx timestamp interval | +| `realtime_sampling` | bool | true | Enable realtime sampling | +| `txlog_service_list` | string | "" | Log group servers configuration | +| `enable_tx_metrics` | bool | true | Enable transaction metrics | +| `enable_log_service_metrics` | bool | true | Enable log service metrics | +| `enable_key_cache` | bool | true | Enable key cache | +| `enable_shard_heap_defragment` | bool | false | Enable shard heap defragmentation | +| `bthread_worker_num` | int | 0 | Number of bthread workers | +| `max_standby_lag` | int | 0 | Maximum standby lag | +| `kickout_data_for_test` | bool | false | Clean data for test | + +### [storage] +Storage handler configuration parameters. + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `storage_type` | string | "rocksdb" | Storage type (rocksdb, dynamodb, data_store_service) | +| `rocksdb_storage_path` | string | "" | RocksDB storage path | +| `enable_cache_replacement` | bool | true | Enable cache replacement | +| `rocksdb_max_write_buffer_number` | int | 3 | RocksDB max write buffer number | +| `rocksdb_max_background_jobs` | int | 4 | RocksDB max background jobs | +| `rocksdb_target_file_size_base` | string | "64MB" | RocksDB target file size base | + +### [dynamodb] +DynamoDB-specific configuration (when storage_type = "dynamodb"). + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `dynamodb_endpoint` | string | "" | DynamoDB endpoint | +| `dynamodb_region` | string | "ap-northeast-1" | DynamoDB region | +| `dynamodb_keyspace` | string | "eloq_data" | DynamoDB keyspace | +| `aws_access_key_id` | string | "" | AWS access key ID | +| `aws_secret_key` | string | "" | AWS secret key | + +### [log_service] +Log service configuration parameters. + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `log_service_data_path` | string | "" | Log service data path | +| `rocksdb_scan_threads` | int | 4 | RocksDB scan threads | +| `txlog_group_replica_num` | int | 3 | Replica number of one log group | +| `logserver_snapshot_interval` | int | 3600 | Log server snapshot interval | +| `enable_txlog_request_checkpoint` | bool | true | Enable txlog request checkpoint | +| `notify_checkpointer_threshold_size` | string | "1GB" | Notify checkpointer threshold size | + +### [metrics] +Metrics configuration parameters. + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `enable_metrics` | bool | true | Enable metrics | +| `metrics_port` | int | 9090 | Metrics port | + +## Command Line Flags + +All configuration parameters can be overridden using command line flags. The flag names match the parameter names in the configuration file. + +### Example Usage + +```bash +# Use configuration file only +./data_substrate --config_file=config/sample_data_substrate.ini + +# Override specific parameters via command line +./data_substrate --config_file=config/sample_data_substrate.ini \ + --core_num=16 \ + --node_memory_limit_mb=16384 \ + --storage_type=dynamodb + +# Use command line flags only (no config file) +./data_substrate \ + --eloq_data_path=/custom/path \ + --core_num=8 \ + --storage_type=rocksdb \ + --rocksdb_storage_path=/custom/rocksdb/path +``` + +## Configuration Loading Process + +1. **Parse Command Line Flags**: gflags parses command line arguments +2. **Load Configuration File**: INI file is parsed if provided +3. **Apply Overrides**: Command line flags override config file values +4. **Use Defaults**: Any unspecified parameters use default values + +## Helper Functions + +The configuration system uses helper functions to handle the priority logic: + +- `GetConfigValue()`: Gets string values with gflags override +- `GetConfigBool()`: Gets boolean values with gflags override +- `GetConfigInt()`: Gets integer values with gflags override + +## Sample Configuration File + +See `config/sample_data_substrate.ini` for a complete example configuration file with all available parameters. + +## Integration with Engines + +The configuration system is designed to work with multiple database engines: + +- **eloqsql**: Uses MariaDB-specific catalog factory and system handler +- **eloqkv**: Uses eloqkv-specific catalog factory +- **eloqdoc**: Uses eloqdoc-specific catalog factory and system handler + +Engine-specific configurations are handled through compile-time macros (`#ifdef ELOQSQL`, `#ifdef ELOQKV`, `#ifdef ELOQDOC`). + +## Best Practices + +1. **Use Configuration Files**: For production deployments, use configuration files for consistency +2. **Command Line Overrides**: Use command line flags for testing and development +3. **Default Values**: Rely on sensible defaults for optional parameters +4. **Validation**: Always validate configuration values before use +5. **Documentation**: Keep configuration documentation up to date with code changes diff --git a/data_substrate/core/CONFIG_MAPPING.md b/data_substrate/core/CONFIG_MAPPING.md new file mode 100644 index 00000000000..cd50072a022 --- /dev/null +++ b/data_substrate/core/CONFIG_MAPPING.md @@ -0,0 +1,209 @@ +# Configuration Parameter Mapping: eloqsql, eloqkv, and Data Substrate + +## Overview + +This document provides a comprehensive mapping of all configuration parameters across the three systems: +- **eloqsql**: MySQL system variables (MYSQL_SYSVAR) +- **eloqkv**: gflags (DEFINE_*) +- **Data Substrate**: gflags (DEFINE_*) + +## Configuration Categories + +### 1. Core Data Substrate Parameters + +| Parameter | eloqsql (MYSQL_SYSVAR) | eloqkv (DEFINE_*) | Data Substrate (DEFINE_*) | Description | +|-----------|------------------------|-------------------|---------------------------|-------------| +| **Data Path** | `eloq_data_path` | `eloq_data_path` | `eloq_data_path` | Path for cc_ng and tx_log | +| **Core Number** | `eloq_core_num` | `core_number` | `core_num` | Number of cores for transaction service | +| **Checkpointer Interval** | `eloq_checkpointer_interval_sec` | `checkpoint_interval` | `checkpointer_interval` | Interval time(seconds) of checkpoint | +| **Node Memory Limit** | `eloq_node_memory_limit_mb` | `node_memory_limit_mb` | `node_memory_limit_mb` | txservice node_memory_limit_mb | +| **Checkpointer Delay** | `eloq_checkpointer_delay_sec` | - | `checkpointer_delay_seconds` | Checkpointer delay in seconds | +| **Active TX TS Interval** | `eloq_collect_active_tx_ts_interval_sec` | - | `collect_active_tx_ts_interval_seconds` | Collect active tx timestamp interval | +| **Max Standby Lag** | - | `max_standby_lag` | `max_standby_lag` | Maximum standby lag | +| **Kickout Data for Test** | `eloq_kickout_data_for_test` | `kickout_data_for_test` | `kickout_data_for_test` | Clean data for test | +| **Use Key Cache** | `eloq_use_key_cache` | - | `enable_key_cache` | Enable key cache | +| **Enable Heap Defragment** | `eloq_enable_heap_defragment` | `enable_heap_defragment` | `enable_shard_heap_defragment` | Enable heap defragmentation | +| **Enable data WAL Log** | `enable_wal` | Enable WAL | +| **Enbale Data Store** | `enable_data_store` | Enable data storage | + +### 2. Network and Cluster Configuration + +| Parameter | eloqsql (MYSQL_SYSVAR) | eloqkv (DEFINE_*) | Data Substrate (DEFINE_*) | Description | +|-----------|------------------------|-------------------|---------------------------|-------------| +| **Local IP** | `eloq_local_ip` | `ip` | - | Local IP address | +| **IP List** | `eloq_ip_list` | `ip_port_list` | - | Server cluster ip port list | +| **Standby IP List** | `eloq_standby_ip_list` | `standby_ip_port_list` | - | Standby nodes ip:port list | +| **Voter IP List** | `eloq_voter_ip_list` | `voter_ip_port_list` | - | Voter nodes ip:port list | +| **Host Manager IP** | `eloq_hm_ip` | `hm_ip` | - | Host manager ip address | +| **Host Manager Port** | - | `hm_port` | - | Host manager port | +| **Host Manager Bin** | `eloq_hm_bin_path` | `hm_bin` | - | Host manager binary path | +| **Cluster Config File** | `eloq_cluster_config_file` | `cluster_config_file` | - | Path for cluster config file | +| **Node Group Replica Num** | `eloq_node_group_replica_num` | `tx_nodegroup_replica_num` | - | Replica number of one txservice node group | + +### 3. Log Service Configuration + +| Parameter | eloqsql (MYSQL_SYSVAR) | eloqkv (DEFINE_*) | Data Substrate (DEFINE_*) | Description | +|-----------|------------------------|-------------------|---------------------------|-------------| +| **TxLog Service List** | `eloq_txlog_service_list` | `txlog_service_list` | `txlog_service_list` | Log group servers configuration | +| **TxLog Group Replica Num** | `eloq_txlog_group_replica_num` | `txlog_group_replica_num` | `txlog_group_replica_num` | Replica number of one log group | +| **TxLog RocksDB Storage Path** | `eloq_txlog_rocksdb_storage_path` | `txlog_rocksdb_storage_path` | - | Storage path for txlog rocksdb state | +| **Log Service Data Path** | - | `log_service_data_path` | `log_service_data_path` | Log service data path | +| **RocksDB Scan Threads** | `eloq_logserver_rocksdb_scan_thread_num` | `txlog_rocksdb_scan_threads` | `rocksdb_scan_threads` | RocksDB scan threads | +| **Log Server Snapshot Interval** | `eloq_logserver_snapshot_interval` | `logserver_snapshot_interval` | `logserver_snapshot_interval` | Log server snapshot interval | +| **Enable TxLog Request Checkpoint** | `eloq_enable_txlog_request_checkpoint` | `enable_txlog_request_checkpoint` | `enable_txlog_request_checkpoint` | Enable txlog request checkpoint | +| **Notify Checkpointer Threshold** | `eloq_notify_checkpointer_threshold_size` | `notify_checkpointer_threshold_size` | `notify_checkpointer_threshold_size` | Notify checkpointer threshold size | +| **Check Replay Log Size Interval** | `eloq_check_replay_log_size_interval_sec` | `check_replay_log_size_interval_sec` | - | Check replay log size interval | +| **TxLog RocksDB Max Write Buffer Number** | `eloq_txlog_rocksdb_max_write_buffer_number` | `txlog_rocksdb_max_write_buffer_number` | `txlog_rocksdb_max_write_buffer_number` | RocksDB max write buffer number for txlog | +| **TxLog RocksDB Max Background Jobs** | `eloq_txlog_rocksdb_max_background_jobs` | `txlog_rocksdb_max_background_jobs` | `txlog_rocksdb_max_background_jobs` | RocksDB max background jobs for txlog | +| **TxLog RocksDB Target File Size Base** | `eloq_txlog_rocksdb_target_file_size_base` | `txlog_rocksdb_target_file_size_base` | `txlog_rocksdb_target_file_size_base` | RocksDB target file size base for txlog | +| **TxLog RocksDB SST Files Size Limit** | `eloq_txlog_rocksdb_sst_files_size_limit` | `txlog_rocksdb_sst_files_size_limit` | - | RocksDB SST files size limit for txlog | +| **TxLog RocksDB Cloud Region** | `eloq_txlog_rocksdb_cloud_region` | `txlog_rocksdb_cloud_region` | `txlog_rocksdb_cloud_region` | Cloud service region for txlog | +| **TxLog RocksDB Cloud Bucket Name** | `eloq_txlog_rocksdb_cloud_bucket_name` | `txlog_rocksdb_cloud_bucket_name` | `txlog_rocksdb_cloud_bucket_name` | Cloud storage bucket name for txlog | +| **TxLog RocksDB Cloud Bucket Prefix** | `eloq_txlog_rocksdb_cloud_bucket_prefix` | `txlog_rocksdb_cloud_bucket_prefix` | `txlog_rocksdb_cloud_bucket_prefix` | Cloud storage bucket prefix for txlog | +| **TxLog RocksDB Cloud Object Path** | `eloq_txlog_rocksdb_cloud_object_path` | `txlog_rocksdb_cloud_object_path` | `txlog_rocksdb_cloud_object_path` | Cloud object path for txlog | +| **TxLog RocksDB Cloud Endpoint URL** | `eloq_txlog_rocksdb_cloud_endpoint_url` | `txlog_rocksdb_cloud_endpoint_url` | `txlog_rocksdb_cloud_endpoint_url` | Cloud storage endpoint URL for txlog | +| **TxLog RocksDB Cloud SST File Cache Size** | `eloq_txlog_rocksdb_cloud_sst_file_cache_size` | `txlog_rocksdb_cloud_sst_file_cache_size` | `txlog_rocksdb_cloud_sst_file_cache_size` | Cloud SST file cache size for txlog | +| **TxLog RocksDB Cloud Ready Timeout** | `eloq_txlog_rocksdb_cloud_ready_timeout` | `txlog_rocksdb_cloud_ready_timeout` | `txlog_rocksdb_cloud_ready_timeout` | Cloud ready timeout for txlog | +| **TxLog RocksDB Cloud File Deletion Delay** | `eloq_txlog_rocksdb_cloud_file_deletion_delay` | `txlog_rocksdb_cloud_file_deletion_delay` | `txlog_rocksdb_cloud_file_deletion_delay` | Cloud file deletion delay for txlog | + +### 4. Storage Configuration + +| Parameter | eloqsql (MYSQL_SYSVAR) | eloqkv (DEFINE_*) | Data Substrate (DEFINE_*) | Description | +|-----------|------------------------|-------------------|---------------------------|-------------| +| **Enable Cache Replacement** | - | `enable_cache_replacement` | `enable_cache_replacement` | Enable cache replacement | +| **RocksDB Storage Path** | - | - | `rocksdb_storage_path` | RocksDB storage path | + +#### 4.1. DynamoDB Configuration + +| Parameter | eloqsql (MYSQL_SYSVAR) | eloqkv (DEFINE_*) | Data Substrate (DEFINE_*) | Description | +|-----------|------------------------|-------------------|---------------------------|-------------| +| **DynamoDB Endpoint** | `eloq_dynamodb_endpoint` | `dynamodb_endpoint` | `dynamodb_endpoint` | DynamoDB endpoint | +| **DynamoDB Region** | `eloq_dynamodb_region` | `dynamodb_region` | `dynamodb_region` | DynamoDB region | +| **DynamoDB Keyspace** | - | `dynamodb_keyspace` | `dynamodb_keyspace` | DynamoDB keyspace | +| **AWS Access Key ID** | `eloq_aws_access_key_id` | `aws_access_key_id` | `aws_access_key_id` | AWS access key ID | +| **AWS Secret Key** | `eloq_aws_secret_key` | `aws_secret_key` | `aws_secret_key` | AWS secret key | + +#### 4.2. BigTable Configuration (eloqsql only) + +| Parameter | eloqsql (MYSQL_SYSVAR) | Description | +|-----------|------------------------|-------------| +| **BigTable Project ID** | `eloq_bigtable_project_id` | BigTable project ID | +| **BigTable Instance ID** | `eloq_bigtable_instance_id` | BigTable instance ID | + +#### 4.3. Data Store Service (ELOQDS) +| Parameter | eloqsql (MYSQL_SYSVAR) | eloqkv (DEFINE_*) | Description | +|-----------|------------------------|-------------------|-------------| +| **DSS Config File Path** | `eloq_dss_config_file_path` | - | DSS config file path | +| **DSS Peer Node** | `eloq_dss_peer_node` | `eloq_dss_peer_node` | DSS peer node | +| **DSS Branch Name** | `eloq_dss_branch_name` | `eloq_dss_branch_name` | DSS branch name | +| **DSS RocksDB Cloud Config** | Multiple `eloq_dss_rocksdb_cloud_*` | - | DSS RocksDB cloud configuration | + +#### 4.4. EloqStore (DATA_STORE_TYPE_ELOQDSS_ELOQSTORE) +| Parameter | eloqsql (MYSQL_SYSVAR) | Description | +|-----------|------------------------|-------------| +| **EloqStore Worker Num** | `eloq_eloqstore_worker_num` | EloqStore worker number | +| **EloqStore Data Path List** | `eloq_eloqstore_data_path_list` | EloqStore data path list | +| **EloqStore Open Files Limit** | `eloq_eloqstore_open_files_limit` | EloqStore open files limit | +| **EloqStore Cloud Store Path** | `eloq_eloqstore_cloud_store_path` | EloqStore cloud store path | +| **EloqStore GC Threads** | `eloq_eloqstore_gc_threads` | EloqStore GC threads | +| **EloqStore Cloud Worker Count** | `eloq_eloqstore_cloud_worker_count` | EloqStore cloud worker count | +| **EloqStore Data Page Restart Interval** | `eloq_eloqstore_data_page_restart_interval` | EloqStore data page restart interval | +| **EloqStore Index Page Restart Interval** | `eloq_eloqstore_index_page_restart_interval` | EloqStore index page restart interval | +| **EloqStore Init Page Count** | `eloq_eloqstore_init_page_count` | EloqStore init page count | +| **EloqStore Skip Verify Checksum** | `eloq_eloqstore_skip_verify_checksum` | EloqStore skip verify checksum | +| **EloqStore Index Buffer Pool Size** | `eloq_eloqstore_index_buffer_pool_size` | EloqStore index buffer pool size | +| **EloqStore Manifest Limit** | `eloq_eloqstore_manifest_limit` | EloqStore manifest limit | +| **EloqStore IO Queue Size** | `eloq_eloqstore_io_queue_size` | EloqStore IO queue size | +| **EloqStore Max Inflight Write** | `eloq_eloqstore_max_inflight_write` | EloqStore max inflight write | +| **EloqStore Max Write Batch Pages** | `eloq_eloqstore_max_write_batch_pages` | EloqStore max write batch pages | +| **EloqStore Buf Ring Size** | `eloq_eloqstore_buf_ring_size` | EloqStore buf ring size | +| **EloqStore Coroutine Stack Size** | `eloq_eloqstore_coroutine_stack_size` | EloqStore coroutine stack size | +| **EloqStore Num Retained Archives** | `eloq_eloqstore_num_retained_archives` | EloqStore num retained archives | +| **EloqStore Archive Interval** | `eloq_eloqstore_archive_interval_secs` | EloqStore archive interval | +| **EloqStore Max Archive Tasks** | `eloq_eloqstore_max_archive_tasks` | EloqStore max archive tasks | +| **EloqStore File Amplify Factor** | `eloq_eloqstore_file_amplify_factor` | EloqStore file amplify factor | +| **EloqStore Local Space Limit** | `eloq_eloqstore_local_space_limit` | EloqStore local space limit | +| **EloqStore Reserve Space Ratio** | `eloq_eloqstore_reserve_space_ratio` | EloqStore reserve space ratio | +| **EloqStore Data Page Size** | `eloq_eloqstore_data_page_size` | EloqStore data page size | +| **EloqStore Pages Per File Shift** | `eloq_eloqstore_pages_per_file_shift` | EloqStore pages per file shift | +| **EloqStore Overflow Pointers** | `eloq_eloqstore_overflow_pointers` | EloqStore overflow pointers | +| **EloqStore Data Append Mode** | `eloq_e + +### 5. Metrics Configuration + +| Parameter | eloqsql (MYSQL_SYSVAR) | eloqkv (DEFINE_*) | Data Substrate (DEFINE_*) | Description | +|-----------|------------------------|-------------------|---------------------------|-------------| +| **Enable Metrics** | `eloq_enable_metrics` | - | `enable_metrics` | Enable metrics | +| **Metrics Port** | `eloq_metrics_port` | - | `metrics_port` | Metrics port | +| **Enable TX Metrics** | `eloq_enable_tx_metrics` | - | `enable_tx_metrics` | Enable transaction metrics | +| **Enable Log Service Metrics** | `eloq_enable_log_service_metrics` | - | `enable_log_service_metrics` | Enable log service metrics | +| **Enable Cache Hit Rate** | `eloq_enable_cache_hit_rate` | - | `enable_cache_hit_rate` | Enable cache hit rate metrics | +| **Enable KV Metrics** | `eloq_enable_kv_metrics` | - | `enable_kv_metrics` | Enable KV store metrics | +| **Enable Remote Request Metrics** | `eloq_enable_remote_request_metrics` | - | `enable_remote_request_metrics` | Enable remote request metrics | +| **Collect TX Duration Round** | `eloq_collect_tx_duration_round` | - | `collect_tx_duration_round` | Collect transaction duration round | +| **Busy Round Threshold** | `eloq_busy_round_threshold` | - | `busy_round_threshold` | Busy round threshold | +| **Collect Memory Usage Round** | `eloq_collect_memory_usage_round` | - | - | Collect memory usage round | +| **Enable Busy Round Metrics** | `eloq_enable_busy_round_metrics` | - | - | Enable busy round metrics | +| **Enable Memory Usage** | `eloq_enable_memory_usage` | - | - | Enable memory usage metrics | +| **Realtime Sampling** | `eloq_realtime_sampling` | - | `realtime_sampling` | Enable realtime sampling | + +### 6. Engine-Specific Configuration + +#### eloqsql-specific (MySQL system variables) +| Parameter | Description | +|-----------|-------------| +| `eloq_kv_storage` | KV storage type | +| `eloq_insert_semantic` | Insert semantic (insert/upsert) | +| `eloq_auto_increment` | Auto increment configuration | +| `eloq_invalidate_cache_once` | Invalidate cache once | +| `eloq_ddl_skip_kv` | Skip KV for DDL operations | +| `eloq_skip_redo_log` | Skip redo log | +| `eloq_scan_skip_kv` | Skip KV for scan operations | +| `eloq_random_scan_sort` | Random scan sort | +| `eloq_report_debug_info` | Report debug info | +| `eloq_deadlock_interval_sec` | Deadlock check interval | +| `eloq_signal_monitor` | Signal monitor | +| `eloq_partition_type` | Partition type | +| `eloq_range_split_worker_num` | Range split worker number | + +#### eloqkv-specific (gflags) +| Parameter | Description | +|-----------|-------------| +| `auto_redirect` | Auto redirect request to remote node | +| `cc_notify` | Notify txrequest sender when cc request finishes | +| `enable_io_uring` | Enable io_uring as IO engine | +| `raft_log_async_fsync` | Raft log fsync performed asynchronously | +| `bind_all` | Listen on all interfaces | +| `port` | Redis port | +| `bootstrap` | Init system tables and exit | +| `maxclients` | Maximum clients | +| `slow_log_threshold` | Slow log threshold | +| `slow_log_max_length` | Slow log max length | +| `enable_redis_stats` | Enable redis statistics | +| `enable_cmd_sort` | Enable command sort in Multi-Exec | +| `isolation_level` | Isolation level of simple commands | +| `protocol` | Concurrency control protocol | +| `txn_isolation_level` | Isolation level of MULTI/EXEC | +| `txn_protocol` | Concurrency control protocol of MULTI/EXEC | +| `snapshot_sync_worker_num` | Snapshot sync worker num | +| `retry_on_occ_error` | Retry transaction on OCC error | +| `fork_host_manager` | Fork host manager process | + +### Total Configuration Parameters: +- **eloqsql**: ~150+ parameters (including compile-time macro parameters) +- **eloqkv**: ~75+ parameters (including compile-time macro parameters) +- **Data Substrate**: ~40+ parameters (core unified parameters) + +### Key Observations: +1. **Naming Inconsistencies**: Different naming conventions across systems (e.g., `core_num` vs `core_number`) +2. **Missing Parameters**: Some parameters exist in one system but not others +3. **Compile-time Macros**: Many parameters are wrapped in compile-time macros for conditional compilation +4. **Engine-Specific**: Each system has unique parameters for its specific functionality +5. **Unified Approach**: Data Substrate provides a unified configuration system that can support all three engines + +### Recommendations: +1. **Standardize Naming**: Use consistent parameter names across all systems +2. **Complete Coverage**: Ensure all necessary parameters are available in Data Substrate +3. **Documentation**: Maintain this mapping as systems evolve +4. **Validation**: Implement parameter validation and type checking +5. **Migration Path**: Provide clear migration path from existing systems to Data Substrate diff --git a/data_substrate/core/config/sample_data_substrate.ini b/data_substrate/core/config/sample_data_substrate.ini new file mode 100644 index 00000000000..7e071a9674c --- /dev/null +++ b/data_substrate/core/config/sample_data_substrate.ini @@ -0,0 +1,41 @@ +[data_substrate] +eloq_data_path = /tmp/eloq_data +core_num = 8 +checkpointer_interval = 10 +node_memory_limit_mb = 8192 +checkpointer_delay_seconds = 5 +collect_active_tx_ts_interval_seconds = 2 +realtime_sampling = true +txlog_service_list = 127.0.0.1:8001,127.0.0.1:8002,127.0.0.1:8003 +txlog_group_replica_num = 3 +enable_tx_metrics = true +enable_log_service_metrics = true +enable_cache_hit_rate = true +enable_kv_metrics = true +enable_remote_request_metrics = true +collect_tx_duration_round = 1 +busy_round_threshold = 10 +enable_key_cache = true +enable_shard_heap_defragment = false +bthread_worker_num = 0 +max_standby_lag = 0 +kickout_data_for_test = false + +[storage] +storage_type = rocksdb +rocksdb_storage_path = /tmp/eloq_data/rocksdb +enable_cache_replacement = true +rocksdb_max_write_buffer_number = 3 +rocksdb_max_background_jobs = 4 +rocksdb_target_file_size_base = 67108864 + +[log_service] +log_service_data_path = /tmp/eloq_data/log_service +rocksdb_scan_threads = 4 +logserver_snapshot_interval = 3600 +enable_txlog_request_checkpoint = true +notify_checkpointer_threshold_size = 1GB + +[metrics] +enable_metrics = true +metrics_port = 9090 diff --git a/data_substrate/core/include/data_substrate.h b/data_substrate/core/include/data_substrate.h new file mode 100644 index 00000000000..792250a8c10 --- /dev/null +++ b/data_substrate/core/include/data_substrate.h @@ -0,0 +1,257 @@ +/** + * Copyright (C) 2025 EloqData Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under either of the following two licenses: + * 1. GNU Affero General Public License, version 3, as published by the Free + * Software Foundation. + * 2. GNU General Public License as published by the Free Software + * Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License or GNU General Public License for more + * details. + * + * You should have received a copy of the GNU Affero General Public License + * and GNU General Public License V2 along with this program. If not, see + * . + * + */ +#pragma once + +#include + +#include +#include +#include +#include +#include + +#if (defined(DATA_STORE_TYPE_ELOQDSS_ROCKSDB_CLOUD_S3) || \ + defined(DATA_STORE_TYPE_ELOQDSS_ROCKSDB_CLOUD_GCS) || \ + defined(DATA_STORE_TYPE_ELOQDSS_ROCKSDB) || \ + defined(DATA_STORE_TYPE_ELOQDSS_ELOQSTORE)) +#define ELOQDS 1 +#endif + +// Log state type +#if !defined(LOG_STATE_TYPE_RKDB_CLOUD) + +// Only if LOG_STATE_TYPE_RKDB_CLOUD undefined +#if ((defined(LOG_STATE_TYPE_RKDB_S3) || defined(LOG_STATE_TYPE_RKDB_GCS)) && \ + !defined(LOG_STATE_TYPE_RKDB)) +#define LOG_STATE_TYPE_RKDB_CLOUD 1 +#endif + +#endif + +#if !defined(LOG_STATE_TYPE_RKDB_ALL) + +// Only if LOG_STATE_TYPE_RKDB_ALL undefined +#if (defined(LOG_STATE_TYPE_RKDB_S3) || defined(LOG_STATE_TYPE_RKDB_GCS) || \ + defined(LOG_STATE_TYPE_RKDB)) +#define LOG_STATE_TYPE_RKDB_ALL 1 +#endif + +#endif + +#if defined(DATA_STORE_TYPE_DYNAMODB) || \ + (ROCKSDB_CLOUD_FS_TYPE == ROCKSDB_CLOUD_FS_TYPE_S3) +#include +#endif +#include "metrics.h" +#include "type.h" + +// Forward declaration for INIReader +class INIReader; + +// Forward declarations +namespace txservice +{ +class TxService; +struct NodeConfig; +class CatalogFactory; +class SystemHandler; +namespace store +{ +class DataStoreHandler; +} +} // namespace txservice + +#ifdef ELOQDS +namespace EloqDS +{ +class DataStoreService; +} // namespace EloqDS +#endif + +namespace txlog +{ +class LogServer; +class LogAgent; +} // namespace txlog + +namespace metrics +{ +class MetricsRegistry; +} // namespace metrics + +class DataSubstrate +{ +public: + struct EngineConfig + { + txservice::CatalogFactory *catalog_factory; + txservice::TableEngine engine; + bool enable_engine; + }; + + // Core data substrate configuration + struct CoreConfig + { + std::string data_path; + uint32_t core_num; + bool enable_heap_defragment; + bool enable_wal; + bool enable_data_store; + bool bootstrap; + bool enable_cache_replacement; + bool enable_mvcc; + uint32_t maxclients; + }; + + // Network and cluster configuration + struct NetworkConfig + { + std::string local_ip; + uint32_t local_port; + std::string ip_list; + std::string standby_ip_list; + std::string voter_ip_list; + uint32_t node_group_replica_num; + bool bind_all; + // Parsed service lists + std::unordered_map> + ng_configs; + uint64_t cluster_config_version; + uint32_t node_id; + uint32_t native_ng_id; + std::string cluster_config_file_path; + }; + + struct LogServiceConfig + { + std::vector txlog_ips; + std::vector txlog_ports; + }; + + static std::unique_ptr Initialize( + const std::string &config_file_path); + + // Global initialization function + static bool InitializeGlobal(const std::string &config_file_path); + + // Get global DataSubstrate instance + static DataSubstrate *GetGlobal(); + + // Shutdown DataSubstrate instance + void Shutdown(); + + // Component accessors + txservice::TxService *GetTxService() const + { + return tx_service_.get(); + } + txservice::store::DataStoreHandler *GetStoreHandler() const + { + return store_hd_.get(); + } + txlog::LogServer *GetLogServer() const + { + return log_server_.get(); + } + metrics::MetricsRegistry *GetMetricsRegistry() const + { + return metrics_registry_.get(); + } + // Configuration accessors + const CoreConfig &GetCoreConfig() const + { + return core_config_; + } + const NetworkConfig &GetNetworkConfig() const + { + return network_config_; + } + + ~DataSubstrate(); + + // Constructor (public for static factory method) + DataSubstrate() = default; + +private: + // Private methods that call component initialization files + bool LoadCoreAndNetworkConfig(const INIReader &config_file_reader); + bool InitializeStorageHandler(const INIReader &config_file_reader); + bool InitializeLogService(const INIReader &config_file_reader); + bool InitializeTxService(const INIReader &config_file_reader); + bool InitializeMetrics(const INIReader &config_file_reader); + bool RegisterEngines(); + + // Configuration storage + CoreConfig core_config_; + NetworkConfig network_config_; + LogServiceConfig log_service_config_; + metrics::CommonLabels tx_service_common_labels_{}; + // TODO(liunyl): system handler is used to refresh auth related cache. In + // converged db, there should only be one system handler. + txservice::SystemHandler *system_handler_{nullptr}; + + // Component instances + std::unique_ptr tx_service_; +#if defined(DATA_STORE_TYPE_DYNAMODB) || \ + defined(DATA_STORE_TYPE_ELOQDSS_ROCKSDB_CLOUD_S3) || \ + defined(LOG_STATE_TYPE_RKDB_S3) + Aws::SDKOptions aws_options_; +#endif + + std::unique_ptr store_hd_; +#ifdef ELOQDS + std::unique_ptr data_store_service_; +#endif + std::unique_ptr log_server_; + std::unique_ptr metrics_registry_{nullptr}; + + // Engine registry + EngineConfig engines_[NUM_EXTERNAL_ENGINES]; +// #ifdef ELOQ_MODULE_ELOQKV +// EloqKV::RedisCatalogFactory eloqkv_catalog_factory_; +// #endif + +// #ifdef ELOQ_MODULE_ELOQSQL +// MyEloq::MariaCatalogFactory eloqsql_catalog_factory_; +// #endif + + + std::unordered_map prebuilt_tables_; +}; + +// Global DataSubstrate instance +extern std::unique_ptr g_data_substrate; + + +// Helper function to check if a gflag was set from command line +static inline bool CheckCommandLineFlagIsDefault(const char *name) +{ + gflags::CommandLineFlagInfo flag_info; + bool flag_found = gflags::GetCommandLineFlagInfo(name, &flag_info); + // Make sure the flag is declared. + assert(flag_found); + (void) flag_found; + + // Return `true` if the flag has the default value and has not been set + // explicitly from the cmdline or via SetCommandLineOption + return flag_info.is_default; +} \ No newline at end of file diff --git a/data_substrate/core/include/mysql_metrics.h b/data_substrate/core/include/mysql_metrics.h new file mode 100644 index 00000000000..c0f98a25965 --- /dev/null +++ b/data_substrate/core/include/mysql_metrics.h @@ -0,0 +1,81 @@ +#pragma once +#include +#include "metrics.h" +#include "meter.h" + +namespace metrics +{ +extern std::unique_ptr mysql_meter; +inline bool enable_mysql_tx_metrics{false}; +inline bool enable_mysql_dml_metrics{false}; +extern Map cmd_to_string_map; + +inline const Name NAME_MYSQL_TX_DURATION{"mysql_tx_duration"}; +inline const Name NAME_MYSQL_PROCESSED_TX_TOTAL{"mysql_processed_tx_total"}; +inline const Name NAME_MYSQL_PROCESSED_QUERY_TOTAL{ + "mysql_processed_query_total"}; +inline const Name NAME_CONNECTION_COUNT{"mysql_connection_count"}; +inline const Name NAME_MAX_CONNECTIONS{"mysql_max_connections"}; +inline const Name NAME_MYSQL_DML_TOTAL{"mysql_dml_total"}; +inline const Name NAME_MYSQL_DML_DURATION{"mysql_dml_duration"}; + +void register_mysql_metrics(MetricsRegistry *metrics_registry, + CommonLabels common_labels= {}); + +#define TRACK_TX_METRICS(thd) \ + if (metrics::enable_mysql_tx_metrics && \ + !(thd)->transaction->tx_registered_) \ + { \ + (thd)->transaction->tx_start_= metrics::Clock::now(); \ + (thd)->transaction->tx_registered_= true; \ + } + +#define COLLECT_TX_METRICS(thd) \ + if (metrics::enable_mysql_tx_metrics && (thd)->transaction->tx_registered_) \ + { \ + (thd)->transaction->tx_registered_= false; \ + metrics::mysql_meter->CollectDuration(metrics::NAME_MYSQL_TX_DURATION, \ + (thd)->transaction->tx_start_); \ + metrics::mysql_meter->Collect(metrics::NAME_MYSQL_PROCESSED_TX_TOTAL, 1); \ + } + +#define IS_MONITORED_DML(dml) \ + ((dml) == SQLCOM_SELECT || \ + (dml) == SQLCOM_INSERT || \ + (dml) == SQLCOM_INSERT_SELECT || \ + (dml) == SQLCOM_UPDATE || \ + (dml) == SQLCOM_UPDATE_MULTI || \ + (dml) == SQLCOM_DELETE || \ + (dml) == SQLCOM_DELETE_MULTI || \ + (dml) == SQLCOM_LOAD) + +/* + Usage: + metrics::TimePoint dml_start; + TRACK_DML_METRICS(dml, dml_start) + */ +#define TRACK_DML_METRICS(dml, dml_start) \ + { \ + if (metrics::enable_mysql_dml_metrics && IS_MONITORED_DML(dml)) \ + { \ + dml_start= metrics::Clock::now(); \ + } \ + } + +/* + Usage: + TRACK_DML_METRICS_DONE(dml, dml_start) + */ +#define TRACK_DML_METRICS_DONE(dml, dml_start) \ + { \ + int dml_type= static_cast(dml); \ + if (metrics::enable_mysql_dml_metrics && IS_MONITORED_DML(dml)) \ + { \ + std::string_view dml_sv= metrics::cmd_to_string_map.at(dml_type); \ + metrics::mysql_meter->CollectDuration(metrics::NAME_MYSQL_DML_DURATION, \ + dml_start, dml_sv); \ + metrics::mysql_meter->Collect(metrics::NAME_MYSQL_DML_TOTAL, 1, \ + dml_sv); \ + } \ + } +} // namespace metrics diff --git a/data_substrate/core/src/data_substrate.cpp b/data_substrate/core/src/data_substrate.cpp new file mode 100644 index 00000000000..fe5473360ab --- /dev/null +++ b/data_substrate/core/src/data_substrate.cpp @@ -0,0 +1,528 @@ +/** + * Copyright (C) 2025 EloqData Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under either of the following two licenses: + * 1. GNU Affero General Public License, version 3, as published by the Free + * Software Foundation. + * 2. GNU General Public License as published by the Free Software + * Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License or GNU General Public License for more + * details. + * + * You should have received a copy of the GNU Affero General Public License + * and GNU General Public License V2 along with this program. If not, see + * . + * + */ + +#include "data_substrate.h" + +#include +#include + +#include +#include + +#include "INIReader.h" +#include "log_server.h" +#include "store/data_store_handler.h" +#include "tx_service.h" +#include "sequences/sequences.h" + +#ifdef ELOQDS +#include "eloq_data_store_service/data_store_service.h" +#endif + + +// Data Substrate gflags definitions +DEFINE_string(eloq_data_path, "/tmp/eloq_data", "Data substrate data path"); +DEFINE_int32(core_number, 8, "Number of cores for data substrate"); +DEFINE_bool(enable_heap_defragment, false, "Enable heap defragmentation"); +DEFINE_bool(enable_wal, true, "Enable WAL"); +DEFINE_bool(enable_data_store, true, "Enable data store"); +DEFINE_bool(enable_io_uring, false, "Enable io_uring as the IO engine"); +DEFINE_bool(raft_log_async_fsync, false, + "Whether raft log fsync is performed asynchronously (blocking the " + "bthread) instead of blocking the worker thread"); +DEFINE_bool(bootstrap, false, "init system tables and exit"); +DEFINE_bool(enable_cache_replacement, true, "Enable cache replacement"); +DEFINE_bool(enable_mvcc, false, "Enable mvcc"); + +// Network configuration gflags +DEFINE_string(tx_ip, "127.0.0.1", "Local tx service IP address"); +DEFINE_int32(tx_port, 16379, "Local tx service port"); +DEFINE_string(ip_port_list, "", "IP list for cluster"); +DEFINE_string(standby_ip_port_list, "", "Standby IP list"); +DEFINE_string(voter_ip_port_list, "", "Voter IP list"); + +DEFINE_string(cluster_config_file, "", "Cluster configuration file"); +DEFINE_int32(node_group_replica_num, 3, "Node group replica number"); +DEFINE_bool( + bind_all, false, + "Listen on all interfaces if enabled, otherwise listen on local.ip"); +DEFINE_uint32(maxclients, 10000, "maxclients"); +// DEFINE_string(log_file_name_prefix, +// "eloqdb.log", +// "Sets the prefix for log files. Default is 'eloqdb.log'"); +// Declare as weak symbols to allow optional linking +extern txservice::CatalogFactory *eloqsql_catalog_factory __attribute__((weak)); +extern txservice::CatalogFactory *eloqkv_catalog_factory __attribute__((weak)); +extern txservice::SystemHandler *eloqsql_system_handler __attribute__((weak)); + +namespace brpc +{ +DECLARE_int32(event_dispatcher_num); +} +const auto NUM_VCPU= std::thread::hardware_concurrency(); + +// Global DataSubstrate instance +std::unique_ptr g_data_substrate= nullptr; + +std::unique_ptr +DataSubstrate::Initialize(const std::string &config_file_path) +{ + auto data_substrate= std::make_unique(); + + INIReader config_file_reader(config_file_path); + if (!config_file_path.empty() && config_file_reader.ParseError() != 0) + { + LOG(ERROR) << "Failed to parse config file: " << config_file_path; + return nullptr; + } + // Phase 1: Load core and network configuration + if (!data_substrate->LoadCoreAndNetworkConfig(config_file_reader)) + { + return nullptr; + } + + // Phase 2: Register engines + if (!data_substrate->RegisterEngines()) + { + return nullptr; + } + + // Phase 3: Initialize log service + if (!data_substrate->InitializeLogService(config_file_reader)) + { + return nullptr; + } + + // Phase 4: Initialize storage handler + if (data_substrate->GetCoreConfig().enable_data_store && + !data_substrate->InitializeStorageHandler(config_file_reader)) + { + return nullptr; + } + + // Phase 5: Initialize metrics + if (!data_substrate->InitializeMetrics(config_file_reader)) + { + return nullptr; + } + + + // Phase 6: Initialize TxService + if (!data_substrate->InitializeTxService(config_file_reader)) + { + return nullptr; + } + return data_substrate; +} + +bool DataSubstrate::InitializeGlobal(const std::string &config_file_path) +{ + if (g_data_substrate != nullptr) + { + LOG(WARNING) << "Global DataSubstrate already initialized"; + return true; + } + + g_data_substrate= Initialize(config_file_path); + if (g_data_substrate == nullptr) + { + LOG(ERROR) << "Failed to initialize global DataSubstrate"; + return false; + } + + LOG(INFO) << "Global DataSubstrate initialized successfully"; + return true; +} + +DataSubstrate *DataSubstrate::GetGlobal() { return g_data_substrate.get(); } + +DataSubstrate::~DataSubstrate() +{ + // Note: We can't call Shutdown() here because TxService is an incomplete + // type The actual shutdown will be handled by the component that owns the + // TxService For now, just reset the pointers + tx_service_= nullptr; + store_hd_= nullptr; +#ifdef ELOQDS + data_store_service_= nullptr; +#endif + log_server_= nullptr; + metrics_registry_= nullptr; +} + +void DataSubstrate::Shutdown() +{ + if (tx_service_ != nullptr) + { + LOG(INFO) << "Shutting down the tx service."; + tx_service_->Shutdown(); + LOG(INFO) << "Tx service shut down."; + } + + if (store_hd_ != nullptr) + { + LOG(INFO) << "Shutting down the storage handler."; + store_hd_= nullptr; // Wait for all in-fight requests complete. +#if ELOQDS + if (data_store_service_ != nullptr) + { + data_store_service_->DisconnectDataStore(); + data_store_service_= nullptr; + } +#endif + LOG(INFO) << "Storage handler shut down."; + } + +#if (WITH_LOG_SERVICE) + if (log_server_ != nullptr) + { + LOG(INFO) << "Shutting down the internal logservice."; + log_server_= nullptr; + LOG(INFO) << "Internal logservice shut down."; + } +#endif + txservice::Sequences::Destory(); +// #ifdef ELOQ_MODULE_ELOQSQL +// MyEloq::MariaSystemHandler::Instance().Shutdown(); +// #endif + +#if defined(DATA_STORE_TYPE_DYNAMODB) || \ + defined(DATA_STORE_TYPE_ELOQDSS_ROCKSDB_CLOUD_S3) || \ + defined(LOG_STATE_TYPE_RKDB_S3) + Aws::ShutdownAPI(aws_options_); +#endif + + tx_service_= nullptr; +} + +bool DataSubstrate::RegisterEngines() +{ + engines_[0]= {nullptr, txservice::TableEngine::EloqSql, false}; + engines_[1]= {nullptr, txservice::TableEngine::EloqKv, false}; + engines_[2]= {nullptr, txservice::TableEngine::EloqDoc, false}; + +#ifdef ELOQ_MODULE_ELOQSQL + engines_[0].enable_engine= true; + if (eloqsql_catalog_factory) { + engines_[0].catalog_factory= eloqsql_catalog_factory; + } + if (eloqsql_system_handler) { + system_handler_ = eloqsql_system_handler; + } +#endif + +#ifdef ELOQ_MODULE_ELOQKV + engines_[1].enable_engine= true; + if (eloqkv_catalog_factory) { + engines_[1].catalog_factory= eloqkv_catalog_factory; + } + for (uint16_t i= 0; i < 16; i++) + { + std::string table_name("data_table_"); + table_name.append(std::to_string(i)); + txservice::TableName redis_table_name(std::move(table_name), + txservice::TableType::Primary, + txservice::TableEngine::EloqKv); + std::string image= redis_table_name.String(); + prebuilt_tables_.try_emplace(redis_table_name, image); + } +#endif + + return true; +} + +bool DataSubstrate::LoadCoreAndNetworkConfig(const INIReader &config_reader) +{ + // Load core data substrate configuration with gflags priority + core_config_.data_path= + !CheckCommandLineFlagIsDefault("eloq_data_path") + ? FLAGS_eloq_data_path + : config_reader.Get("local", "eloq_data_path", FLAGS_eloq_data_path); + + core_config_.enable_heap_defragment= + !CheckCommandLineFlagIsDefault("enable_heap_defragment") + ? FLAGS_enable_heap_defragment + : config_reader.GetBoolean("local", "enable_heap_defragment", + FLAGS_enable_heap_defragment); + + core_config_.enable_cache_replacement= + !CheckCommandLineFlagIsDefault("enable_cache_replacement") + ? FLAGS_enable_cache_replacement + : config_reader.GetBoolean("local", "enable_cache_replacement", + FLAGS_enable_cache_replacement); + + core_config_.maxclients= + !CheckCommandLineFlagIsDefault("maxclients") + ? FLAGS_maxclients + : config_reader.GetInteger("local", "maxclients", FLAGS_maxclients); + struct rlimit ulimit; + ulimit.rlim_cur= FLAGS_maxclients; + ulimit.rlim_max= FLAGS_maxclients; + if (setrlimit(RLIMIT_NOFILE, &ulimit) == -1) + { + LOG(ERROR) << "Failed to set maxclients."; + return false; + } + core_config_.bootstrap= + !CheckCommandLineFlagIsDefault("bootstrap") + ? FLAGS_bootstrap + : config_reader.GetBoolean("local", "bootstrap", FLAGS_bootstrap); + + core_config_.enable_wal= + !CheckCommandLineFlagIsDefault("enable_wal") + ? FLAGS_enable_wal + : config_reader.GetBoolean("local", "enable_wal", FLAGS_enable_wal); + + core_config_.enable_data_store= + !CheckCommandLineFlagIsDefault("enable_data_store") + ? FLAGS_enable_data_store + : config_reader.GetBoolean("local", "enable_data_store", + FLAGS_enable_data_store); + if (core_config_.enable_wal && !core_config_.enable_data_store) + { + LOG(ERROR) << "When set enable_wal, should also set enable_data_store"; + return false; + } + + if (!core_config_.enable_data_store && core_config_.enable_cache_replacement) + { + LOG(WARNING) << "When set enable_cache_replacement, should also set " + "enable_data_store, reset to false"; + core_config_.enable_cache_replacement= false; + } + + const char *field_core= "core_number"; + core_config_.core_num= FLAGS_core_number; + if (CheckCommandLineFlagIsDefault(field_core)) + { + if (config_reader.HasValue("local", field_core)) + { + core_config_.core_num= config_reader.GetInteger("local", field_core, 0); + assert(core_config_.core_num); + } + else + { + if (!NUM_VCPU) + { + LOG(ERROR) << "config is missing: " << field_core; + return false; + } + const uint min= 1; + if (core_config_.enable_data_store) + { + core_config_.core_num= std::max(min, (NUM_VCPU * 3) / 5); + LOG(INFO) << "give cpus to checkpointer " << core_config_.core_num; + } + else + { + core_config_.core_num= std::max(min, (NUM_VCPU * 7) / 10); + } + LOG(INFO) << "config is automatically set: " << field_core << "=" + << core_config_.core_num << ", vcpu=" << NUM_VCPU; + } + } + + // Load network and cluster configuration with gflags priority + network_config_.local_ip= + !CheckCommandLineFlagIsDefault("tx_ip") + ? FLAGS_tx_ip + : config_reader.Get("local", "tx_ip", FLAGS_tx_ip); + + network_config_.local_port= + !CheckCommandLineFlagIsDefault("tx_port") + ? FLAGS_tx_port + : config_reader.GetInteger("local", "tx_port", FLAGS_tx_port); + std::string local_ip_port= network_config_.local_ip + ":" + + std::to_string(network_config_.local_port); + DLOG(INFO) << "Local tx service ip port: " << local_ip_port; + + network_config_.ip_list= + !CheckCommandLineFlagIsDefault("ip_port_list") + ? FLAGS_ip_port_list + : config_reader.Get("cluster", "ip_port_list", local_ip_port); + + network_config_.standby_ip_list= + !CheckCommandLineFlagIsDefault("standby_ip_port_list") + ? FLAGS_standby_ip_port_list + : config_reader.Get("cluster", "standby_ip_port_list", ""); + + network_config_.voter_ip_list= + !CheckCommandLineFlagIsDefault("voter_ip_port_list") + ? FLAGS_voter_ip_port_list + : config_reader.Get("cluster", "voter_ip_port_list", ""); + + network_config_.cluster_config_file_path= + !CheckCommandLineFlagIsDefault("cluster_config_file") + ? FLAGS_cluster_config_file + : config_reader.Get("cluster", "cluster_config_file", + FLAGS_cluster_config_file); + + network_config_.node_group_replica_num= + !CheckCommandLineFlagIsDefault("node_group_replica_num") + ? FLAGS_node_group_replica_num + : config_reader.GetInteger("cluster", "node_group_replica_num", + FLAGS_node_group_replica_num); + + network_config_.bind_all= + !CheckCommandLineFlagIsDefault("bind_all") + ? FLAGS_bind_all + : config_reader.GetBoolean("local", "bind_all", FLAGS_bind_all); + + core_config_.enable_mvcc= + !CheckCommandLineFlagIsDefault("enable_mvcc") + ? FLAGS_enable_mvcc + : config_reader.GetBoolean("local", "enable_mvcc", + FLAGS_enable_mvcc); + + // Try to read cluster config from file. Cluster config file is written by + // host manager when cluster config updates. + if (network_config_.cluster_config_file_path.empty()) + { + network_config_.cluster_config_file_path= + core_config_.data_path + "/tx_service/cluster_config"; + } + + network_config_.cluster_config_version= 2; + if (!txservice::ReadClusterConfigFile( + network_config_.cluster_config_file_path, network_config_.ng_configs, + network_config_.cluster_config_version)) + { + // Read cluster topology from general config file in this case + auto parse_res= txservice::ParseNgConfig( + network_config_.ip_list, network_config_.standby_ip_list, + network_config_.voter_ip_list, network_config_.ng_configs, + network_config_.node_group_replica_num, 0); + if (!parse_res) + { + LOG(ERROR) << "Failed to extract cluster configs from ip_port_list."; + return false; + } + } + + // print out ng_configs + for (auto &pair : network_config_.ng_configs) + { + DLOG(INFO) << "ng_id: " << pair.first; + for (auto &node : pair.second) + { + DLOG(INFO) << "node_id: " << node.node_id_ + << ", host_name: " << node.host_name_ + << ", port: " << node.port_; + } + } + + // check whether this node is in cluster. + bool found= false; + network_config_.node_id= 0; + network_config_.native_ng_id= 0; + for (auto &pair : network_config_.ng_configs) + { + auto &ng_nodes= pair.second; + for (size_t i= 0; i < ng_nodes.size(); i++) + { + if (ng_nodes[i].host_name_ == network_config_.local_ip && + ng_nodes[i].port_ == network_config_.local_port) + { + network_config_.node_id= ng_nodes[i].node_id_; + found= true; + if (ng_nodes[i].is_candidate_) + { + // found native_ng_id. + network_config_.native_ng_id= pair.first; + break; + } + } + } + } + + if (!found) + { + LOG(ERROR) << "!!!!!!!! Current node does not belong to the " + "cluster, startup is terminated !!!!!!!!"; + return false; + } + const char *field_ed= "event_dispatcher_num"; + uint num_iothreads= brpc::FLAGS_event_dispatcher_num; + if (CheckCommandLineFlagIsDefault(field_ed)) + { + if (config_reader.HasValue("local", field_ed)) + { + num_iothreads= config_reader.GetInteger("local", field_ed, 0); + assert(num_iothreads); + } + else + { + if (!NUM_VCPU) + { + LOG(ERROR) << "config is missing: " << field_ed; + return false; + } + num_iothreads= std::max(uint(1), (core_config_.core_num + 5) / 6); + LOG(INFO) << "config is automatically set: " << field_ed << "=" + << num_iothreads << ", vcpu=" << NUM_VCPU; + } + } + GFLAGS_NAMESPACE::SetCommandLineOption( + field_ed, std::to_string(num_iothreads).c_str()); + + // Set bthread concurrency + GFLAGS_NAMESPACE::SetCommandLineOption( + "bthread_concurrency", std::to_string(core_config_.core_num).c_str()); + // TODO(liunyl): wrap with eloq module macro? + GFLAGS_NAMESPACE::SetCommandLineOption("worker_polling_time_us", "1000"); + GFLAGS_NAMESPACE::SetCommandLineOption("brpc_worker_as_ext_processor", + "true"); + GFLAGS_NAMESPACE::SetCommandLineOption("use_pthread_event_dispatcher", + "true"); + GFLAGS_NAMESPACE::SetCommandLineOption("max_body_size", "536870912"); + GFLAGS_NAMESPACE::SetCommandLineOption("graceful_quit_on_sigterm", "true"); + + bool enable_io_uring= + !CheckCommandLineFlagIsDefault("enable_io_uring") + ? FLAGS_enable_io_uring + : config_reader.GetBoolean("local", "enable_io_uring", + FLAGS_enable_io_uring); + + bool raft_log_async_fsync= + !CheckCommandLineFlagIsDefault("raft_log_async_fsync") + ? FLAGS_raft_log_async_fsync + : config_reader.GetBoolean("local", "raft_log_async_fsync", + FLAGS_raft_log_async_fsync); + if (raft_log_async_fsync && !enable_io_uring) + { + LOG(ERROR) << "Invalid config: when set `enable_io_uring`, " + "should also set `enable_io_uring`."; + return false; + } + GFLAGS_NAMESPACE::SetCommandLineOption("use_io_uring", + enable_io_uring ? "true" : "false"); + GFLAGS_NAMESPACE::SetCommandLineOption( + "raft_use_bthread_fsync", raft_log_async_fsync ? "true" : "false"); +#if defined(DATA_STORE_TYPE_DYNAMODB) || \ + defined(DATA_STORE_TYPE_ELOQDSS_ROCKSDB_CLOUD_S3) || \ + defined(LOG_STATE_TYPE_RKDB_S3) + aws_options_.loggingOptions.logLevel= Aws::Utils::Logging::LogLevel::Info; + Aws::InitAPI(aws_options_); +#endif + + return true; +} \ No newline at end of file diff --git a/data_substrate/core/src/log_init.cpp b/data_substrate/core/src/log_init.cpp new file mode 100644 index 00000000000..cf45f43ca50 --- /dev/null +++ b/data_substrate/core/src/log_init.cpp @@ -0,0 +1,541 @@ +#include "INIReader.h" +#include "data_substrate.h" +#include "log_server.h" +#include "log_utils.h" +#include "sharder.h" +#if defined(LOG_STATE_TYPE_RKDB_CLOUD) +#include "rocksdb_cloud_config.h" +#endif + +DEFINE_string(txlog_service_list, "", "Log group servers configuration"); +DEFINE_int32(txlog_group_replica_num, 3, "Replica number of one log group"); +DEFINE_string(log_service_data_path, "", "path for log_service data"); + +DEFINE_string(txlog_rocksdb_storage_path, + "", + "Storage path for txlog rocksdb state"); +DEFINE_uint32(txlog_rocksdb_max_write_buffer_number, + 8, + "Max write buffer number"); +DEFINE_string(txlog_rocksdb_sst_files_size_limit, + "500MB", + "The total RocksDB sst files size before purge"); +DEFINE_uint32(txlog_rocksdb_scan_threads, + 1, + "The number of rocksdb scan threads"); + +DEFINE_uint32(txlog_rocksdb_max_background_jobs, 12, "Max background jobs"); +DEFINE_string(txlog_rocksdb_target_file_size_base, + "64MB", + "Target file size base for rocksdb"); + +DEFINE_uint32(logserver_snapshot_interval, 600, "logserver_snapshot interval"); + +DEFINE_bool(enable_txlog_request_checkpoint, + true, + "Enable txlog server sending checkpoint requests when the criteria " + "are met."); + +#if defined(LOG_STATE_TYPE_RKDB_CLOUD) +DEFINE_string(txlog_rocksdb_cloud_region, + "ap-northeast-1", + "Cloud service regin"); +DEFINE_string(txlog_rocksdb_cloud_bucket_name, + "rocksdb-cloud-test", + "Cloud storage bucket name"); +DEFINE_string(txlog_rocksdb_cloud_bucket_prefix, + "txlog-", + "Cloud storage bucket prefix"); +DEFINE_string(txlog_rocksdb_cloud_object_path, + "eloqkv_txlog", + "Cloud storage object path, if not set, will use bucket name and " + "prefix"); +DEFINE_uint32( + txlog_rocksdb_cloud_ready_timeout, + 10, + "Timeout before rocksdb cloud becoming ready on new log group leader"); +DEFINE_uint32(txlog_rocksdb_cloud_file_deletion_delay, + 3600, + "The file deletion delay for rocksdb cloud file"); +DEFINE_uint32(txlog_rocksdb_cloud_log_retention_days, + 90, + "The number of days for which logs should be retained"); +DEFINE_string(txlog_rocksdb_cloud_log_purger_schedule, + "00:00:01", + "Time (in regular format: HH:MM:SS) to run log purger daily, " + "deleting logs older than log_retention_days."); +DEFINE_uint32(txlog_in_mem_data_log_queue_size_high_watermark, + 50 * 10000, + "In memory data log queue max size"); +DEFINE_string(txlog_rocksdb_cloud_endpoint_url, + "", + "Endpoint url of cloud storage service"); +DEFINE_string(txlog_rocksdb_cloud_sst_file_cache_size, + "1GB", + "Local sst cache size for txlog"); +DEFINE_uint32(txlog_rocksdb_cloud_sst_file_cache_num_shard_bits, + 5, + "TxLog RocksDB Cloud SST file cache num shard bits"); +#endif +#ifdef WITH_CLOUD_AZ_INFO +DEFINE_string(txlog_rocksdb_cloud_prefer_zone, + "", + "user preferred deployed availability zone"); +DEFINE_string(txlog_rocksdb_cloud_current_zone, + "", + "the log service server node deployed on currently"); +#endif + +DEFINE_uint32(check_replay_log_size_interval_sec, + 10, + "The interval for checking replay log size."); + +DEFINE_string(notify_checkpointer_threshold_size, + "1GB", + "When the replay log size reaches this threshold the txlog " + "server sends a checkpoint request to tx_service."); + + +bool DataSubstrate::InitializeLogService(const INIReader &config_reader) +{ + std::string txlog_service = + !CheckCommandLineFlagIsDefault("txlog_service_list") + ? FLAGS_txlog_service_list + : config_reader.GetString( + "cluster", "txlog_service_list", FLAGS_txlog_service_list); + int txlog_group_replica_num = + !CheckCommandLineFlagIsDefault("txlog_group_replica_num") + ? FLAGS_txlog_group_replica_num + : config_reader.GetInteger("cluster", + "txlog_group_replica_num", + FLAGS_txlog_group_replica_num); + + std::string log_service_data_path = + !CheckCommandLineFlagIsDefault("log_service_data_path") + ? FLAGS_log_service_data_path + : config_reader.GetString("local", "log_service_data_path", ""); + std::string log_path("local://"); + if (log_service_data_path.empty()) + { + log_path.append(core_config_.data_path); + } + else + { + log_path.append(log_service_data_path); + } + log_path.append("/log_service"); + // parse standalone txlog_service_list + bool is_standalone_txlog_service = false; + std::vector &txlog_ips = log_service_config_.txlog_ips; + std::vector &txlog_ports = log_service_config_.txlog_ports; + if (!txlog_service.empty()) + { + is_standalone_txlog_service = true; + std::string token; + std::istringstream txlog_ip_port_list_stream(txlog_service); + while (std::getline(txlog_ip_port_list_stream, token, ',')) + { + size_t c_idx = token.find_first_of(':'); + if (c_idx != std::string::npos) + { + txlog_ips.emplace_back(token.substr(0, c_idx)); + uint16_t pt = std::stoi(token.substr(c_idx + 1)); + txlog_ports.emplace_back(pt); + } + } + } +#if defined(WITH_LOG_SERVICE) + else + { + uint32_t txlog_node_id = 0; + uint32_t next_txlog_node_id = 0; + uint16_t log_server_port = network_config_.local_port + 2; + std::unordered_set tx_node_ids; + for (uint32_t ng = 0; ng < network_config_.ng_configs.size(); ng++) + { + for (uint32_t nidx = 0; + nidx < network_config_.ng_configs[ng].size(); + nidx++) + { + if (tx_node_ids.count( + network_config_.ng_configs[ng][nidx].node_id_) == 0) + { + tx_node_ids.insert( + network_config_.ng_configs[ng][nidx].node_id_); + if (network_config_.ng_configs[ng][nidx].port_ == + network_config_.local_port && + network_config_.ng_configs[ng][nidx].host_name_ == + network_config_.local_ip) + { + // is local node, set txlog_node_id + txlog_node_id = next_txlog_node_id; + } + next_txlog_node_id++; + txlog_ports.emplace_back( + network_config_.ng_configs[ng][nidx].port_ + 2); + txlog_ips.emplace_back( + network_config_.ng_configs[ng][nidx].host_name_); + } + } + } + + // Init local txlog service if it is not standalone txlog service + // Init before store handler so they can init in parallel + if (!is_standalone_txlog_service) + { + if (txlog_ips.empty() || txlog_ports.empty()) + { + LOG(ERROR) + << "WAL is enabled but `txlog_service_list` is empty and " + "built-in log server initialization failed, " + "unable to proceed."; + return false; + } + + [[maybe_unused]] size_t txlog_rocksdb_scan_threads = + !CheckCommandLineFlagIsDefault("txlog_rocksdb_scan_threads") + ? FLAGS_txlog_rocksdb_scan_threads + : config_reader.GetInteger( + "local", + "txlog_rocksdb_scan_threads", + FLAGS_txlog_rocksdb_scan_threads); + + size_t txlog_rocksdb_max_write_buffer_number = + !CheckCommandLineFlagIsDefault( + "txlog_rocksdb_max_write_buffer_number") + ? FLAGS_txlog_rocksdb_max_write_buffer_number + : config_reader.GetInteger( + "local", + "txlog_rocksdb_max_write_buffer_number", + FLAGS_txlog_rocksdb_max_write_buffer_number); + + size_t txlog_rocksdb_max_background_jobs = + !CheckCommandLineFlagIsDefault( + "txlog_rocksdb_max_background_jobs") + ? FLAGS_txlog_rocksdb_max_background_jobs + : config_reader.GetInteger( + "local", + "txlog_rocksdb_max_background_jobs", + FLAGS_txlog_rocksdb_max_background_jobs); + + size_t txlog_rocksdb_target_file_size_base_val = + !CheckCommandLineFlagIsDefault( + "txlog_rocksdb_target_file_size_base") + ? txlog::parse_size( + FLAGS_txlog_rocksdb_target_file_size_base) + : txlog::parse_size(config_reader.GetString( + "local", + "txlog_rocksdb_target_file_size_base", + FLAGS_txlog_rocksdb_target_file_size_base)); + + [[maybe_unused]] size_t logserver_snapshot_interval = + !CheckCommandLineFlagIsDefault("logserver_snapshot_interval") + ? FLAGS_logserver_snapshot_interval + : config_reader.GetInteger( + "local", + "logserver_snapshot_interval", + FLAGS_logserver_snapshot_interval); + + [[maybe_unused]] bool enable_txlog_request_checkpoint = + !CheckCommandLineFlagIsDefault( + "enable_txlog_request_checkpoint") + ? FLAGS_enable_txlog_request_checkpoint + : config_reader.GetBoolean( + "local", + "enable_txlog_request_checkpoint", + FLAGS_enable_txlog_request_checkpoint); + + [[maybe_unused]] size_t check_replay_log_size_interval_sec = + !CheckCommandLineFlagIsDefault( + "check_replay_log_size_interval_sec") + ? FLAGS_check_replay_log_size_interval_sec + : config_reader.GetInteger( + "local", + "check_replay_log_size_interval_sec", + FLAGS_check_replay_log_size_interval_sec); + + [[maybe_unused]] size_t notify_checkpointer_threshold_size_val = + !CheckCommandLineFlagIsDefault( + "notify_checkpointer_threshold_size") + ? txlog::parse_size( + FLAGS_notify_checkpointer_threshold_size) + : txlog::parse_size(config_reader.GetString( + "local", + "notify_checkpointer_threshold_size", + FLAGS_notify_checkpointer_threshold_size)); + +#if defined(LOG_STATE_TYPE_RKDB_ALL) + std::string txlog_rocksdb_storage_path = + !CheckCommandLineFlagIsDefault("txlog_rocksdb_storage_path") + ? FLAGS_txlog_rocksdb_storage_path + : config_reader.GetString( + "local", "txlog_rocksdb_storage_path", ""); + if (txlog_rocksdb_storage_path.empty()) + { + // remove "local://" prefix from log_path + txlog_rocksdb_storage_path = log_path.substr(8) + "/rocksdb"; + } + +#if defined(LOG_STATE_TYPE_RKDB_CLOUD) + txlog::RocksDBCloudConfig txlog_rocksdb_cloud_config; +#if defined(LOG_STATE_TYPE_RKDB_S3) + txlog_rocksdb_cloud_config.aws_access_key_id_ = + !CheckCommandLineFlagIsDefault("aws_access_key_id") + ? FLAGS_aws_access_key_id + : config_reader.GetString("store", "aws_access_key_id", ""); + txlog_rocksdb_cloud_config.aws_secret_key_ = + !CheckCommandLineFlagIsDefault("aws_secret_key") + ? FLAGS_aws_secret_key + : config_reader.GetString("store", "aws_secret_key", ""); +#endif /* LOG_STATE_TYPE_RKDB_S3 */ + txlog_rocksdb_cloud_config.endpoint_url_ = + !CheckCommandLineFlagIsDefault( + "txlog_rocksdb_cloud_endpoint_url") + ? FLAGS_txlog_rocksdb_cloud_endpoint_url + : config_reader.GetString( + "local", + "txlog_rocksdb_cloud_endpoint_url", + FLAGS_txlog_rocksdb_cloud_endpoint_url); + txlog_rocksdb_cloud_config.bucket_name_ = + !CheckCommandLineFlagIsDefault( + "txlog_rocksdb_cloud_bucket_name") + ? FLAGS_txlog_rocksdb_cloud_bucket_name + : config_reader.GetString( + "local", + "txlog_rocksdb_cloud_bucket_name", + FLAGS_txlog_rocksdb_cloud_bucket_name); + txlog_rocksdb_cloud_config.bucket_prefix_ = + !CheckCommandLineFlagIsDefault( + "txlog_rocksdb_cloud_bucket_prefix") + ? FLAGS_txlog_rocksdb_cloud_bucket_prefix + : config_reader.GetString( + "local", + "txlog_rocksdb_cloud_bucket_prefix", + FLAGS_txlog_rocksdb_cloud_bucket_prefix); + txlog_rocksdb_cloud_config.object_path_ = + !CheckCommandLineFlagIsDefault( + "txlog_rocksdb_cloud_object_path") + ? FLAGS_txlog_rocksdb_cloud_object_path + : config_reader.GetString( + "local", + "txlog_rocksdb_cloud_object_path", + FLAGS_txlog_rocksdb_cloud_object_path); + txlog_rocksdb_cloud_config.region_ = + !CheckCommandLineFlagIsDefault("txlog_rocksdb_cloud_region") + ? FLAGS_txlog_rocksdb_cloud_region + : config_reader.GetString("local", + "txlog_rocksdb_cloud_region", + FLAGS_txlog_rocksdb_cloud_region); + uint32_t db_ready_timeout_us = + !CheckCommandLineFlagIsDefault( + "txlog_rocksdb_cloud_ready_timeout") + ? FLAGS_txlog_rocksdb_cloud_ready_timeout + : config_reader.GetInteger( + "local", + "txlog_rocksdb_cloud_ready_timeout", + FLAGS_txlog_rocksdb_cloud_ready_timeout); + txlog_rocksdb_cloud_config.db_ready_timeout_us_ = + db_ready_timeout_us * 1000 * 1000; + txlog_rocksdb_cloud_config.db_file_deletion_delay_ = + !CheckCommandLineFlagIsDefault( + "txlog_rocksdb_cloud_file_deletion_delay") + ? FLAGS_txlog_rocksdb_cloud_file_deletion_delay + : config_reader.GetInteger( + "local", + "txlog_rocksdb_cloud_file_deletion_delay", + FLAGS_txlog_rocksdb_cloud_file_deletion_delay); + txlog_rocksdb_cloud_config.log_retention_days_ = + !CheckCommandLineFlagIsDefault( + "txlog_rocksdb_cloud_log_retention_days") + ? FLAGS_txlog_rocksdb_cloud_log_retention_days + : config_reader.GetInteger( + "local", + "txlog_rocksdb_cloud_log_retention_days", + FLAGS_txlog_rocksdb_cloud_log_retention_days); + txlog_rocksdb_cloud_config.sst_file_cache_num_shard_bits_ = + !CheckCommandLineFlagIsDefault( + "txlog_rocksdb_cloud_sst_file_cache_num_shard_bits") + ? FLAGS_txlog_rocksdb_cloud_sst_file_cache_num_shard_bits + : config_reader.GetInteger( + "local", + "txlog_rocksdb_cloud_sst_file_cache_num_shard_bits", + FLAGS_txlog_rocksdb_cloud_sst_file_cache_num_shard_bits); + txlog_rocksdb_cloud_config.sst_file_cache_size_ = + !CheckCommandLineFlagIsDefault( + "txlog_rocksdb_cloud_sst_file_cache_size") + ? txlog::parse_size( + FLAGS_txlog_rocksdb_cloud_sst_file_cache_size) + : txlog::parse_size(config_reader.GetString( + "local", + "txlog_rocksdb_cloud_sst_file_cache_size", + FLAGS_txlog_rocksdb_cloud_sst_file_cache_size)); + std::tm log_purger_tm{}; + std::string log_purger_schedule = + !CheckCommandLineFlagIsDefault( + "txlog_rocksdb_cloud_log_purger_schedule") + ? FLAGS_txlog_rocksdb_cloud_log_purger_schedule + : config_reader.GetString( + "local", + "txlog_rocksdb_cloud_log_purger_schedule", + FLAGS_txlog_rocksdb_cloud_log_purger_schedule); + std::istringstream iss(log_purger_schedule); + iss >> std::get_time(&log_purger_tm, "%H:%M:%S"); + + if (iss.fail()) + { + LOG(ERROR) << "Invalid time format." << std::endl; + } + else + { + txlog_rocksdb_cloud_config.log_purger_starting_hour_ = + log_purger_tm.tm_hour; + txlog_rocksdb_cloud_config.log_purger_starting_minute_ = + log_purger_tm.tm_min; + txlog_rocksdb_cloud_config.log_purger_starting_second_ = + log_purger_tm.tm_sec; + } + if (core_config_.bootstrap) + { + log_server_ = std::make_unique<::txlog::LogServer>( + txlog_node_id, + log_server_port, + txlog_ips, + txlog_ports, + log_path, + 0, + txlog_group_replica_num, +#ifdef WITH_CLOUD_AZ_INFO + FLAGS_txlog_rocksdb_cloud_prefer_zone, + FLAGS_txlog_rocksdb_cloud_current_zone, +#endif + txlog_rocksdb_storage_path, + txlog_rocksdb_scan_threads, + txlog_rocksdb_cloud_config, + FLAGS_txlog_in_mem_data_log_queue_size_high_watermark, + txlog_rocksdb_max_write_buffer_number, + txlog_rocksdb_max_background_jobs, + txlog_rocksdb_target_file_size_base_val, + logserver_snapshot_interval); + } + else + { + log_server_ = std::make_unique<::txlog::LogServer>( + txlog_node_id, + log_server_port, + txlog_ips, + txlog_ports, + log_path, + 0, + txlog_group_replica_num, +#ifdef WITH_CLOUD_AZ_INFO + FLAGS_txlog_rocksdb_cloud_prefer_zone, + FLAGS_txlog_rocksdb_cloud_current_zone, +#endif + txlog_rocksdb_storage_path, + txlog_rocksdb_scan_threads, + txlog_rocksdb_cloud_config, + FLAGS_txlog_in_mem_data_log_queue_size_high_watermark, + txlog_rocksdb_max_write_buffer_number, + txlog_rocksdb_max_background_jobs, + txlog_rocksdb_target_file_size_base_val, + logserver_snapshot_interval, + enable_txlog_request_checkpoint, + check_replay_log_size_interval_sec, + notify_checkpointer_threshold_size_val); + } +#else + size_t txlog_rocksdb_sst_files_size_limit_val = + !CheckCommandLineFlagIsDefault( + "txlog_rocksdb_sst_files_size_limit") + ? txlog::parse_size( + FLAGS_txlog_rocksdb_sst_files_size_limit) + : txlog::parse_size(config_reader.GetString( + "local", + "txlog_rocksdb_sst_files_size_limit", + FLAGS_txlog_rocksdb_sst_files_size_limit)); + + // Start internal logserver. +#if defined(OPEN_LOG_SERVICE) + if (core_config_.bootstrap) + { + log_server_ = std::make_unique<::txlog::LogServer>( + txlog_node_id, + log_server_port, + log_path, + 1, + txlog_rocksdb_sst_files_size_limit_val, + txlog_rocksdb_max_write_buffer_number, + txlog_rocksdb_max_background_jobs, + txlog_rocksdb_target_file_size_base_val); + } + else + { + log_server_ = std::make_unique<::txlog::LogServer>( + txlog_node_id, + log_server_port, + log_path, + 1, + txlog_rocksdb_sst_files_size_limit_val, + txlog_rocksdb_max_write_buffer_number, + txlog_rocksdb_max_background_jobs, + txlog_rocksdb_target_file_size_base_val); + } +#else + if (core_config_.bootstrap) + { + log_server_ = std::make_unique<::txlog::LogServer>( + txlog_node_id, + log_server_port, + txlog_ips, + txlog_ports, + log_path, + 0, + txlog_group_replica_num, + txlog_rocksdb_storage_path, + txlog_rocksdb_scan_threads, + txlog_rocksdb_sst_files_size_limit_val, + txlog_rocksdb_max_write_buffer_number, + txlog_rocksdb_max_background_jobs, + txlog_rocksdb_target_file_size_base_val, + logserver_snapshot_interval); + } + else + { + log_server_ = std::make_unique<::txlog::LogServer>( + txlog_node_id, + log_server_port, + txlog_ips, + txlog_ports, + log_path, + 0, + txlog_group_replica_num, + txlog_rocksdb_storage_path, + txlog_rocksdb_scan_threads, + txlog_rocksdb_sst_files_size_limit_val, + txlog_rocksdb_max_write_buffer_number, + txlog_rocksdb_max_background_jobs, + txlog_rocksdb_target_file_size_base_val, + logserver_snapshot_interval, + enable_txlog_request_checkpoint, + check_replay_log_size_interval_sec, + notify_checkpointer_threshold_size_val); + } +#endif +#endif +#endif + DLOG(INFO) << "Log server started, node_id: " << txlog_node_id + << ", log_server_port: " << log_server_port + << ", txlog_group_replica_num: " + << txlog_group_replica_num << ", log_path: " << log_path; + int err = log_server_->Start(); + + if (err != 0) + { + LOG(ERROR) << "Failed to start the log service."; + return false; + } + } + } +#endif + return true; +} diff --git a/data_substrate/core/src/metrics_init.cpp b/data_substrate/core/src/metrics_init.cpp new file mode 100644 index 00000000000..20cdb15ce7f --- /dev/null +++ b/data_substrate/core/src/metrics_init.cpp @@ -0,0 +1,163 @@ +#include "data_substrate.h" +#include "metrics.h" +#include "glog/logging.h" +#include "gflags/gflags.h" +#include "metrics_registry_impl.h" +#include "tx_service_metrics.h" +#include "store/data_store_handler.h" +#include "INIReader.h" + +#if (WITH_LOG_SERVICE) +#include "log_service_metrics.h" +#endif +#ifdef ELOQ_MODULE_ELOQSQL +#include "mysql_metrics.h" +#endif + +DEFINE_bool(enable_metrics, false, "Enable metrics"); +DEFINE_string(metrics_port, "18081", "Metrics port"); + +bool DataSubstrate::InitializeMetrics(const INIReader &config_reader) +{ + /* Parse metrics config */ + metrics::enable_metrics= + config_reader.GetBoolean("metrics", "enable_metrics", false); + DLOG(INFO) << "enable_metrics: " << (metrics::enable_metrics ? "ON" : "OFF"); + if (metrics::enable_metrics) + { + FLAGS_metrics_port= std::to_string(config_reader.GetInteger( + "metrics", "metrics_port", std::stoi(FLAGS_metrics_port))); + + LOG(INFO) << "metrics_port: " << FLAGS_metrics_port; + + // global metrics + metrics::enable_memory_usage= + config_reader.GetBoolean("metrics", "enable_memory_usage", true); + LOG(INFO) << "enable_memory_usage: " + << (metrics::enable_memory_usage ? "ON" : "OFF"); + if (metrics::enable_memory_usage) + { + metrics::collect_memory_usage_round= config_reader.GetInteger( + "metrics", "collect_memory_usage_round", 10000); + LOG(INFO) << "collect memory usage every " + << metrics::collect_memory_usage_round << " round(s)"; + } + + metrics::enable_cache_hit_rate= + config_reader.GetBoolean("metrics", "enable_cache_hit_rate", true); + LOG(INFO) << "enable_cache_hit_rate: " + << (metrics::enable_cache_hit_rate ? "ON" : "OFF"); + + // tx metrics + metrics::enable_tx_metrics= + config_reader.GetBoolean("metrics", "enable_tx_metrics", true); + LOG(INFO) << "enable_tx_metrics: " + << (metrics::enable_tx_metrics ? "ON" : "OFF"); + if (metrics::enable_tx_metrics) + { + metrics::collect_tx_duration_round= + config_reader.GetInteger("metrics", "collect_tx_duration_round", + metrics::collect_tx_duration_round); + LOG(INFO) << "collect tx duration every " + << metrics::collect_tx_duration_round << " round(s)"; + } + + // busy round metrics + metrics::enable_busy_round_metrics= + config_reader.GetBoolean("metrics", "enable_busy_round_metrics", true); + LOG(INFO) << "enable_busy_round_metrics: " + << (metrics::enable_busy_round_metrics ? "ON" : "OFF"); + if (metrics::enable_busy_round_metrics) + { + metrics::busy_round_threshold= + config_reader.GetInteger("metrics", "busy_round_threshold", 10); + LOG(INFO) << "collect busy round metrics when reaching the " + "busy round " + "threshold " + << metrics::busy_round_threshold; + } + + // remote request metrics + metrics::enable_remote_request_metrics= config_reader.GetBoolean( + "metrics", "enable_busy_round_metrics", metrics::enable_tx_metrics); + LOG(INFO) << "enable_remote_request_metrics: " + << (metrics::enable_remote_request_metrics ? "ON" : "OFF"); + if (core_config_.enable_data_store) + { + metrics::enable_kv_metrics= + config_reader.GetBoolean("metrics", "enable_kv_metrics", true); + LOG(INFO) << "enable_kv_metrics: " + << (metrics::enable_kv_metrics ? "ON" : "OFF"); + } + +#if (WITH_LOG_SERVICE) + if (core_config_.enable_wal) + { + // log_service metrics + metrics::enable_log_service_metrics= config_reader.GetBoolean( + "metrics", "enable_log_service_metrics", true); + LOG(INFO) << "enable_log_service_metrics: " + << (metrics::enable_log_service_metrics ? "ON" : "OFF"); + } +#endif + + // failed forward msgs metrics + metrics::enable_standby_metrics= + config_reader.GetBoolean("metrics", "enable_standby_metrics", true); + LOG(INFO) << "enable standby metrics: " + << (metrics::enable_standby_metrics ? "ON" : "OFF"); + if (metrics::enable_standby_metrics) + { + metrics::collect_standby_metrics_round= config_reader.GetInteger( + "metrics", "collect_standby_metrics_round", 10000); + LOG(INFO) << "collect standby metrics every " + << metrics::collect_standby_metrics_round << " round(s)"; + } +#ifdef ELOQ_MODULE_ELOQSQL + metrics::enable_mysql_tx_metrics= + config_reader.GetBoolean("metrics", "enable_mysql_tx_metrics", true); + metrics::enable_mysql_dml_metrics= + config_reader.GetBoolean("metrics", "enable_mysql_dml_metrics", true); +#endif + setenv("ELOQ_METRICS_PORT", FLAGS_metrics_port.c_str(), false); + eloq_metrics_app::MetricsRegistryImpl::MetricsRegistryResult + metrics_registry_result= + eloq_metrics_app::MetricsRegistryImpl::GetRegistry(); + + if (metrics_registry_result.not_ok_ != nullptr) + { + LOG(ERROR) << "!!!!!!!! Failed to initialize MetricsRegristry !!!!!!!!"; + return false; + } + + metrics_registry_= std::move(metrics_registry_result.metrics_registry_); + + // TODO(liunyl): Finish all metrics registration before collect metrics. + // We might need to finish all registration before starting tx service. + if (core_config_.enable_data_store) + { + metrics::CommonLabels kv_common_common_labels{}; + kv_common_common_labels["node_ip"]= network_config_.local_ip; + kv_common_common_labels["node_port"]= + std::to_string(network_config_.local_port); + store_hd_->RegisterKvMetrics(metrics_registry_.get(), + kv_common_common_labels); + } + +#ifdef ELOQ_MODULE_ELOQSQL + metrics::CommonLabels mysql_common_labels{}; + mysql_common_labels["node_ip"]= network_config_.local_ip; + mysql_common_labels["node_port"]= std::to_string(network_config_.local_port); + metrics::register_mysql_metrics(metrics_registry_.get(), + mysql_common_labels); +#endif + + tx_service_common_labels_["node_ip"]= network_config_.local_ip; + tx_service_common_labels_["node_port"]= + std::to_string(network_config_.local_port); + tx_service_common_labels_["node_id"]= + std::to_string(network_config_.node_id); + } + + return true; +} \ No newline at end of file diff --git a/data_substrate/core/src/storage_init.cpp b/data_substrate/core/src/storage_init.cpp new file mode 100644 index 00000000000..f4a32fb894f --- /dev/null +++ b/data_substrate/core/src/storage_init.cpp @@ -0,0 +1,365 @@ +/** + * Copyright (C) 2025 EloqData Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under either of the following two licenses: + * 1. GNU Affero General Public License, version 3, as published by the Free + * Software Foundation. + * 2. GNU General Public License as published by the Free Software + * Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License or GNU General Public License for more + * details. + * + * You should have received a copy of the GNU Affero General Public License + * and GNU General Public License V2 along with this program. If not, see + * . + * + */ +#include + +#include "INIReader.h" +#include "data_store_handler.h" +#include "data_substrate.h" +#include "kv_store.h" +#if ELOQDS +#include + +#include "eloq_data_store_service/data_store_service.h" +#include "eloq_data_store_service/data_store_service_config.h" +#include "data_store_service_client.h" +#endif +#if defined(DATA_STORE_TYPE_ROCKSDB) +#include "store_handler/rocksdb_handler.h" +#endif + +#if defined(DATA_STORE_TYPE_ELOQDSS_ROCKSDB_CLOUD_S3) || \ + defined(DATA_STORE_TYPE_ELOQDSS_ROCKSDB_CLOUD_GCS) +#include "eloq_data_store_service/rocksdb_cloud_data_store_factory.h" +#elif defined(DATA_STORE_TYPE_ELOQDSS_ROCKSDB) +#include "eloq_data_store_service/rocksdb_data_store_factory.h" +#elif defined(DATA_STORE_TYPE_ELOQDSS_ELOQSTORE) +#include "eloq_data_store_service/eloq_store_data_store_factory.h" +#endif + +#ifdef DATA_STORE_TYPE_DYNAMODB +#include "store_handler/dynamo_handler.h" +#endif + +#ifdef DATA_STORE_TYPE_BIGTABLE +#include "store_handler/bigtable_handler.h" +#endif + +#if defined(DATA_STORE_TYPE_DYNAMODB) +DEFINE_string(dynamodb_endpoint, "", "Endpoint of KvStore Dynamodb"); +DEFINE_string(dynamodb_keyspace, "eloq", "KeySpace of Dynamodb KvStore"); +DEFINE_string(dynamodb_region, + "ap-northeast-1", + "Region of the used trable in DynamoDB"); +#endif + +#ifdef DATA_STORE_TYPE_BIGTABLE +DEFINE_string(bigtable_project_id, "", "Project id of BigTable"); +DEFINE_string(bigtable_instance_id, "", "Instance id of BigTable"); +DEFINE_string(bigtable_keyspace, "eloq", "KeySpace of BigTable"); +#endif +// aws_secret_key +#if defined(DATA_STORE_TYPE_ELOQDSS_ROCKSDB_CLOUD_S3) +DECLARE_string(aws_access_key_id); +DECLARE_string(aws_secret_key); +#elif defined(DATA_STORE_TYPE_DYNAMODB) || \ + (defined(USE_ROCKSDB_LOG_STATE) && defined(WITH_ROCKSDB_CLOUD)) +DEFINE_string(aws_access_key_id, "", "AWS SDK access key id"); +DEFINE_string(aws_secret_key, "", "AWS SDK secret key"); +#endif + +#if ELOQDS +DEFINE_string(eloq_dss_peer_node, + "", + "EloqDataStoreService peer node address. Used to fetch eloq-dss " + "topology from a working eloq-dss server."); +DEFINE_string(eloq_dss_branch_name, + "development", + "Branch name of EloqDataStore"); +DEFINE_string(eloq_dss_config_file_path, + "", + "EloqDataStoreService config file path. Used to load eloq-dss " + "config from a file."); +#endif + +DEFINE_uint32(snapshot_sync_worker_num, 0, "Snpashot sync worker num"); + +bool DataSubstrate::InitializeStorageHandler(const INIReader &config_reader) +{ + if (CheckCommandLineFlagIsDefault("snapshot_sync_worker_num") && + !config_reader.HasValue("store", "snapshot_sync_worker_num")) + { + FLAGS_snapshot_sync_worker_num = + std::max(core_config_.core_num / 4, static_cast(1)); + } +#if defined(DATA_STORE_TYPE_DYNAMODB) + std::string dynamodb_endpoint = + !CheckCommandLineFlagIsDefault("dynamodb_endpoint") + ? FLAGS_dynamodb_endpoint + : config_reader.GetString( + "store", "dynamodb_endpoint", FLAGS_dynamodb_endpoint); + std::string dynamodb_keyspace = + !CheckCommandLineFlagIsDefault("dynamodb_keyspace") + ? FLAGS_dynamodb_keyspace + : config_reader.GetString( + "store", "dynamodb_keyspace", FLAGS_dynamodb_keyspace); + std::string dynamodb_region = + !CheckCommandLineFlagIsDefault("dynamodb_region") + ? FLAGS_dynamodb_region + : config_reader.GetString( + "store", "dynamodb_region", FLAGS_dynamodb_region); + std::string aws_access_key_id = + !CheckCommandLineFlagIsDefault("aws_access_key_id") + ? FLAGS_aws_access_key_id + : config_reader.GetString( + "store", "aws_access_key_id", FLAGS_aws_access_key_id); + std::string aws_secret_key = + !CheckCommandLineFlagIsDefault("aws_secret_key") + ? FLAGS_aws_secret_key + : config_reader.GetString( + "store", "aws_secret_key", FLAGS_aws_secret_key); + bool ddl_skip_kv = false; + uint16_t worker_pool_size = core_num_ * 2; + + store_hd_ = std::make_unique(dynamodb_keyspace, + dynamodb_endpoint, + dynamodb_region, + aws_access_key_id, + aws_secret_key, + core_config_.bootstrap, + ddl_skip_kv, + worker_pool_size, + false); + +#elif defined(DATA_STORE_TYPE_BIGTABLE) + std::string bigtable_keyspace = + !CheckCommandLineFlagIsDefault("bigtable_keyspace") + ? FLAGS_bigtable_keyspace + : config_reader.GetString( + "store", "bigtable_keyspace", FLAGS_bigtable_keyspace); + std::string bigtable_project_id = + !CheckCommandLineFlagIsDefault("bigtable_project_id") + ? FLAGS_bigtable_project_id + : config_reader.GetString( + "store", "bigtable_project_id", FLAGS_bigtable_project_id); + std::string bigtable_instance_id = + !CheckCommandLineFlagIsDefault("bigtable_instance_id") + ? FLAGS_bigtable_instance_id + : config_reader.GetString( + "store", "bigtable_instance_id", FLAGS_bigtable_instance_id); + bool ddl_skip_kv = false; + store_hd_ = std::make_unique( + bigtable_keyspace, bigtable_project_id, bigtable_instance_id, core_config_.bootstrap, + ddl_skip_kv); + +#elif defined(DATA_STORE_TYPE_ROCKSDB) + bool is_single_node = + (network_config_.standby_ip_list.empty() && + network_config_.voter_ip_list.empty() && + network_config_.ip_list.find(',') == network_config_.ip_list.npos); + + EloqShare::RocksDBConfig rocksdb_config(config_reader, + core_config_.data_path); + + store_hd_ = std::make_unique( + rocksdb_config, + (core_config_.bootstrap || is_single_node), + core_config_.enable_cache_replacement); + +#elif ELOQDS + bool is_single_node = + (network_config_.standby_ip_list.empty() && + network_config_.voter_ip_list.empty() && + network_config_.ip_list.find(',') == network_config_.ip_list.npos); + + std::string eloq_dss_peer_node = + !CheckCommandLineFlagIsDefault("eloq_dss_peer_node") + ? FLAGS_eloq_dss_peer_node + : config_reader.GetString( + "store", "eloq_dss_peer_node", FLAGS_eloq_dss_peer_node); + + std::string eloq_dss_data_path = core_config_.data_path + "/eloq_dss"; + if (!std::filesystem::exists(eloq_dss_data_path)) + { + std::filesystem::create_directories(eloq_dss_data_path); + } + + std::string dss_config_file_path = !CheckCommandLineFlagIsDefault("eloq_dss_config_file_path") + ? FLAGS_eloq_dss_config_file_path + : config_reader.GetString( + "store", "eloq_dss_config_file_path", FLAGS_eloq_dss_config_file_path); + + if (dss_config_file_path.empty()) + { + dss_config_file_path = eloq_dss_data_path + "/dss_config.ini"; + } + + EloqDS::DataStoreServiceClusterManager ds_config; + if (std::filesystem::exists(dss_config_file_path)) + { + bool load_res = ds_config.Load(dss_config_file_path); + if (!load_res) + { + LOG(ERROR) << "Failed to load dss config file : " + << dss_config_file_path; + return false; + } + } + else + { + if (!eloq_dss_peer_node.empty()) + { + ds_config.SetThisNode(network_config_.local_ip, + network_config_.local_port + 7); + // Fetch ds topology from peer node + if (!EloqDS::DataStoreService::FetchConfigFromPeer( + eloq_dss_peer_node, ds_config)) + { + LOG(ERROR) << "Failed to fetch config from peer node: " + << eloq_dss_peer_node; + return false; + } + + // Save the fetched config to the local file + if (!ds_config.Save(dss_config_file_path)) + { + LOG(ERROR) << "Failed to save config to file: " + << dss_config_file_path; + return false; + } + } + else if (core_config_.bootstrap || is_single_node) + { + // Initialize the data store service config + ds_config.Initialize(network_config_.local_ip, + network_config_.local_port + 7); + if (!ds_config.Save(dss_config_file_path)) + { + LOG(ERROR) << "Failed to save config to file: " + << dss_config_file_path; + return false; + } + } + else + { + LOG(ERROR) << "Failed to load data store service config file: " + << dss_config_file_path; + return false; + } + } + +#if defined(DATA_STORE_TYPE_ELOQDSS_ROCKSDB_CLOUD_S3) || \ + defined(DATA_STORE_TYPE_ELOQDSS_ROCKSDB_CLOUD_GCS) + EloqDS::RocksDBConfig rocksdb_config(config_reader, eloq_dss_data_path); + EloqDS::RocksDBCloudConfig rocksdb_cloud_config(config_reader); + rocksdb_cloud_config.branch_name_ = FLAGS_eloq_dss_branch_name; + auto ds_factory = std::make_unique( + rocksdb_config, + rocksdb_cloud_config, + core_config_.enable_cache_replacement); + +#elif defined(DATA_STORE_TYPE_ELOQDSS_ROCKSDB) + EloqDS::RocksDBConfig rocksdb_config(config_reader, eloq_dss_data_path); + auto ds_factory = std::make_unique( + rocksdb_config, core_config_.enable_cache_replacement); + +#elif defined(DATA_STORE_TYPE_ELOQDSS_ELOQSTORE) + EloqDS::EloqStoreConfig eloq_store_config(config_reader, + eloq_dss_data_path); + auto ds_factory = std::make_unique( + std::move(eloq_store_config)); +#endif + + data_store_service_ = std::make_unique( + ds_config, + dss_config_file_path, + eloq_dss_data_path + "/DSMigrateLog", + std::move(ds_factory)); + std::vector dss_shards = ds_config.GetShardsForThisNode(); + std::unordered_map > + dss_shards_map; + // setup rocksdb cloud data store + for (int shard_id : dss_shards) + { +#if defined(DATA_STORE_TYPE_ELOQDSS_ROCKSDB_CLOUD_S3) || \ + defined(DATA_STORE_TYPE_ELOQDSS_ROCKSDB_CLOUD_GCS) + // TODO(lzx): move setup datastore to data_store_service + auto ds = std::make_unique( + rocksdb_cloud_config, + rocksdb_config, + (core_config_.bootstrap || is_single_node), + core_config_.enable_cache_replacement, + shard_id, + data_store_service_.get()); +#elif defined(DATA_STORE_TYPE_ELOQDSS_ROCKSDB) + // TODO(lzx): move setup datastore to data_store_service + auto ds = std::make_unique( + rocksdb_config, + (core_config_.bootstrap || is_single_node), + core_config_.enable_cache_replacement, + shard_id, + data_store_service_.get()); + +#elif defined(DATA_STORE_TYPE_ELOQDSS_ELOQSTORE) + auto ds = std::make_unique( + shard_id, data_store_service_.get()); +#endif + ds->Initialize(); + + // Start db if the shard status is not closed + if (ds_config.FetchDSShardStatus(shard_id) != + EloqDS::DSShardStatus::Closed) + { + bool ret = ds->StartDB(); + if (!ret) + { + LOG(ERROR) + << "Failed to start db instance in data store service"; + return false; + } + } + dss_shards_map[shard_id] = std::move(ds); + } + + // setup local data store service + bool ret = data_store_service_->StartService(); + if (!ret) + { + LOG(ERROR) << "Failed to start data store service"; + return false; + } + data_store_service_->ConnectDataStore(std::move(dss_shards_map)); + // setup data store service client + txservice::CatalogFactory *catalog_factory[NUM_EXTERNAL_ENGINES] = {nullptr, nullptr, nullptr}; + + for (size_t i = 0; i < NUM_EXTERNAL_ENGINES; i++) + { + catalog_factory[i] = engines_[i].catalog_factory; + } + + store_hd_ = std::make_unique( + catalog_factory, ds_config, data_store_service_.get()); + +#endif + + for (const auto &[table_name, _] : prebuilt_tables_) + { + store_hd_->AppendPreBuiltTable(table_name); + } + + if (!store_hd_->Connect()) + { + LOG(ERROR) << "!!!!!!!! Failed to connect to kvstore, startup is " + "terminated !!!!!!!!"; + return false; + } + return true; +} \ No newline at end of file diff --git a/data_substrate/core/src/tx_service_init.cpp b/data_substrate/core/src/tx_service_init.cpp new file mode 100644 index 00000000000..1dcae56c381 --- /dev/null +++ b/data_substrate/core/src/tx_service_init.cpp @@ -0,0 +1,253 @@ +#include +#include +#include +#include +#include + +#include "data_substrate.h" +#include "tx_service.h" +#include "eloq_log_wrapper.h" +#include "sequences/sequences.h" +DEFINE_int32(checkpointer_interval, 10, "Checkpointer interval in seconds"); +DEFINE_int32(node_memory_limit_mb, 8192, "Node memory limit in MB"); +DEFINE_int32(checkpointer_delay_seconds, 5, "Checkpointer delay in seconds"); +DEFINE_int32(collect_active_tx_ts_interval_seconds, 2, + "Active transaction timestamp collection interval"); +DEFINE_bool(kickout_data_for_test, false, "Kickout data for test"); +DEFINE_bool(enable_key_cache, false, "Enable key cache"); +DEFINE_uint32(max_standby_lag, 400000, + "txservice max msg lag between primary and standby"); +DEFINE_string(tx_service_data_path, "", "path for tx_service data"); +DEFINE_bool(fork_host_manager, true, "fork host manager process"); +DEFINE_string(hm_ip, "", "Host manager IP"); +DEFINE_int32(hm_port, 0, "Host manager port"); +DEFINE_string(hm_bin_path, "", + "host manager binary path if forking host manager process from " + "main process"); +DEFINE_uint32(deadlock_check_interval_seconds, 10, + "Deadlock check interval in seconds"); +DEFINE_bool(realtime_sampling, true, + "Whether enable realtime sampling. If disable it, user may need " + "to execute " + "analyze command at some time. Different from Innodb, Eloq never " + "analyze table automatically."); +DEFINE_uint32(range_split_worker_num, 0, "Range split worker number"); +DEFINE_bool(auto_redirect, false, "If redirect remote object command to remote node internally"); + +bool DataSubstrate::InitializeTxService(const INIReader &config_reader) +{ + uint64_t checkpointer_interval= + !CheckCommandLineFlagIsDefault("checkpointer_interval") + ? FLAGS_checkpointer_interval + : config_reader.GetInteger("local", "checkpointer_interval", + FLAGS_checkpointer_interval); + + uint64_t checkpointer_delay_seconds= + !CheckCommandLineFlagIsDefault("checkpointer_delay_seconds") + ? FLAGS_checkpointer_delay_seconds + : config_reader.GetInteger("local", "checkpointer_delay_seconds", + FLAGS_checkpointer_delay_seconds); + + uint64_t collect_active_tx_ts_interval_seconds= + !CheckCommandLineFlagIsDefault("collect_active_tx_ts_interval_seconds") + ? FLAGS_collect_active_tx_ts_interval_seconds + : config_reader.GetInteger( + "local", "collect_active_tx_ts_interval_seconds", + FLAGS_collect_active_tx_ts_interval_seconds); + + uint64_t max_standby_lag= + !CheckCommandLineFlagIsDefault("max_standby_lag") + ? FLAGS_max_standby_lag + : config_reader.GetInteger("local", "max_standby_lag", + FLAGS_max_standby_lag); + + bool kickout_data_for_test= + !CheckCommandLineFlagIsDefault("kickout_data_for_test") + ? FLAGS_kickout_data_for_test + : config_reader.GetBoolean("local", "kickout_data_for_test", + FLAGS_kickout_data_for_test); + + bool enable_key_cache= + !CheckCommandLineFlagIsDefault("enable_key_cache") + ? FLAGS_enable_key_cache + : config_reader.GetBoolean("local", "enable_key_cache", + FLAGS_enable_key_cache); + + std::string tx_service_data_path= + !CheckCommandLineFlagIsDefault("tx_service_data_path") + ? FLAGS_tx_service_data_path + : config_reader.GetString("local", "tx_service_data_path", + FLAGS_tx_service_data_path); + std::string tx_path("local://"); + if (tx_service_data_path.empty()) + { + tx_path.append(core_config_.data_path); + } + else + { + tx_path.append(tx_service_data_path); + } + + const char *field_mem= "node_memory_limit_mb"; + uint64_t node_memory_limit_mb= FLAGS_node_memory_limit_mb; + if (CheckCommandLineFlagIsDefault(field_mem)) + { + if (config_reader.HasValue("local", field_mem)) + { + node_memory_limit_mb= config_reader.GetInteger("local", field_mem, 0); + assert(node_memory_limit_mb); + } + else + { + struct sysinfo meminfo; + if (sysinfo(&meminfo)) + { + LOG(ERROR) << "config is missing: " << field_mem; + } + uint32_t mem_mib= + ((uint64_t) meminfo.totalram * meminfo.mem_unit) / (1024 * 1024); + node_memory_limit_mb= std::max(uint32_t(512), (mem_mib * 4) / 5); + LOG(INFO) << "config is automatically set: " << field_mem << "=" + << node_memory_limit_mb << "(MiB), total memory=" << mem_mib; + } + } + + uint64_t deadlock_check_interval_seconds= + !CheckCommandLineFlagIsDefault("deadlock_check_interval_seconds") + ? FLAGS_deadlock_check_interval_seconds + : config_reader.GetInteger("local", + "deadlock_check_interval_seconds", + FLAGS_deadlock_check_interval_seconds); + txservice::DeadLockCheck::SetTimeInterval(deadlock_check_interval_seconds); + + bool realtime_sampling= + !CheckCommandLineFlagIsDefault("realtime_sampling") + ? FLAGS_realtime_sampling + : config_reader.GetBoolean("local", "realtime_sampling", + FLAGS_realtime_sampling); + + uint64_t range_split_worker_num= + !CheckCommandLineFlagIsDefault("range_split_worker_num") + ? FLAGS_range_split_worker_num + : config_reader.GetInteger("local", "range_split_worker_num", + FLAGS_range_split_worker_num); + + bool auto_redirect = + !CheckCommandLineFlagIsDefault("auto_redirect") + ? FLAGS_auto_redirect + : config_reader.GetBoolean("local", "auto_redirect", + FLAGS_auto_redirect); + + bool fork_host_manager= + !CheckCommandLineFlagIsDefault("fork_host_manager") + ? FLAGS_fork_host_manager + : config_reader.GetBoolean("local", "fork_host_manager", + FLAGS_fork_host_manager); + + std::string hm_ip= !CheckCommandLineFlagIsDefault("hm_ip") + ? FLAGS_hm_ip + : config_reader.Get("local", "hm_ip", FLAGS_hm_ip); + + uint16_t hm_port= + !CheckCommandLineFlagIsDefault("hm_port") + ? FLAGS_hm_port + : config_reader.GetInteger("local", "hm_port", FLAGS_hm_port); + + std::string hm_bin_path= + !CheckCommandLineFlagIsDefault("hm_bin_path") + ? FLAGS_hm_bin_path + : config_reader.Get("local", "hm_bin_path", FLAGS_hm_bin_path); +#ifdef FORK_HM_PROCESS + if (hm_ip.empty()) + { + hm_ip= network_config_.local_ip; + } + if (hm_port == 0) + { + hm_port= network_config_.local_port + 4; + } + if (hm_bin_path.empty()) + { + char path_buf[PATH_MAX]; + ssize_t len= ::readlink("/proc/self/exe", path_buf, sizeof(path_buf)); + path_buf[len]= '\0'; + std::string s_path(path_buf); + std::string::size_type pos= s_path.find_last_of("/"); + std::string parent_path= s_path.substr(0, pos); + hm_bin_path= parent_path + "/host_manager"; + } +#endif + + std::map tx_service_conf{ + {"core_num", core_config_.core_num}, + {"checkpointer_interval", checkpointer_interval}, + {"node_memory_limit_mb", node_memory_limit_mb}, + {"checkpointer_delay_seconds", checkpointer_delay_seconds}, + {"collect_active_tx_ts_interval_seconds", + collect_active_tx_ts_interval_seconds}, + {"realtime_sampling", realtime_sampling ? 1 : 0}, + {"rep_group_cnt", network_config_.node_group_replica_num}, + {"range_split_worker_num", range_split_worker_num}, + {"enable_shard_heap_defragment", + core_config_.enable_heap_defragment ? 1 : 0}, + {"enable_key_cache", enable_key_cache}, + {"max_standby_lag", max_standby_lag}, + {"kickout_data_for_test", kickout_data_for_test ? 1 : 0}}; + + txservice::CatalogFactory *catalog_factory[NUM_EXTERNAL_ENGINES]= { + nullptr, nullptr, nullptr}; + + for (size_t i= 0; i < NUM_EXTERNAL_ENGINES; i++) + { + catalog_factory[i]= engines_[i].catalog_factory; + } + + auto log_agent= std::make_unique( + network_config_.node_group_replica_num); + + tx_service_= std::make_unique( + catalog_factory, system_handler_, tx_service_conf, + network_config_.node_id, network_config_.native_ng_id, + &network_config_.ng_configs, network_config_.cluster_config_version, + core_config_.enable_data_store ? store_hd_.get() : nullptr, + log_agent.get(), + core_config_.enable_mvcc, // enable_mvcc + !core_config_.enable_wal, // skip_wal + !core_config_.enable_data_store, // skip_kv + core_config_.enable_cache_replacement, // enable_cache_replacement + auto_redirect, // auto_redirect + metrics_registry_.get(), // metrics_registry + tx_service_common_labels_, // common_labels + &prebuilt_tables_ + // , + // publish_func, // TODO(liunyl) + // external_metrics + ); + + if (core_config_.enable_data_store) + { + store_hd_->SetTxService(tx_service_.get()); + } + + if (tx_service_->Start( + network_config_.node_id, network_config_.native_ng_id, + &network_config_.ng_configs, network_config_.cluster_config_version, + &log_service_config_.txlog_ips, &log_service_config_.txlog_ports, + &hm_ip, &hm_port, &hm_bin_path, tx_service_conf, + std::move(log_agent), tx_path, + network_config_.cluster_config_file_path, fork_host_manager) != 0) + { + LOG(ERROR) << "Failed to start tx service!!!!!"; + return false; + } + + txservice::Sequences::InitSequence(tx_service_.get(), store_hd_.get()); + + // tx_service is a distributed service, should wait for all the tx_service + // nodes to finish the log recovery process and setup the cc_stream_sender. + tx_service_->WaitClusterReady(); + // wait for the tx_service node to become the native group leader. + tx_service_->WaitNodeBecomeNativeGroupLeader(); + + return true; +} \ No newline at end of file diff --git a/storage/eloq/eloq_metrics b/data_substrate/eloq_metrics similarity index 100% rename from storage/eloq/eloq_metrics rename to data_substrate/eloq_metrics diff --git a/storage/eloq/log_service b/data_substrate/log_service similarity index 100% rename from storage/eloq/log_service rename to data_substrate/log_service diff --git a/data_substrate/store_handler b/data_substrate/store_handler new file mode 160000 index 00000000000..d64612f04ed --- /dev/null +++ b/data_substrate/store_handler @@ -0,0 +1 @@ +Subproject commit d64612f04ed7256fe97fa1558a4ca53372114a73 diff --git a/data_substrate/tx_service b/data_substrate/tx_service new file mode 160000 index 00000000000..734a724c359 --- /dev/null +++ b/data_substrate/tx_service @@ -0,0 +1 @@ +Subproject commit 734a724c3593a08a3e14c7e5882040f8e45f8894 diff --git a/scripts/mysql_install_db.sh b/scripts/mysql_install_db.sh index 7c39d2b2122..33028fded8a 100644 --- a/scripts/mysql_install_db.sh +++ b/scripts/mysql_install_db.sh @@ -540,11 +540,28 @@ fi mysqld_bootstrap="${MYSQLD_BOOTSTRAP-$mysqld}" mysqld_install_cmd_line() { - "$mysqld_bootstrap" $defaults $defaults_group_suffix "$mysqld_opt" --bootstrap $silent_startup\ - "--basedir=$basedir" "--datadir=$ldata" --log-warnings=0 --enforce-storage-engine="" \ - "--plugin-dir=${plugindir}" \ - $args --max_allowed_packet=8M \ - --net_buffer_length=16K + # Create temporary config file for bootstrap + local temp_config=$(mktemp) + + cat > "$temp_config" << EOF +[mysqld] +basedir=$basedir +datadir=$ldata +plugin-dir=$plugindir +log-warnings=0 +enforce-storage-engine= +max-allowed-packet=8M +net-buffer-length=16K +EOF + + # Call mysqld with config file and bootstrap flag + "$mysqld_bootstrap" --eloqsql_config="$temp_config" --bootstrap $args + local exit_code=$? + + # Clean up temp file + rm -f "$temp_config" + + return $exit_code } # Use $auth_root_socket_user if explicitly specified. diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt index efbd84edcf4..5d7402df1c8 100644 --- a/sql/CMakeLists.txt +++ b/sql/CMakeLists.txt @@ -310,11 +310,11 @@ MAYBE_DISABLE_IPO(sql) DTRACE_INSTRUMENT(sql) TARGET_LINK_LIBRARIES(sql ${SQL_DEP_LIBS}) -IF(PLUGIN_ELOQ) +# IF(PLUGIN_ELOQ) ADD_DEFINITIONS(-DWITH_ELOQ_STORAGE_ENGINE) SET(ELOQ_SOURCE_DIR ${CMAKE_SOURCE_DIR}/storage/eloq) set(SQL_INCLUDE_DIR ${SQL_INCLUDE_DIR} ${ELOQ_SOURCE_DIR}) -ENDIF() +# ENDIF() TARGET_INCLUDE_DIRECTORIES(sql PRIVATE ${SQL_INCLUDE_DIR}) diff --git a/sql/main.cc b/sql/main.cc index 2d919943249..eb52420eb52 100644 --- a/sql/main.cc +++ b/sql/main.cc @@ -19,6 +19,16 @@ Calls mysqld_main() entry point exported by sql library. On Windows, might do some service handling. */ +#include +#include + +// Only MySQL-related CLI flag we accept +DEFINE_string(eloqsql_config, "", + "Path to MySQL configuration file. All MySQL options must be in this config file."); + +DEFINE_string(data_substrate_config, "", + "Path to data substrate configuration file."); + #ifdef WITH_GLOG #include "glog_error_logging.h" #endif /* WITH_GLOG */ @@ -30,8 +40,23 @@ extern int mysqld_win_main(int argc, char **argv); extern int mysqld_main(int argc, char **argv); #endif +extern txservice::CatalogFactory *eloqsql_catalog_factory; + int main(int argc, char **argv) { + // Initialize gflags - will error on unknown flags + gflags::SetUsageMessage( + "mysqld [gflags options]\n\n" + "MySQL system variables must be in config file specified by --eloqsql_config.\n" + "Data substrate flags can be passed on command line.\n" + "Use --help to see all available flags."); + + + gflags::ParseCommandLineFlags(&argc, &argv, true); + + DataSubstrate::InitializeGlobal(FLAGS_data_substrate_config); + + #ifdef WITH_GLOG InitGoogleLogging(argv); #endif /* WITH_GLOG */ diff --git a/sql/mdl.h b/sql/mdl.h index c416307739f..92ada782018 100644 --- a/sql/mdl.h +++ b/sql/mdl.h @@ -23,7 +23,7 @@ #include #include -#include "../storage/eloq/tx_service/include/circular_queue.h" +#include "../data_substrate/tx_service/include/circular_queue.h" class THD; diff --git a/sql/mysql_metrics.h b/sql/mysql_metrics.h index 801be1ccd22..496f03c799e 100644 --- a/sql/mysql_metrics.h +++ b/sql/mysql_metrics.h @@ -1,7 +1,7 @@ #pragma once #include -#include "../storage/eloq/eloq_metrics/include/metrics.h" -#include "../storage/eloq/eloq_metrics/include/meter.h" +#include "../data_substrate/eloq_metrics/include/metrics.h" +#include "../data_substrate/eloq_metrics/include/meter.h" namespace metrics { diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 11b58b2352a..dce912a542e 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -16,6 +16,7 @@ #include "sql_plugin.h" // Includes mariadb.h #include "sql_priv.h" +#include #include "unireg.h" #include #ifndef _WIN32 @@ -638,6 +639,12 @@ struct system_variables global_system_variables; */ const char *current_dbug_option=""; +// Declare gflags +DECLARE_string(eloqsql_config); + +// Global string to hold defaults file argument +static std::string g_defaults_file_arg; + struct system_variables max_system_variables; struct system_status_var global_status_var; @@ -4032,6 +4039,8 @@ static int init_common_variables() SQLCOM_END + 10); #endif + // Process config file options (loaded by load_defaults_or_exit) + // Note: argc/argv from load_defaults only contains config file values if (get_options(&remaining_argc, &remaining_argv)) exit(1); if (IS_SYSVAR_AUTOSIZE(&server_version_ptr)) @@ -5545,11 +5554,29 @@ int mysqld_main(int argc, char **argv) orig_argc= argc; orig_argv= argv; my_defaults_mark_files= TRUE; - load_defaults_or_exit(MYSQL_CONFIG_NAME, load_default_groups, &argc, &argv); - defaults_argc= argc; - defaults_argv= argv; - remaining_argc= argc; - remaining_argv= argv; + + // Build argv for load_defaults + // If --eloqsql_config specified, pass as --defaults-file + char *load_default_argv[3]; + int load_default_argc = 1; + load_default_argv[0] = argv[0]; + + if (!FLAGS_eloqsql_config.empty()) + { + // Pass as --defaults-file=path + g_defaults_file_arg = std::string("--defaults-file=") + FLAGS_eloqsql_config; + load_default_argv[1] = const_cast(g_defaults_file_arg.c_str()); + load_default_argc = 2; + } + + char **load_default_argv_ptr = load_default_argv; + load_defaults_or_exit(MYSQL_CONFIG_NAME, load_default_groups, + &load_default_argc, &load_default_argv_ptr); + + defaults_argc= load_default_argc; + defaults_argv= load_default_argv_ptr; + remaining_argc= load_default_argc; + remaining_argv= load_default_argv_ptr; /* Must be initialized early for comparison of options name */ system_charset_info= &my_charset_utf8mb3_general_ci; @@ -5574,6 +5601,7 @@ int mysqld_main(int argc, char **argv) #endif /* WITH_PERFSCHEMA_STORAGE_ENGINE */ my_timer_init(&sys_timer_info); + // Process early options from config file int ho_error __attribute__((unused))= handle_early_options(); /* fix tdc_size */ diff --git a/sql/sql_class.h b/sql/sql_class.h index 754ec86ae30..058634ae253 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -51,7 +51,7 @@ #include "backup.h" #include "xa.h" #include "ddl_log.h" /* DDL_LOG_STATE */ -#include "../storage/eloq/eloq_metrics/include/metrics.h" +#include "../data_substrate/eloq_metrics/include/metrics.h" extern "C" void set_thd_stage_info(void *thd, diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index 262f49d9d5d..4e55b793cb3 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -103,7 +103,7 @@ #define FLAGSTR(V,F) ((V)&(F)?#F" ":"") -#include "eloq_db_dl.h" +#include "../storage/eloq/eloq_db_dl.h" #ifdef WITH_ARIA_STORAGE_ENGINE #include "../storage/maria/ha_maria.h" diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 727f0b5db39..54a5058812f 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -25,7 +25,7 @@ @{ */ -#include "eloq_errors.h" +#include "../storage/eloq/eloq_errors.h" #ifdef USE_PRAGMA_IMPLEMENTATION #pragma implementation // gcc: Class implementation #endif diff --git a/sql/threadpool_generic.h b/sql/threadpool_generic.h index de62974aa9f..193c4107472 100644 --- a/sql/threadpool_generic.h +++ b/sql/threadpool_generic.h @@ -29,7 +29,7 @@ #endif #ifdef COROUTINE_ENABLED -#include "../storage/eloq/tx_service/include/concurrent_queue_wsize.h" +#include "../data_substrate/tx_service/include/concurrent_queue_wsize.h" #include #include #include diff --git a/storage/eloq/CMakeLists.txt b/storage/eloq/CMakeLists.txt index 4b3643a3258..bb90f250b31 100644 --- a/storage/eloq/CMakeLists.txt +++ b/storage/eloq/CMakeLists.txt @@ -15,164 +15,141 @@ cmake_minimum_required(VERSION 3.8) project(eloq) -set(WITH_DATA_STORE "ELOQDSS_ROCKSDB" CACHE STRING "Which key-value storage to use") -set_property(CACHE WITH_DATA_STORE PROPERTY STRINGS "DYNAMODB" "BIGTABLE" "ELOQDSS_ROCKSDB_CLOUD_S3" "ELOQDSS_ELOQSTORE" "ELOQDSS_ROCKSDB") -message(NOTICE "WITH_DATA_STORE" : ${WITH_DATA_STORE}) +# set(WITH_DATA_STORE "ELOQDSS_ROCKSDB" CACHE STRING "Which key-value storage to use") +# set_property(CACHE WITH_DATA_STORE PROPERTY STRINGS "DYNAMODB" "BIGTABLE" "ELOQDSS_ROCKSDB_CLOUD_S3" "ELOQDSS_ELOQSTORE" "ELOQDSS_ROCKSDB") +# message(NOTICE "WITH_DATA_STORE" : ${WITH_DATA_STORE}) set(METRICS_LIB "eloq-metrics" CACHE STRING "metrics library name.") -if(WITH_DATA_STORE STREQUAL "DYNAMODB") - set(KV_STORAGE_VAL 1 CACHE STRING "dynamodb" FORCE) - add_compile_definitions(DATA_STORE_TYPE_DYNAMODB) - find_package(AWSSDK REQUIRED COMPONENTS dynamodb) -elseif(WITH_DATA_STORE STREQUAL "BIGTABLE") - set(KV_STORAGE_VAL 2 CACHE STRING "big table" FORCE) - add_compile_definitions(DATA_STORE_TYPE_BIGTABLE) - find_package(google_cloud_cpp_bigtable REQUIRED) -elseif(WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB_CLOUD_S3") - set(KV_STORAGE_VAL 3 CACHE STRING "eloq_ds" FORCE) - add_compile_definitions(DATA_STORE_TYPE_ELOQDSS_ROCKSDB_CLOUD_S3) -elseif(WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB") - set(KV_STORAGE_VAL 3 CACHE STRING "eloq_ds" FORCE) - add_compile_definitions(DATA_STORE_TYPE_ELOQDSS_ROCKSDB) -elseif(WITH_DATA_STORE STREQUAL "ELOQDSS_ELOQSTORE") - set(KV_STORAGE_VAL 3 CACHE STRING "eloq_ds" FORCE) - add_compile_definitions(DATA_STORE_TYPE_ELOQDSS_ELOQSTORE) - option(WITH_TXSERVICE "Whether compile eloqstore with txservice" ON) - set(ELOQSTORE_PARENT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/store_handler/eloq_data_store_service CACHE PATH "EloqStore parent directory") - INCLUDE(${CMAKE_CURRENT_SOURCE_DIR}/store_handler/eloq_data_store_service/build_eloq_store.cmake) -else() - message(FATAL_ERROR "Unset WITH_DATA_STORE") -endif() - -if (WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB") - find_path(ROCKSDB_INCLUDE_PATH NAMES rocksdb/db.h) - if (NOT ROCKSDB_INCLUDE_PATH) - message(FATAL_ERROR "Fail to find RocksDB include path") - endif() - message(STATUS "ROCKSDB_INCLUDE_PATH: ${ROCKSDB_INCLUDE_PATH}") - include_directories(${ROCKSDB_INCLUDE_PATH}) - - find_library(ROCKSDB_LIB NAMES rocksdb) - if (NOT ROCKSDB_LIB) - message(FATAL_ERROR "Fail to find RocksDB lib path") - endif() - message(STATUS "ROCKSDB_LIB: ${ROCKSDB_LIB}") - set(ROCKSDB_LIBRARIES ${ROCKSDB_LIBRARIES} ${ROCKSDB_LIB}) -endif() - -if ((WITH_DATA_STORE STREQUAL "ROCKSDB_CLOUD_S3") OR (WITH_DATA_STORE STREQUAL "ROCKSDB_CLOUD_GCS") - OR (WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB_CLOUD_S3") OR (WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB_CLOUD_GCS")) - if ((WITH_DATA_STORE STREQUAL "ROCKSDB_CLOUD_S3") OR (WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB_CLOUD_S3")) - find_path(AWS_CORE_INCLUDE_PATH aws/core/Aws.h) - if((NOT AWS_CORE_INCLUDE_PATH)) - message(FATAL_ERROR "Fail to find aws/core include path") - endif() - message(STATUS "aws/core include path: ${AWS_CORE_INCLUDE_PATH}") - - find_library(AWS_CORE_LIB aws-cpp-sdk-core) - if((NOT AWS_CORE_LIB )) - message(FATAL_ERROR "Fail to find aws-cpp-sdk-core lib") - endif() - message(STATUS "aws-cpp-sdk-core library: ${AWS_CORE_LIB}") - - find_path(AWS_KINESIS_INCLUDE_PATH aws/kinesis/KinesisClient.h) - if((NOT AWS_KINESIS_INCLUDE_PATH)) - message(FATAL_ERROR "Fail to find aws/kinesis include path") - endif() - message(STATUS "aws/kinesis include path: ${AWS_KINESIS_INCLUDE_PATH}") - - find_library(AWS_KINESIS_LIB aws-cpp-sdk-kinesis) - if((NOT AWS_KINESIS_LIB)) - message(FATAL_ERROR "Fail to find aws-cpp-sdk-kinesis lib") - endif() - message(STATUS "aws-cpp-sdk-kinesis library: ${AWS_KINESIS_LIB}") - - find_path(AWS_KINESIS_INCLUDE_PATH aws/kinesis/KinesisClient.h) - if((NOT AWS_KINESIS_INCLUDE_PATH)) - message(FATAL_ERROR "Fail to find aws/kinesis include path") - endif() - message(STATUS "aws/kinesis include path: ${AWS_KINESIS_INCLUDE_PATH}") - - find_library(AWS_KINESIS_LIB aws-cpp-sdk-kinesis) - if((NOT AWS_KINESIS_LIB)) - message(FATAL_ERROR "Fail to find aws-cpp-sdk-kinesis lib") - endif() - message(STATUS "aws-cpp-sdk-kinesis library: ${AWS_KINESIS_LIB}") - - find_path(AWS_S3_INCLUDE_PATH aws/s3/S3Client.h) - if((NOT AWS_S3_INCLUDE_PATH)) - message(FATAL_ERROR "Fail to find aws/s3 include path") - endif() - message(STATUS "aws/s3 include path: ${AWS_S3_INCLUDE_PATH}") - - find_library(AWS_S3_LIB aws-cpp-sdk-s3) - if((NOT AWS_S3_LIB )) - message(FATAL_ERROR "Fail to find aws-cpp-sdk-s3 lib") - endif() - message(STATUS "aws-cpp-sdk-s3 library: ${AWS_S3_LIB}") - - set(ROCKSDB_INCLUDE_PATH ${ROCKSDB_INCLUDE_PATH} ${AWS_CORE_INCLUDE_PATH}) - set(ROCKSDB_INCLUDE_PATH ${ROCKSDB_INCLUDE_PATH} ${AWS_KINESIS_INCLUDE_PATH}) - set(ROCKSDB_INCLUDE_PATH ${ROCKSDB_INCLUDE_PATH} ${AWS_S3_INCLUDE_PATH}) - - set(ROCKSDB_LIBRARIES ${ROCKSDB_LIBRARIES} ${AWS_CORE_LIB}) - set(ROCKSDB_LIBRARIES ${ROCKSDB_LIBRARIES} ${AWS_KINESIS_LIB}) - set(ROCKSDB_LIBRARIES ${ROCKSDB_LIBRARIES} ${AWS_S3_LIB}) - - find_library(ROCKSDB_CLOUD_LIB NAMES rocksdb-cloud-aws) - elseif ((WITH_DATA_STORE STREQUAL "ROCKSDB_CLOUD_GCS") OR (WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB_CLOUD_GCS")) - find_path(GCP_CS_INCLUDE_PATH google/cloud/storage/client.h) - if((NOT GCP_CS_INCLUDE_PATH)) - message(FATAL_ERROR "Fail to find google/cloud/storage include path") - endif() - message(STATUS "google/cloud/storage include path: ${GCP_CS_INCLUDE_PATH}") - - find_library(GCP_COMMON_LIB google_cloud_cpp_common) - if((NOT GCP_COMMON_LIB)) - message(FATAL_ERROR "Fail to find google_cloud_cpp_common lib") - endif() - message(STATUS "google_cloud_cpp_common library: ${GCP_COMMON_LIB}") - - find_library(GCP_CS_LIB google_cloud_cpp_storage) - if((NOT GCP_CS_LIB)) - message(FATAL_ERROR "Fail to find google_cloud_cpp_storage lib") - endif() - message(STATUS "google_cloud_cpp_storage library: ${GCP_CS_LIB}") - - set(ROCKSDB_LIBRARIES ${ROCKSDB_LIBRARIES} ${GCP_COMMON_LIB}) - set(ROCKSDB_LIBRARIES ${ROCKSDB_LIBRARIES} ${GCP_CS_LIB}) - - find_library(ROCKSDB_CLOUD_LIB NAMES rocksdb-cloud-gcp) - endif () - - # Common RocksDB Cloud setup (outside the S3/GCS specific blocks) - find_path(ROCKSDB_CLOUD_INCLUDE_PATH NAMES rocksdb/db.h PATH_SUFFIXES "rocksdb_cloud_header") - if (NOT ROCKSDB_CLOUD_INCLUDE_PATH) - message(FATAL_ERROR "Fail to find RocksDB Cloud include path") - endif () - message(STATUS "ROCKSDB_CLOUD_INCLUDE_PATH: ${ROCKSDB_CLOUD_INCLUDE_PATH}") - set(ROCKSDB_INCLUDE_PATH ${ROCKSDB_INCLUDE_PATH} ${ROCKSDB_CLOUD_INCLUDE_PATH}) - include_directories(${ROCKSDB_INCLUDE_PATH}) - - if (NOT ROCKSDB_CLOUD_LIB) - message(FATAL_ERROR "Fail to find RocksDB Cloud lib path") - endif () - message(STATUS "ROCKSDB_CLOUD_LIB: ${ROCKSDB_CLOUD_LIB}") - set(ROCKSDB_LIBRARIES ${ROCKSDB_LIBRARIES} ${ROCKSDB_CLOUD_LIB}) -endif() - -if ((WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB_CLOUD_S3") OR (WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB_CLOUD_GCS") OR (WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB") OR (WITH_DATA_STORE STREQUAL "ELOQDSS_ELOQSTORE")) - # compile proto file - set(DS_PROTO_DIR ${CMAKE_CURRENT_SOURCE_DIR}/store_handler/eloq_data_store_service) - message(NOTICE "data store service proto dir: ${DS_PROTO_DIR}") - set(PROTO_SRC ${DS_PROTO_DIR}) - set(PROTO_NAME ds_request) - execute_process( - COMMAND protoc ./${PROTO_NAME}.proto --cpp_out=./ - WORKING_DIRECTORY ${PROTO_SRC} - ) -endif() +# add_compile_definitions(ELOQ_MODULE_ELOQSQL) +# if (WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB") +# find_path(ROCKSDB_INCLUDE_PATH NAMES rocksdb/db.h) +# if (NOT ROCKSDB_INCLUDE_PATH) +# message(FATAL_ERROR "Fail to find RocksDB include path") +# endif() +# message(STATUS "ROCKSDB_INCLUDE_PATH: ${ROCKSDB_INCLUDE_PATH}") +# include_directories(${ROCKSDB_INCLUDE_PATH}) + +# find_library(ROCKSDB_LIB NAMES rocksdb) +# if (NOT ROCKSDB_LIB) +# message(FATAL_ERROR "Fail to find RocksDB lib path") +# endif() +# message(STATUS "ROCKSDB_LIB: ${ROCKSDB_LIB}") +# set(ROCKSDB_LIBRARIES ${ROCKSDB_LIBRARIES} ${ROCKSDB_LIB}) +# endif() + +# if ((WITH_DATA_STORE STREQUAL "ROCKSDB_CLOUD_S3") OR (WITH_DATA_STORE STREQUAL "ROCKSDB_CLOUD_GCS") +# OR (WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB_CLOUD_S3") OR (WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB_CLOUD_GCS")) +# if ((WITH_DATA_STORE STREQUAL "ROCKSDB_CLOUD_S3") OR (WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB_CLOUD_S3")) +# find_path(AWS_CORE_INCLUDE_PATH aws/core/Aws.h) +# if((NOT AWS_CORE_INCLUDE_PATH)) +# message(FATAL_ERROR "Fail to find aws/core include path") +# endif() +# message(STATUS "aws/core include path: ${AWS_CORE_INCLUDE_PATH}") + +# find_library(AWS_CORE_LIB aws-cpp-sdk-core) +# if((NOT AWS_CORE_LIB )) +# message(FATAL_ERROR "Fail to find aws-cpp-sdk-core lib") +# endif() +# message(STATUS "aws-cpp-sdk-core library: ${AWS_CORE_LIB}") + +# find_path(AWS_KINESIS_INCLUDE_PATH aws/kinesis/KinesisClient.h) +# if((NOT AWS_KINESIS_INCLUDE_PATH)) +# message(FATAL_ERROR "Fail to find aws/kinesis include path") +# endif() +# message(STATUS "aws/kinesis include path: ${AWS_KINESIS_INCLUDE_PATH}") + +# find_library(AWS_KINESIS_LIB aws-cpp-sdk-kinesis) +# if((NOT AWS_KINESIS_LIB)) +# message(FATAL_ERROR "Fail to find aws-cpp-sdk-kinesis lib") +# endif() +# message(STATUS "aws-cpp-sdk-kinesis library: ${AWS_KINESIS_LIB}") + +# find_path(AWS_KINESIS_INCLUDE_PATH aws/kinesis/KinesisClient.h) +# if((NOT AWS_KINESIS_INCLUDE_PATH)) +# message(FATAL_ERROR "Fail to find aws/kinesis include path") +# endif() +# message(STATUS "aws/kinesis include path: ${AWS_KINESIS_INCLUDE_PATH}") + +# find_library(AWS_KINESIS_LIB aws-cpp-sdk-kinesis) +# if((NOT AWS_KINESIS_LIB)) +# message(FATAL_ERROR "Fail to find aws-cpp-sdk-kinesis lib") +# endif() +# message(STATUS "aws-cpp-sdk-kinesis library: ${AWS_KINESIS_LIB}") + +# find_path(AWS_S3_INCLUDE_PATH aws/s3/S3Client.h) +# if((NOT AWS_S3_INCLUDE_PATH)) +# message(FATAL_ERROR "Fail to find aws/s3 include path") +# endif() +# message(STATUS "aws/s3 include path: ${AWS_S3_INCLUDE_PATH}") + +# find_library(AWS_S3_LIB aws-cpp-sdk-s3) +# if((NOT AWS_S3_LIB )) +# message(FATAL_ERROR "Fail to find aws-cpp-sdk-s3 lib") +# endif() +# message(STATUS "aws-cpp-sdk-s3 library: ${AWS_S3_LIB}") + +# set(ROCKSDB_INCLUDE_PATH ${ROCKSDB_INCLUDE_PATH} ${AWS_CORE_INCLUDE_PATH}) +# set(ROCKSDB_INCLUDE_PATH ${ROCKSDB_INCLUDE_PATH} ${AWS_KINESIS_INCLUDE_PATH}) +# set(ROCKSDB_INCLUDE_PATH ${ROCKSDB_INCLUDE_PATH} ${AWS_S3_INCLUDE_PATH}) + +# set(ROCKSDB_LIBRARIES ${ROCKSDB_LIBRARIES} ${AWS_CORE_LIB}) +# set(ROCKSDB_LIBRARIES ${ROCKSDB_LIBRARIES} ${AWS_KINESIS_LIB}) +# set(ROCKSDB_LIBRARIES ${ROCKSDB_LIBRARIES} ${AWS_S3_LIB}) + +# find_library(ROCKSDB_CLOUD_LIB NAMES rocksdb-cloud-aws) +# elseif ((WITH_DATA_STORE STREQUAL "ROCKSDB_CLOUD_GCS") OR (WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB_CLOUD_GCS")) +# find_path(GCP_CS_INCLUDE_PATH google/cloud/storage/client.h) +# if((NOT GCP_CS_INCLUDE_PATH)) +# message(FATAL_ERROR "Fail to find google/cloud/storage include path") +# endif() +# message(STATUS "google/cloud/storage include path: ${GCP_CS_INCLUDE_PATH}") + +# find_library(GCP_COMMON_LIB google_cloud_cpp_common) +# if((NOT GCP_COMMON_LIB)) +# message(FATAL_ERROR "Fail to find google_cloud_cpp_common lib") +# endif() +# message(STATUS "google_cloud_cpp_common library: ${GCP_COMMON_LIB}") + +# find_library(GCP_CS_LIB google_cloud_cpp_storage) +# if((NOT GCP_CS_LIB)) +# message(FATAL_ERROR "Fail to find google_cloud_cpp_storage lib") +# endif() +# message(STATUS "google_cloud_cpp_storage library: ${GCP_CS_LIB}") + +# set(ROCKSDB_LIBRARIES ${ROCKSDB_LIBRARIES} ${GCP_COMMON_LIB}) +# set(ROCKSDB_LIBRARIES ${ROCKSDB_LIBRARIES} ${GCP_CS_LIB}) + +# find_library(ROCKSDB_CLOUD_LIB NAMES rocksdb-cloud-gcp) +# endif () + +# # Common RocksDB Cloud setup (outside the S3/GCS specific blocks) +# find_path(ROCKSDB_CLOUD_INCLUDE_PATH NAMES rocksdb/db.h PATH_SUFFIXES "rocksdb_cloud_header") +# if (NOT ROCKSDB_CLOUD_INCLUDE_PATH) +# message(FATAL_ERROR "Fail to find RocksDB Cloud include path") +# endif () +# message(STATUS "ROCKSDB_CLOUD_INCLUDE_PATH: ${ROCKSDB_CLOUD_INCLUDE_PATH}") +# set(ROCKSDB_INCLUDE_PATH ${ROCKSDB_INCLUDE_PATH} ${ROCKSDB_CLOUD_INCLUDE_PATH}) +# include_directories(${ROCKSDB_INCLUDE_PATH}) + +# if (NOT ROCKSDB_CLOUD_LIB) +# message(FATAL_ERROR "Fail to find RocksDB Cloud lib path") +# endif () +# message(STATUS "ROCKSDB_CLOUD_LIB: ${ROCKSDB_CLOUD_LIB}") +# set(ROCKSDB_LIBRARIES ${ROCKSDB_LIBRARIES} ${ROCKSDB_CLOUD_LIB}) +# endif() + +# if ((WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB_CLOUD_S3") OR (WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB_CLOUD_GCS") OR (WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB") OR (WITH_DATA_STORE STREQUAL "ELOQDSS_ELOQSTORE")) +# # compile proto file +# set(DS_PROTO_DIR ${CMAKE_CURRENT_SOURCE_DIR}/store_handler/eloq_data_store_service) +# message(NOTICE "data store service proto dir: ${DS_PROTO_DIR}") +# set(PROTO_SRC ${DS_PROTO_DIR}) +# set(PROTO_NAME ds_request) +# execute_process( +# COMMAND protoc ./${PROTO_NAME}.proto --cpp_out=./ +# WORKING_DIRECTORY ${PROTO_SRC} +# ) +# endif() option(WITH_LOG_SERVICE "Compile with built-in log service." ON) @@ -180,6 +157,21 @@ option(WITH_LOG_SERVICE "Compile with built-in log service." ON) if(WITH_LOG_SERVICE) message("WITH_LOG_SERVICE: " ${WITH_LOG_SERVICE}) add_compile_definitions(WITH_LOG_SERVICE) +# Add compile flags for LOG STATE TYPE +if(WITH_LOG_STATE STREQUAL "MEMORY") + add_compile_definitions(LOG_STATE_TYPE_MEM) +elseif(WITH_LOG_STATE STREQUAL "ROCKSDB") + add_compile_definitions(LOG_STATE_TYPE_RKDB) + add_compile_definitions(LOG_STATE_TYPE_RKDB_ALL) +elseif(WITH_LOG_STATE STREQUAL "ROCKSDB_CLOUD_S3") + add_compile_definitions(LOG_STATE_TYPE_RKDB_S3) +elseif(WITH_LOG_STATE STREQUAL "ROCKSDB_CLOUD_GCS") + add_compile_definitions(LOG_STATE_TYPE_RKDB_GCS) +else() + message(FATAL_ERROR "Unknown WITH_LOG_STATE: ${WITH_LOG_STATE}") +endif() + + endif() option(OPEN_LOG_SERVICE "Compile with open log service." ON) @@ -188,38 +180,55 @@ if (OPEN_LOG_SERVICE) add_compile_definitions(OPEN_LOG_SERVICE=1) endif() -set(STATISTICS ON CACHE BOOL "Turn on STATISTICS") - -INCLUDE(build_tx_service.cmake) +# # Add log state type definition to match data_substrate configuration +# if(WITH_LOG_STATE STREQUAL "MEMORY") +# add_compile_definitions(LOG_STATE_TYPE_MEM) +# elseif(WITH_LOG_STATE STREQUAL "ROCKSDB") +# add_compile_definitions(LOG_STATE_TYPE_RKDB) +# elseif(WITH_LOG_STATE STREQUAL "ROCKSDB_CLOUD_S3") +# add_compile_definitions(LOG_STATE_TYPE_RKDB_S3) +# elseif(WITH_LOG_STATE STREQUAL "ROCKSDB_CLOUD_GCS") +# add_compile_definitions(LOG_STATE_TYPE_RKDB_GCS) +# endif() -if(WITH_LOG_SERVICE) - if (OPEN_LOG_SERVICE) - INCLUDE(build_log_service.cmake) - else() - INCLUDE(build_eloq_log_service.cmake) - endif() -endif() +set(STATISTICS ON CACHE BOOL "Turn on STATISTICS") -INCLUDE(build_eloq_metrics.cmake) +# Data substrate components now build separately in data_substrate/ directory +# INCLUDE(build_tx_service.cmake) +# +# if(WITH_LOG_SERVICE) +# if (OPEN_LOG_SERVICE) +# INCLUDE(build_log_service.cmake) +# else() +# INCLUDE(build_eloq_log_service.cmake) +# endif() +# endif() +# +# INCLUDE(build_eloq_metrics.cmake) find_package(MIMALLOC REQUIRED) include_directories(${MIMALLOC_INCLUDE_DIR}) -include_directories(${CMAKE_CURRENT_SOURCE_DIR}) -include_directories(${CMAKE_CURRENT_SOURCE_DIR}/tx_service/include) -include_directories(${CMAKE_CURRENT_SOURCE_DIR}/tx_service/include/cc) -include_directories(${CMAKE_CURRENT_SOURCE_DIR}/tx_service/include/remote) -include_directories(${CMAKE_CURRENT_SOURCE_DIR}/tx_service/include/fault) -include_directories(${CMAKE_CURRENT_SOURCE_DIR}/tx_service/include/store) -include_directories(${CMAKE_CURRENT_SOURCE_DIR}/tx_service/tx-log-protos) -include_directories(${CMAKE_CURRENT_SOURCE_DIR}/eloq_metrics/include) +include_directories( + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_SOURCE_DIR}/data_substrate/ + ${CMAKE_SOURCE_DIR}/data_substrate/core/include + ${CMAKE_SOURCE_DIR}/data_substrate/tx_service/include + ${CMAKE_SOURCE_DIR}/data_substrate/tx_service/include/cc + ${CMAKE_SOURCE_DIR}/data_substrate/tx_service/include/remote + ${CMAKE_SOURCE_DIR}/data_substrate/tx_service/include/fault + ${CMAKE_SOURCE_DIR}/data_substrate/tx_service/include/store + ${CMAKE_SOURCE_DIR}/data_substrate/tx_service/tx-log-protos + ${CMAKE_SOURCE_DIR}/data_substrate/eloq_metrics/include + ${CMAKE_SOURCE_DIR}/data_substrate/store_handler +) if(WITH_LOG_SERVICE) if (OPEN_LOG_SERVICE) - include_directories(${CMAKE_CURRENT_SOURCE_DIR}/log_service/include) - else () - include_directories(${CMAKE_CURRENT_SOURCE_DIR}/eloq_log_service/include) - endif () + include_directories(${CMAKE_SOURCE_DIR}/data_substrate/log_service/include) + else() + include_directories(${CMAKE_SOURCE_DIR}/data_substrate/eloq_log_service/include) + endif() endif() option(SMALL_RANGE "Whether enable small range" OFF) @@ -249,100 +258,38 @@ SET(MYELOQ_SOURCES slice.cc # eloq_tests.hpp - store_handler/kv_store.h + # store_handler/kv_store.h # Now in data_substrate library ) -if(WITH_DATA_STORE STREQUAL "DYNAMODB") - SET(MYELOQ_SOURCES ${MYELOQ_SOURCES} - store_handler/dynamo_handler.h - store_handler/dynamo_handler.cpp - store_handler/dynamo_scanner.h - store_handler/dynamo_scanner.cpp) - SET(MYELOQ_LIBRARY txservice ${AWSSDK_LINK_LIBRARIES}) -elseif(WITH_DATA_STORE STREQUAL "BIGTABLE") - SET(MYELOQ_SOURCES - ${MYELOQ_SOURCES} - store_handler/bigtable_handler.h - store_handler/bigtable_handler.cpp - store_handler/bigtable_scanner.h - store_handler/bigtable_scanner.cpp) - SET(MYELOQ_LIBRARY txservice google-cloud-cpp::bigtable) -elseif(WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB_CLOUD_S3" OR WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB") - SET(MYELOQ_SOURCES - ${MYELOQ_SOURCES} - store_handler/data_store_service_client.h - store_handler/data_store_service_client.cpp - store_handler/data_store_service_client_closure.h - store_handler/data_store_service_client_closure.cpp - store_handler/data_store_service_scanner.h - store_handler/data_store_service_scanner.cpp - store_handler/store_util.h - store_handler/store_util.cpp - - store_handler/eloq_data_store_service/ini.h - store_handler/eloq_data_store_service/ini.c - store_handler/eloq_data_store_service/INIReader.h - store_handler/eloq_data_store_service/INIReader.cpp - store_handler/eloq_data_store_service/thread_worker_pool.h - store_handler/eloq_data_store_service/thread_worker_pool.cpp - store_handler/eloq_data_store_service/data_store_service.h - store_handler/eloq_data_store_service/data_store_service.cpp - store_handler/eloq_data_store_service/data_store_fault_inject.h - store_handler/eloq_data_store_service/data_store_fault_inject.cpp - store_handler/eloq_data_store_service/data_store_service_config.h - store_handler/eloq_data_store_service/data_store_service_config.cpp - store_handler/eloq_data_store_service/ds_request.pb.h - store_handler/eloq_data_store_service/ds_request.pb.cc - store_handler/eloq_data_store_service/rocksdb_config.h - store_handler/eloq_data_store_service/rocksdb_config.cpp - store_handler/eloq_data_store_service/rocksdb_data_store_common.h - store_handler/eloq_data_store_service/rocksdb_data_store_common.cpp) - - if (WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB_CLOUD_S3") - SET(MYELOQ_SOURCES - ${MYELOQ_SOURCES} - store_handler/eloq_data_store_service/rocksdb_cloud_data_store.h - store_handler/eloq_data_store_service/rocksdb_cloud_data_store.cpp - store_handler/eloq_data_store_service/purger_event_listener.cpp - store_handler/eloq_data_store_service/purger_sliding_window.cpp - ) - elseif (WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB") - SET(MYELOQ_SOURCES - ${MYELOQ_SOURCES} - store_handler/eloq_data_store_service/rocksdb_data_store.h - store_handler/eloq_data_store_service/rocksdb_data_store.cpp - ) - endif() - SET(MYELOQ_LIBRARY ${MYELOQ_LIBRARY} txservice ${ROCKSDB_LIBRARIES}) - - include_directories(${CMAKE_CURRENT_SOURCE_DIR}/store_handler/eloq_data_store_service) -elseif(WITH_DATA_STORE STREQUAL "ELOQDSS_ELOQSTORE") - SET(MYELOQ_SOURCES - ${MYELOQ_SOURCES} - store_handler/data_store_service_client.cpp - store_handler/data_store_service_client_closure.cpp - store_handler/data_store_service_scanner.cpp - store_handler/store_util.cpp - store_handler/eloq_data_store_service/thread_worker_pool.cpp - store_handler/eloq_data_store_service/data_store_service.cpp - store_handler/eloq_data_store_service/data_store_fault_inject.cpp - store_handler/eloq_data_store_service/data_store_service_config.cpp - store_handler/eloq_data_store_service/eloq_store_data_store.cpp - store_handler/eloq_data_store_service/ds_request.pb.cc - store_handler/eloq_data_store_service/eloq_store_config.cpp - ) - SET(MYELOQ_LIBRARY ${MYELOQ_LIBRARY} txservice eloqstore) - - include_directories( - ${CMAKE_CURRENT_SOURCE_DIR}/store_handler - ${CMAKE_CURRENT_SOURCE_DIR}/store_handler/eloq_data_store_service) -else() - message(FATAL_ERROR "Unset WITH_DATA_STORE") -endif() - -if(WITH_LOG_SERVICE) - SET(MYELOQ_LIBRARY ${MYELOQ_LIBRARY} logservice) -endif() +# if(WITH_DATA_STORE STREQUAL "DYNAMODB") +# # SET(MYELOQ_SOURCES ${MYELOQ_SOURCES} +# # store_handler/dynamo_handler.h +# # store_handler/dynamo_handler.cpp +# # store_handler/dynamo_scanner.h +# # store_handler/dynamo_scanner.cpp) +# SET(MYELOQ_LIBRARY txservice ${AWSSDK_LINK_LIBRARIES}) +# elseif(WITH_DATA_STORE STREQUAL "BIGTABLE") +# # SET(MYELOQ_SOURCES +# # ${MYELOQ_SOURCES} +# # store_handler/bigtable_handler.h +# # store_handler/bigtable_handler.cpp +# # store_handler/bigtable_scanner.h +# # store_handler/bigtable_scanner.cpp) +# SET(MYELOQ_LIBRARY txservice google-cloud-cpp::bigtable) +# elseif(WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB_CLOUD_S3" OR WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB") +# # Store handler sources are now in data_substrate library +# SET(MYELOQ_LIBRARY ${MYELOQ_LIBRARY} ${ROCKSDB_LIBRARIES}) +# elseif(WITH_DATA_STORE STREQUAL "ELOQDSS_ELOQSTORE") +# # Store handler sources are now in data_substrate library +# SET(MYELOQ_LIBRARY ${MYELOQ_LIBRARY} eloqstore) +# else() +# message(FATAL_ERROR "Unset WITH_DATA_STORE") +# endif() + +# Log service is now in data_substrate library +# if(WITH_LOG_SERVICE) +# SET(MYELOQ_LIBRARY ${MYELOQ_LIBRARY} logservice) +# endif() # add_compile_definitions(WITH_DATA_STORE=${KV_STORAGE_VAL}) @@ -351,12 +298,13 @@ if(USE_ONE_DATA_STORE_SHARD_ENABLED) add_compile_definitions(USE_ONE_DATA_STORE_SHARD) endif() -SET(MYELOQ_SOURCES ${MYELOQ_SOURCES} metrics_registry_impl.h metrics_registry_impl.cpp) SET(MYELOQ_LIBRARY ${MYELOQ_LIBRARY} ${METRICS_LIB}) MYSQL_ADD_PLUGIN(eloq ${MYELOQ_SOURCES} STORAGE_ENGINE - MODULE_ONLY + #MODULE_ONLY + MANDATORY + STATIC_ONLY MODULE_OUTPUT_NAME ha_eloq - LINK_LIBRARIES ${MYELOQ_LIBRARY} + LINK_LIBRARIES data_substrate ${MYELOQ_LIBRARY} COMPONENT eloq-engine) diff --git a/storage/eloq/eloq_catalog_factory.h b/storage/eloq/eloq_catalog_factory.h index e55f91b870a..fa7ba37ed43 100644 --- a/storage/eloq/eloq_catalog_factory.h +++ b/storage/eloq/eloq_catalog_factory.h @@ -30,7 +30,7 @@ #include "tx_service/include/table_statistics.h" #include "tx_service/include/type.h" // TableName -extern std::unique_ptr storage_hd; +extern txservice::store::DataStoreHandler *storage_hd; namespace txservice { diff --git a/storage/eloq/eloq_i_s.cc b/storage/eloq/eloq_i_s.cc index 44799e19b49..cade0547ab5 100644 --- a/storage/eloq/eloq_i_s.cc +++ b/storage/eloq/eloq_i_s.cc @@ -31,7 +31,7 @@ using namespace MyEloq; using namespace txservice; extern std::unique_ptr tx_service; -extern std::unique_ptr storage_hd; +extern store::DataStoreHandler *storage_hd; extern std::pair *, const std::function *> thd_get_coro_functors(const THD *thd); diff --git a/storage/eloq/ha_eloq.cc b/storage/eloq/ha_eloq.cc index 71d8157e003..4b6f2ba6d26 100644 --- a/storage/eloq/ha_eloq.cc +++ b/storage/eloq/ha_eloq.cc @@ -112,8 +112,6 @@ */ #include "mysql/plugin.h" -#include "mysql_metrics.h" -#include "mysql_version.h" #include #include #ifdef USE_PRAGMA_IMPLEMENTATION @@ -127,10 +125,8 @@ #include #include #include -#include #include #include -#include #include "my_global.h" #include "thr_lock.h" /* THR_LOCK, THR_LOCK_DATA */ @@ -144,75 +140,23 @@ #include "key.h" #include "mysql/service_thd_wait.h" -#include "log_wrapper.h" #include "eloq_i_s.h" #include "eloq_catalog_factory.h" #include "eloq_catalog_name.h" #include "eloq_system_handler.h" #include "eloq_errors.h" #include "tx_util.h" +#include "data_substrate.h" // #include "eloq_tests.hpp" #include "ha_eloq.h" #include "tx_service/include/sequences/sequences.h" #include "slice.h" -#include "store_handler/kv_store.h" - -#if (defined(DATA_STORE_TYPE_ELOQDSS_ROCKSDB_CLOUD_S3) || \ - defined(DATA_STORE_TYPE_ELOQDSS_ROCKSDB_CLOUD_GCS) || \ - defined(DATA_STORE_TYPE_ELOQDSS_ROCKSDB) || \ - defined(DATA_STORE_TYPE_ELOQDSS_ELOQSTORE)) -#define ELOQDS 1 -#endif - -#if (defined(DATA_STORE_TYPE_ELOQDSS_ROCKSDB_CLOUD_S3) || \ - defined(DATA_STORE_TYPE_ELOQDSS_ROCKSDB_CLOUD_GCS)) -#define ELOQDS_RKDB_CLOUD 1 -#endif - -#if defined(DATA_STORE_TYPE_DYNAMODB) -#include "store_handler/dynamo_handler.h" -#elif defined(DATA_STORE_TYPE_BIGTABLE) -#include "store_handler/bigtable_handler.h" - -#elif ELOQDS -#include "store_handler/eloq_data_store_service/data_store_service.h" -#include "store_handler/eloq_data_store_service/data_store_service_config.h" -#include "store_handler/data_store_service_client.h" -#if ELOQDS_RKDB_CLOUD -#include "store_handler/eloq_data_store_service/rocksdb_cloud_data_store_factory.h" -#include "store_handler/eloq_data_store_service/rocksdb_config.h" -#elif defined(DATA_STORE_TYPE_ELOQDSS_ROCKSDB) -#include "store_handler/eloq_data_store_service/rocksdb_data_store_factory.h" -#include "store_handler/eloq_data_store_service/rocksdb_config.h" -#elif defined(DATA_STORE_TYPE_ELOQDSS_ELOQSTORE) -#include "store_handler/eloq_data_store_service/eloq_store_data_store_factory.h" -#endif -#else -#endif - -#include "metrics_registry_impl.h" - -#include "tx_service/include/constants.h" -#include "tx_service/include/statistics.h" -#include "tx_service/include/type.h" -#include "tx_service/include/tx_execution.h" -#include "tx_service/include/tx_service.h" -#include "tx_service/include/tx_service_metrics.h" -#include "tx_service/include/tx_request.h" -#include "tx_service/include/util.h" - -#if (WITH_LOG_SERVICE) -#include "log_server.h" -#include "log_service_metrics.h" -#include "log_utils.h" -#endif - // Log state type #if !defined(LOG_STATE_TYPE_RKDB_CLOUD) // Only if LOG_STATE_TYPE_RKDB_CLOUD undefined -#if ((defined(LOG_STATE_TYPE_RKDB_S3) || defined(LOG_STATE_TYPE_RKDB_GCS)) && \ +#if ((defined(LOG_STATE_TYPE_RKDB_S3) || defined(LOG_STATE_TYPE_RKDB_GCS)) && \ !defined(LOG_STATE_TYPE_RKDB)) #define LOG_STATE_TYPE_RKDB_CLOUD 1 #endif @@ -222,24 +166,26 @@ #if !defined(LOG_STATE_TYPE_RKDB_ALL) // Only if LOG_STATE_TYPE_RKDB_ALL undefined -#if (defined(LOG_STATE_TYPE_RKDB_S3) || defined(LOG_STATE_TYPE_RKDB_GCS) || \ +#if (defined(LOG_STATE_TYPE_RKDB_S3) || defined(LOG_STATE_TYPE_RKDB_GCS) || \ defined(LOG_STATE_TYPE_RKDB)) #define LOG_STATE_TYPE_RKDB_ALL 1 #endif #endif -#if defined(LOG_STATE_TYPE_RKDB_CLOUD) -#include "rocksdb_cloud_config.h" +#if defined(WITH_LOG_SERVICE) +#include "log_server.h" #endif +#include "store_handler/kv_store.h" - -// Don't put this include after sql_class.h include, it will cause compile -// error -#if (defined(DATA_STORE_TYPE_DYNAMODB) || defined(LOG_STATE_TYPE_RKDB_S3) || \ - defined(DATA_STORE_TYPE_ELOQDSS_ROCKSDB_CLOUD_S3)) -#include -#endif +#include "tx_service/include/constants.h" +#include "tx_service/include/statistics.h" +#include "tx_service/include/type.h" +#include "tx_service/include/tx_execution.h" +#include "tx_service/include/tx_service.h" +// #include "tx_service/include/tx_service_metrics.h" +#include "tx_service/include/tx_request.h" +#include "tx_service/include/util.h" #define DEFAULT_SCAN_TUPLE_SIZE 128 // eloq_debug_set prefix "+d,eloq;" @@ -252,6 +198,10 @@ using namespace MyEloq; using namespace txservice; +MariaCatalogFactory maria_catalog_factory; +txservice::CatalogFactory *eloqsql_catalog_factory= &maria_catalog_factory; +txservice::SystemHandler *eloqsql_system_handler= + &MyEloq::MariaSystemHandler::Instance(); extern my_bool opt_bootstrap; // Defined in Mariadb context. extern std::function terminate_hook; @@ -273,297 +223,20 @@ static handler *eloq_create_handler(handlerton *hton, TABLE_SHARE *table, handlerton *eloq_hton; -static ulong eloq_kv_storage= 0; -static char *eloq_local_ip= nullptr; -static char *eloq_ip_list= nullptr; -static char *eloq_standby_ip_list= nullptr; -static char *eloq_voter_ip_list= nullptr; -static char *eloq_hm_ip= nullptr; -static char *eloq_hm_bin_path= nullptr; -static char *eloq_cluster_config_file= nullptr; -static char *eloq_keyspace_name= nullptr; -static char *eloq_dynamodb_endpoint= nullptr; -static char *eloq_aws_access_key_id= nullptr; -static char *eloq_aws_secret_key= nullptr; -static char *eloq_dynamodb_region= nullptr; -static char *eloq_bigtable_project_id= nullptr; -static char *eloq_bigtable_instance_id= nullptr; static char *eloq_insert_semantic= nullptr; static char *eloq_auto_increment= nullptr; static char *eloq_invalidate_cache_once= nullptr; -static unsigned int eloq_core_num= 1; static ulong srv_enum_var= 0; static ulong srv_ulong_var= 0; static double srv_double_var= 0; -static unsigned int eloq_checkpointer_interval_sec= 10; -static unsigned int eloq_checkpointer_delay_sec= 5; -static unsigned int eloq_collect_active_tx_ts_interval_sec= 2; -// range split worker number -static unsigned int eloq_range_split_worker_num= 0; -static unsigned int eloq_bthread_worker_num= 0; -static unsigned int eloq_logserver_rocksdb_scan_thread_num= 0; -// eloq_realtime_sampling cannot be changed at runtime until now, because -// broadcast configuration has not been supported. -static my_bool eloq_realtime_sampling= true; -static my_bool eloq_ddl_skip_kv= false; -static my_bool eloq_skip_redo_log= false; -static my_bool eloq_scan_skip_kv= false; static my_bool eloq_random_scan_sort= false; -static my_bool eloq_use_key_cache= false; static my_bool eloq_report_debug_info= false; -// memory limit default 8GB. -static unsigned int eloq_node_memory_limit_mb= 8000; -// log limit default 16GB since rocksdb is used as default engine of log -// service, it's not sensitive to log size. But we still keep the logic of -// triggering checkpoint when log size reach the limit. -static unsigned int eloq_node_log_limit_mb= 16000; -static my_bool eloq_enable_mvcc= true; -static unsigned int eloq_metrics_port= 18081; -static unsigned int eloq_deadlock_interval_sec= 10; -// If not 0, will create a hook function that will close all braft connections -// when mysqld received crash single. static int eloq_signal_monitor= 0; -// tx log service list -static char *eloq_txlog_rocksdb_storage_path= nullptr; -static char *eloq_txlog_service_list= nullptr; -static unsigned int eloq_txlog_group_replica_num= 3; -static ulong eloq_partition_type= 0; - -// metrics collection -static std::unique_ptr metrics_registry= nullptr; -static my_bool eloq_enable_metrics= false; - -// mysql metrics -static my_bool eloq_enable_mysql_tx_metrics= true; -static my_bool eloq_enable_mysql_dml_metrics= true; - -// tx_service metrics -static my_bool eloq_enable_tx_metrics= true; -static my_bool eloq_enable_cache_hit_rate= true; -static my_bool eloq_enable_kv_metrics= true; -static my_bool eloq_enable_busy_round_metrics= true; -static my_bool eloq_enable_memory_usage= true; -static my_bool eloq_enable_remote_request_metrics= true; -static unsigned long long eloq_collect_memory_usage_round= 10000; -static unsigned long long eloq_collect_tx_duration_round= 1; -static unsigned long long eloq_busy_round_threshold= 10; - -// log_service metrics -static my_bool eloq_enable_log_service_metrics= false; - -// log server/rocksdb/cloud -static char *eloq_txlog_rocksdb_cloud_bucket_name= nullptr; -static char *eloq_txlog_rocksdb_cloud_bucket_prefix= nullptr; -static char *eloq_txlog_rocksdb_cloud_object_path= nullptr; -static char *eloq_txlog_rocksdb_cloud_region= nullptr; -static char *eloq_txlog_rocksdb_cloud_endpoint_url= nullptr; -static char *eloq_txlog_rocksdb_cloud_sst_file_cache_size= nullptr; -static int eloq_txlog_rocksdb_cloud_sst_file_cache_num_shard_bits= 5; -static char *eloq_txlog_rocksdb_target_file_size_base= nullptr; -static char *eloq_txlog_rocksdb_sst_files_size_limit= nullptr; -static unsigned int eloq_txlog_rocksdb_cloud_ready_timeout= 10; -static unsigned int eloq_txlog_rocksdb_cloud_file_deletion_delay= 3600; -static unsigned int eloq_node_group_replica_num= 3; -static unsigned int eloq_logserver_snapshot_interval= 600; -static unsigned int eloq_txlog_rocksdb_cloud_in_mem_log_size_high_watermark= - 50 * 10000; -static unsigned int eloq_txlog_rocksdb_max_write_buffer_number= 8; -static unsigned int eloq_txlog_rocksdb_max_background_jobs= 8; - -static my_bool eloq_enable_txlog_request_checkpoint= true; -static unsigned int eloq_check_replay_log_size_interval_sec= 10; -static char *eloq_notify_checkpointer_threshold_size= nullptr; - -// data_store_service -static char *eloq_dss_config_file_path= nullptr; -static char *eloq_dss_peer_node= nullptr; -static char *eloq_dss_branch_name= nullptr; -static char *eloq_dss_rocksdb_cloud_bucket_name= nullptr; -static char *eloq_dss_rocksdb_cloud_bucket_prefix= nullptr; -static char *eloq_dss_rocksdb_cloud_object_path= nullptr; -static char *eloq_dss_rocksdb_cloud_region= nullptr; -static char *eloq_dss_rocksdb_cloud_endpoint_url= nullptr; -static char *eloq_dss_rocksdb_cloud_sst_file_cache_size= nullptr; -static int eloq_dss_rocksdb_cloud_sst_file_cache_num_shard_bits= 5; -static char *eloq_dss_rocksdb_target_file_size_base= nullptr; -static unsigned int eloq_dss_rocksdb_cloud_purger_periodicity_secs= - 600; // 10 minutes -static unsigned int eloq_dss_rocksdb_cloud_file_deletion_delay= 3600; -static unsigned int eloq_dss_rocksdb_max_write_buffer_number= 8; -static unsigned int eloq_dss_rocksdb_max_background_jobs= 8; -#if defined(DATA_STORE_TYPE_ELOQDSS_ELOQSTORE) -static unsigned int eloq_eloqstore_worker_num= 1; -static char *eloq_eloqstore_data_path_list= nullptr; -static unsigned int eloq_eloqstore_open_files_limit= 1024; -static char *eloq_eloqstore_cloud_store_path= nullptr; -static unsigned int eloq_eloqstore_gc_threads= 1; -static unsigned int eloq_eloqstore_cloud_worker_count= 1; -static unsigned int eloq_eloqstore_data_page_restart_interval= 16; -static unsigned int eloq_eloqstore_index_page_restart_interval= 16; -static unsigned int eloq_eloqstore_init_page_count= 1 << 15; -static my_bool eloq_eloqstore_skip_verify_checksum= false; -static unsigned int eloq_eloqstore_index_buffer_pool_size= 1 << 15; -static unsigned int eloq_eloqstore_manifest_limit= 8 << 20; -static unsigned int eloq_eloqstore_io_queue_size= 4096; -static unsigned int eloq_eloqstore_max_inflight_write= 64 << 10; -static unsigned int eloq_eloqstore_max_write_batch_pages= 256; -static unsigned int eloq_eloqstore_buf_ring_size= 1 << 12; -static unsigned int eloq_eloqstore_coroutine_stack_size= 32 * 1024; -static unsigned int eloq_eloqstore_num_retained_archives= 0; -static unsigned int eloq_eloqstore_archive_interval_secs= 86400; -static unsigned int eloq_eloqstore_max_archive_tasks= 256; -static unsigned int eloq_eloqstore_file_amplify_factor= 4; -static unsigned long long eloq_eloqstore_local_space_limit= 1ULL << 40; -static unsigned int eloq_eloqstore_reserve_space_ratio= 100; -static unsigned int eloq_eloqstore_data_page_size= 1 << 12; -static unsigned int eloq_eloqstore_pages_per_file_shift= 11; -static unsigned int eloq_eloqstore_overflow_pointers= 16; -static my_bool eloq_eloqstore_data_append_mode= false; -#endif const char *enum_var_names[]= {"e1", "e2", NullS}; -const char *kv_storage_names[]= {"dynamo", "bigtable", "eloqds", NullS}; -const char *partition_names[]= {"Hash", "Range", NullS}; - -#define KV_DYNAMO 0 -#define KV_BIGTABLE 1 -#define KV_ELOQDS 2 - -static my_bool eloq_enable_heap_defragment= false; -static my_bool eloq_kickout_data_for_test= false; #define DIRTY_KEY_ID_BEGIN 16 -static MYSQL_SYSVAR_STR(local_ip, eloq_local_ip, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_MEMALLOC, - "IP address of the local node", nullptr, nullptr, - "127.0.0.1:8000"); - -static MYSQL_SYSVAR_STR(ip_list, eloq_ip_list, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_MEMALLOC, - "IP addresses of the nodes in the cluster", nullptr, - nullptr, "127.0.0.1:8000"); -static MYSQL_SYSVAR_STR(standby_ip_list, eloq_standby_ip_list, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_MEMALLOC, - "IP addresses of the standby nodes in the cluster", - nullptr, nullptr, ""); -static MYSQL_SYSVAR_STR(voter_ip_list, eloq_voter_ip_list, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_MEMALLOC, - "IP addresses of the voter nodes in the cluster", - nullptr, nullptr, ""); -static MYSQL_SYSVAR_STR(hm_ip, eloq_hm_ip, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_MEMALLOC, - "IP addresses of the host manager", nullptr, nullptr, - ""); - -static MYSQL_SYSVAR_STR(hm_bin_path, eloq_hm_bin_path, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_MEMALLOC, - "Path to host manager binary path.", nullptr, nullptr, - ""); - -static MYSQL_SYSVAR_STR(cluster_config_file, eloq_cluster_config_file, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_MEMALLOC, - "Path to cluster config file.", nullptr, nullptr, ""); - -static MYSQL_SYSVAR_STR(keyspace_name, eloq_keyspace_name, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_MEMALLOC, - "Keyspace of KV Storage", nullptr, nullptr, "mono"); -static MYSQL_SYSVAR_UINT(core_num, eloq_core_num, PLUGIN_VAR_RQCMDARG, - "Number of CPU cores", NULL, NULL, 1, 1, 1024, 1); - -static MYSQL_SYSVAR_UINT(range_split_worker_num, eloq_range_split_worker_num, - PLUGIN_VAR_RQCMDARG, "Number of range split worker", - NULL, NULL, 0, 0, 1024, 0); - -static MYSQL_SYSVAR_UINT(bthread_worker_num, eloq_bthread_worker_num, - PLUGIN_VAR_RQCMDARG, - "Number of bthread worker threads", NULL, NULL, 0, 0, - 1024, 0); - -static MYSQL_SYSVAR_UINT(logserver_rocksdb_scan_thread_num, - eloq_logserver_rocksdb_scan_thread_num, - PLUGIN_VAR_RQCMDARG, "Number of rocksdb scan threads", - NULL, NULL, 1, 1, 1024, 1); - -// global metrics -static MYSQL_SYSVAR_BOOL(enable_metrics, eloq_enable_metrics, - PLUGIN_VAR_RQCMDARG, - "When enabled, creates metric registry", NULL, NULL, - FALSE); -// mysql metrics -static MYSQL_SYSVAR_BOOL( - enable_mysql_tx_metrics, eloq_enable_mysql_tx_metrics, PLUGIN_VAR_RQCMDARG, - "Enables or disables the collection of transaction metrics in MySQL.", - NULL, NULL, TRUE); - -static MYSQL_SYSVAR_BOOL( - enable_mysql_dml_metrics, eloq_enable_mysql_dml_metrics, - PLUGIN_VAR_RQCMDARG, - "Enables or disables the collection of DML metrics in MySQL. When set to " - "ON, the plugin will track DML operation metrics such as SELECT, INSERT, " - "UPDATE, DELETE, etc.", - NULL, NULL, TRUE); - -// tx_service metrics -static MYSQL_SYSVAR_BOOL( - enable_tx_metrics, eloq_enable_tx_metrics, PLUGIN_VAR_RQCMDARG, - "Enable or disable transaction metrics for `tx_service` side.", NULL, NULL, - TRUE); - -static MYSQL_SYSVAR_BOOL(enable_cache_hit_rate, eloq_enable_cache_hit_rate, - PLUGIN_VAR_RQCMDARG, - "Enable or disable cache hit rate metrics", NULL, - NULL, TRUE); - -static MYSQL_SYSVAR_BOOL(enable_busy_round_metrics, - eloq_enable_busy_round_metrics, PLUGIN_VAR_RQCMDARG, - "When enabled, collects process transaction requests " - "latency, and process cc " - "requests latency, cc queue length", - NULL, NULL, TRUE); - -static MYSQL_SYSVAR_BOOL(enable_memory_usage, eloq_enable_memory_usage, - PLUGIN_VAR_RQCMDARG, - "Enable or disable memory usage metrics", NULL, NULL, - TRUE); - -static MYSQL_SYSVAR_BOOL( - enable_remote_request_metrics, eloq_enable_remote_request_metrics, - PLUGIN_VAR_RQCMDARG, - "Enable or disable remote request metrics for `tx_service` side.", NULL, - NULL, TRUE); - -static MYSQL_SYSVAR_BOOL( - enable_kv_metrics, eloq_enable_kv_metrics, PLUGIN_VAR_RQCMDARG, - "Enable or disable KV store metrics for `tx_service` side.", NULL, NULL, - TRUE); - -static MYSQL_SYSVAR_ULONGLONG( - busy_round_threshold, eloq_busy_round_threshold, PLUGIN_VAR_RQCMDARG, - "If CC queue length >= `threshold` (default: 10), then the " - "RunOneRound is busy", - NULL, NULL, 10, 1, ULLONG_MAX, 1); - -static MYSQL_SYSVAR_ULONGLONG( - collect_memory_usage_round, eloq_collect_memory_usage_round, - PLUGIN_VAR_RQCMDARG, - "Interval of collecting memory usage. Only " - "works when enable memory usage (default: 10000)", - NULL, NULL, 10000, 0, ULLONG_MAX, 1); - -static MYSQL_SYSVAR_ULONGLONG( - collect_tx_duration_round, eloq_collect_tx_duration_round, - PLUGIN_VAR_RQCMDARG, - "Interval of collecting transaction duration. Only " - "works when enable tx metrics (default: 1)", - NULL, NULL, 1, 0, ULLONG_MAX, 1); - -// log_service metrics -static MYSQL_SYSVAR_BOOL(enable_log_service_metrics, - eloq_enable_log_service_metrics, PLUGIN_VAR_RQCMDARG, - "When enabled, collects log_service metrics.", NULL, - NULL, FALSE); - const char *cc_protocol_names[]= {"OCC", "OccRead", "Locking", NullS}; TYPELIB cc_protocol_typelib= {array_elements(cc_protocol_names) - 1, @@ -592,43 +265,6 @@ static MYSQL_SYSVAR_ENUM(enum_var, // name 0, // def &enum_var_typelib); // typelib -TYPELIB kv_storage_typelib= {array_elements(kv_storage_names) - 1, - "kv_storage_typelib", kv_storage_names, NULL}; - -static MYSQL_SYSVAR_ENUM(kv_storage, // name - eloq_kv_storage, // varname - PLUGIN_VAR_RQCMDARG, // opt - "Supported Key-Value storage", // comment - NULL, // check - NULL, // update - 0, // def - &kv_storage_typelib); // typelib - -static MYSQL_SYSVAR_STR(aws_access_key_id, eloq_aws_access_key_id, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_MEMALLOC, - "aws sdk access key id", nullptr, nullptr, ""); - -static MYSQL_SYSVAR_STR(aws_secret_key, eloq_aws_secret_key, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_MEMALLOC, - "AWS SDK secret key", nullptr, nullptr, ""); - -static MYSQL_SYSVAR_STR(dynamodb_endpoint, eloq_dynamodb_endpoint, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_MEMALLOC, - "Endpoint override of DynamoDB", nullptr, nullptr, ""); - -static MYSQL_SYSVAR_STR(dynamodb_region, eloq_dynamodb_region, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_MEMALLOC, - "Region of the used trable in DynamoDB", nullptr, - nullptr, "ap-northeast-1"); - -static MYSQL_SYSVAR_STR(bigtable_project_id, eloq_bigtable_project_id, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_MEMALLOC, - "Project id of BigTable", nullptr, nullptr, ""); - -static MYSQL_SYSVAR_STR(bigtable_instance_id, eloq_bigtable_instance_id, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_MEMALLOC, - "Instance id of BigTable", nullptr, nullptr, ""); - static MYSQL_SYSVAR_STR(insert_semantic, eloq_insert_semantic, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_MEMALLOC, "Insert semantic: insert or upsert", nullptr, nullptr, @@ -653,89 +289,10 @@ static MYSQL_THDVAR_ULONG(varopt_default, PLUGIN_VAR_RQCMDARG, "default value of the VAROPT table option", NULL, NULL, 5, 0, 100, 0); -static MYSQL_SYSVAR_UINT(checkpointer_interval_sec, - eloq_checkpointer_interval_sec, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, - "Interval of checkpointer(s)", NULL, NULL, 10, 1, - 86400, 1); - -static MYSQL_SYSVAR_UINT( - checkpointer_delay_sec, eloq_checkpointer_delay_sec, PLUGIN_VAR_RQCMDARG, - "The time(second) which ckpt_ts is less than min lock ts", NULL, NULL, 5, - 0, 86400, 1); - -static MYSQL_SYSVAR_UINT(collect_active_tx_ts_interval_sec, - eloq_collect_active_tx_ts_interval_sec, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, - "Interval of collect active tx start timestamp(s)", - NULL, NULL, 2, 1, 86400, 1); -static MYSQL_SYSVAR_BOOL( - realtime_sampling, eloq_realtime_sampling, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, - "Whether enable realtime sampling. If disable it, user may need execute " - "analyze command at some time. Different from Innodb, Eloq never " - "analyze table automatically.", - NULL, NULL, TRUE); - -static MYSQL_SYSVAR_BOOL( - ddl_skip_kv, eloq_ddl_skip_kv, PLUGIN_VAR_NOCMDARG | PLUGIN_VAR_READONLY, - "Skip create or drop table on kv store, only used to speed up test case", - NULL, NULL, FALSE); -static MYSQL_SYSVAR_BOOL(skip_redo_log, eloq_skip_redo_log, - PLUGIN_VAR_NOCMDARG | PLUGIN_VAR_READONLY, - "Skip write redo log in tx_service", NULL, NULL, - FALSE); -static MYSQL_SYSVAR_BOOL( - use_key_cache, eloq_use_key_cache, - PLUGIN_VAR_NOCMDARG | PLUGIN_VAR_READONLY, - "Use key cache in primary key to avoid kv read if key does not exists.", - NULL, NULL, FALSE); static MYSQL_SYSVAR_BOOL(report_debug_info, eloq_report_debug_info, PLUGIN_VAR_NOCMDARG, "When enabled, report debug information to client", NULL, NULL, FALSE); -static MYSQL_SYSVAR_UINT(node_memory_limit_mb, eloq_node_memory_limit_mb, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, - "memory limit per node (MB)", NULL, NULL, 8000, 1, - 1000000, 1); -static MYSQL_SYSVAR_UINT(node_log_limit_mb, eloq_node_log_limit_mb, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, - "log limit per node (MB)", NULL, NULL, 16000, 1, - 1000000, 1); - -static MYSQL_SYSVAR_UINT( - metrics_port, // name - eloq_metrics_port, // varname - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, // opt - "The port on which the metrics_collector is " // comment - "reported. Default is 18081", // comment - NULL, // check - NULL, // update - 18081, // def - 8080, 65535, 1); - -static MYSQL_SYSVAR_BOOL( - enable_mvcc, eloq_enable_mvcc, PLUGIN_VAR_NOCMDARG, - "When enabled, use muliti-versions. Repeatable Read " - "isolation level will be converted to Snapshot isolation level", - NULL, NULL, TRUE); - -TYPELIB partition_typelib= {array_elements(partition_names) - 1, - "partition_typelib", partition_names, NULL}; - -static MYSQL_SYSVAR_ENUM(partition_type, // name - eloq_partition_type, // varname - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, // opt - "Partition type (Hash|Range)", // comment - NULL, // check - NULL, // update - 0, // default(Hash) - &partition_typelib); // typelib - -static MYSQL_SYSVAR_BOOL(enable_heap_defragment, eloq_enable_heap_defragment, - PLUGIN_VAR_NOCMDARG, - "When enabled, report debug information to client", - NULL, NULL, FALSE); int auto_increment_var_check(MYSQL_THD thd, struct st_mysql_sys_var *var, void *save, struct st_mysql_value *value) @@ -767,10 +324,6 @@ static MYSQL_SYSVAR_STR(auto_increment, eloq_auto_increment, "auto increment parameters", auto_increment_var_check, NULL, ""); -static MYSQL_SYSVAR_BOOL(kickout_data_for_test, eloq_kickout_data_for_test, - PLUGIN_VAR_NOCMDARG, "Clean data for test", NULL, - NULL, FALSE); - static int invalidate_table_cache(MYSQL_THD thd, const std::vector &invalidate_tables); @@ -842,479 +395,24 @@ static MYSQL_SYSVAR_STR( "Evict caches for given tables. Mainly used after physical importing.", invalidate_cache_once_var_check, NULL, ""); -static MYSQL_SYSVAR_BOOL( - scan_skip_kv, eloq_scan_skip_kv, PLUGIN_VAR_NOCMDARG | PLUGIN_VAR_READONLY, - "Skip access kv store for scan request, only used to speed up test case", - NULL, NULL, FALSE); - static MYSQL_SYSVAR_BOOL(random_scan_sort, eloq_random_scan_sort, PLUGIN_VAR_OPCMDARG, "Sort output result when executing random scan, only " "used to run test case", NULL, NULL, FALSE); -static MYSQL_SYSVAR_UINT(deadlock_interval_sec, eloq_deadlock_interval_sec, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, - "Interval of dead lock check(s)", NULL, NULL, 300, 1, - 3600, 1); -static MYSQL_SYSVAR_STR(txlog_rocksdb_storage_path, - eloq_txlog_rocksdb_storage_path, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_MEMALLOC, - "The path for tx log service rocksdb storage", nullptr, - nullptr, ""); -static MYSQL_SYSVAR_STR(txlog_service_list, eloq_txlog_service_list, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_MEMALLOC, - "Ip address of the tx log service node", nullptr, - nullptr, ""); - -static MYSQL_SYSVAR_UINT(txlog_group_replica_num, eloq_txlog_group_replica_num, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, - "Replicate number of tx log group", NULL, NULL, 3, 1, - 10, 1); - static MYSQL_SYSVAR_INT(signal_monitor, eloq_signal_monitor, PLUGIN_VAR_RQCMDARG, "Monitor mysql crash signal", nullptr, nullptr, 0, 0, INT_MAX, 0); -static MYSQL_SYSVAR_STR(txlog_rocksdb_cloud_bucket_name, - eloq_txlog_rocksdb_cloud_bucket_name, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_MEMALLOC, - "TxLog RocksDB cloud bucket name", NULL, NULL, ""); -static MYSQL_SYSVAR_STR(txlog_rocksdb_cloud_bucket_prefix, - eloq_txlog_rocksdb_cloud_bucket_prefix, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_MEMALLOC, - "TxLog RocksDB cloud bucket prefix", NULL, NULL, ""); -static MYSQL_SYSVAR_STR(txlog_rocksdb_cloud_object_path, - eloq_txlog_rocksdb_cloud_object_path, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_MEMALLOC, - "TxLog RocksDB cloud object path", NULL, NULL, - "rocksdb_cloud"); -static MYSQL_SYSVAR_STR(txlog_rocksdb_cloud_region, - eloq_txlog_rocksdb_cloud_region, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_MEMALLOC, - "TxLog RocksDB cloud region", NULL, NULL, ""); -static MYSQL_SYSVAR_STR(txlog_rocksdb_cloud_endpoint_url, - eloq_txlog_rocksdb_cloud_endpoint_url, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_MEMALLOC, - "TxLog RocksDB Cloud endpoint URL", NULL, NULL, ""); -static MYSQL_SYSVAR_STR(txlog_rocksdb_cloud_sst_file_cache_size, - eloq_txlog_rocksdb_cloud_sst_file_cache_size, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_MEMALLOC, - "TxLog RocksDB Cloud SST file cache size", NULL, NULL, - "10GB"); -static MYSQL_SYSVAR_INT(txlog_rocksdb_cloud_sst_file_cache_num_shard_bits, - eloq_txlog_rocksdb_cloud_sst_file_cache_num_shard_bits, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_MEMALLOC, - "TxLog RocksDB Cloud SST file cache num shard bits", - NULL, NULL, 5, 0, 30, 1); -static MYSQL_SYSVAR_STR(txlog_rocksdb_target_file_size_base, - eloq_txlog_rocksdb_target_file_size_base, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_MEMALLOC, - "TxLog RocksDB target file size", NULL, NULL, "64MB"); -static MYSQL_SYSVAR_STR(txlog_rocksdb_sst_files_size_limit, - eloq_txlog_rocksdb_sst_files_size_limit, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_MEMALLOC, - "TxLog RocksDB sst files size limit", NULL, NULL, - "500MB"); -static MYSQL_SYSVAR_UINT(txlog_rocksdb_cloud_ready_timeout, - eloq_txlog_rocksdb_cloud_ready_timeout, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_MEMALLOC, - "TxLog RocksDB Cloud becomes ready timeout(seconds)", - NULL, NULL, 10, 1, 120, 1); -static MYSQL_SYSVAR_UINT(txlog_rocksdb_cloud_file_deletion_delay, - eloq_txlog_rocksdb_cloud_file_deletion_delay, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_MEMALLOC, - "TxLog RocksDB Cloud becomes ready timeout", NULL, - NULL, 60, 1, 3600, 1); -static MYSQL_SYSVAR_UINT(node_group_replica_num, eloq_node_group_replica_num, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, - "Replicate number of node group(Max: 9)", NULL, NULL, - 3, 1, 9, 1); -static MYSQL_SYSVAR_UINT(logserver_snapshot_interval, - eloq_logserver_snapshot_interval, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, - "Logserver log state snapshot interval", NULL, NULL, - 600, 10, 7200, 1); -static MYSQL_SYSVAR_UINT( - txlog_rocksdb_cloud_in_mem_log_high_watermark, - eloq_txlog_rocksdb_cloud_in_mem_log_size_high_watermark, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, - "TxLog RocksDB Cloud in memory log queue high watermark", NULL, NULL, - 50 * 10000, 10000, 1000 * 10000, 1); -static MYSQL_SYSVAR_UINT(txlog_rocksdb_max_write_buffer_number, - eloq_txlog_rocksdb_max_write_buffer_number, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, - "TxLog RocksDB max write buffer number", NULL, NULL, - 8, 4, 100, 1); -static MYSQL_SYSVAR_UINT(txlog_rocksdb_max_background_jobs, - eloq_txlog_rocksdb_max_background_jobs, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, - "TxLog RocksDB max background jobs", NULL, NULL, 8, 4, - 100, 1); - -static MYSQL_SYSVAR_BOOL(enable_txlog_request_checkpoint, - eloq_enable_txlog_request_checkpoint, - PLUGIN_VAR_NOCMDARG | PLUGIN_VAR_READONLY, - "Enable txlog server sending checkpoint requests " - "when the criteria are met.", - NULL, NULL, TRUE); - -static MYSQL_SYSVAR_UINT(check_replay_log_size_interval_sec, - eloq_check_replay_log_size_interval_sec, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, - "The interval for checking replay log size.", NULL, - NULL, 10, 10, UINT_MAX, 1); - -static MYSQL_SYSVAR_STR( - notify_checkpointer_threshold_size, - eloq_notify_checkpointer_threshold_size, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_MEMALLOC, - "When the replay log size reaches this threshold, the " - "txlog server sends a checkpoint request to tx_service.", - NULL, NULL, "1GB"); - -static MYSQL_SYSVAR_STR(dss_config_file_path, eloq_dss_config_file_path, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_MEMALLOC, - "EloqDataStoreService config file path", NULL, NULL, - ""); -static MYSQL_SYSVAR_STR(dss_peer_node, eloq_dss_peer_node, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_MEMALLOC, - "EloqDataStoreService peer node endpoint", NULL, NULL, - ""); -static MYSQL_SYSVAR_STR(dss_branch_name, eloq_dss_branch_name, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_MEMALLOC, - "EloqDataStoreService branch name, specifying the " - "branch to start the data store service", - NULL, NULL, ""); - -static MYSQL_SYSVAR_STR(dss_rocksdb_cloud_bucket_name, - eloq_dss_rocksdb_cloud_bucket_name, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_MEMALLOC, - "EloqDataStoreService RocksDB cloud bucket name", NULL, - NULL, ""); -static MYSQL_SYSVAR_STR(dss_rocksdb_cloud_bucket_prefix, - eloq_dss_rocksdb_cloud_bucket_prefix, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_MEMALLOC, - "EloqDataStoreService RocksDB cloud bucket prefix", - NULL, NULL, ""); -static MYSQL_SYSVAR_STR(dss_rocksdb_cloud_object_path, - eloq_dss_rocksdb_cloud_object_path, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_MEMALLOC, - "EloqDataStoreService RocksDB cloud object path", NULL, - NULL, "rocksdb_cloud"); -static MYSQL_SYSVAR_STR(dss_rocksdb_cloud_region, - eloq_dss_rocksdb_cloud_region, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_MEMALLOC, - "EloqDataStoreService RocksDB cloud region", NULL, - NULL, ""); -static MYSQL_SYSVAR_STR(dss_rocksdb_cloud_endpoint_url, - eloq_dss_rocksdb_cloud_endpoint_url, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_MEMALLOC, - "EloqDataStoreService RocksDB Cloud endpoint URL", - NULL, NULL, ""); -static MYSQL_SYSVAR_STR( - dss_rocksdb_cloud_sst_file_cache_size, - eloq_dss_rocksdb_cloud_sst_file_cache_size, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_MEMALLOC, - "EloqDataStoreService RocksDB Cloud SST file cache size", NULL, NULL, - "10GB"); -static MYSQL_SYSVAR_INT( - dss_rocksdb_cloud_sst_file_cache_num_shard_bits, - eloq_dss_rocksdb_cloud_sst_file_cache_num_shard_bits, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_MEMALLOC, - "EloqDataStoreService RocksDB Cloud SST file cache num shard bits", NULL, - NULL, 5, 0, 30, 1); -static MYSQL_SYSVAR_STR(dss_rocksdb_target_file_size_base, - eloq_dss_rocksdb_target_file_size_base, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_MEMALLOC, - "EloqDataStoreService RocksDB target file size", NULL, - NULL, "64MB"); -static MYSQL_SYSVAR_UINT( - dss_rocksdb_cloud_purger_periodicity_secs, - eloq_dss_rocksdb_cloud_purger_periodicity_secs, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_MEMALLOC, - "EloqDataStoreService RocksDB Cloud purger periodcity seconds", NULL, NULL, - 600, 1, UINT_MAX, 1); - -static MYSQL_SYSVAR_UINT( - dss_rocksdb_cloud_file_deletion_delay, - eloq_dss_rocksdb_cloud_file_deletion_delay, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_MEMALLOC, - "EloqDataStoreService RocksDB Cloud becomes ready timeout", NULL, NULL, 60, - 1, 3600, 1); -static MYSQL_SYSVAR_UINT( - dss_rocksdb_max_write_buffer_number, - eloq_dss_rocksdb_max_write_buffer_number, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, - "EloqDataStoreService RocksDB max write buffer number", NULL, NULL, 8, 4, - 100, 1); -static MYSQL_SYSVAR_UINT(dss_rocksdb_max_background_jobs, - eloq_dss_rocksdb_max_background_jobs, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, - "EloqDataStoreService RocksDB max background jobs", - NULL, NULL, 8, 4, 100, 1); -#if defined(DATA_STORE_TYPE_ELOQDSS_ELOQSTORE) -static MYSQL_SYSVAR_UINT(eloqstore_worker_num, eloq_eloqstore_worker_num, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, - "EloqStore server worker num.", NULL, NULL, 1, 1, - UINT_MAX, 1); -static MYSQL_SYSVAR_STR( - eloqstore_data_path_list, eloq_eloqstore_data_path_list, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY | PLUGIN_VAR_MEMALLOC, - "The data paths of the EloqStore (use memory store if empty).", NULL, NULL, - ""); -static MYSQL_SYSVAR_UINT(eloqstore_open_files_limit, - eloq_eloqstore_open_files_limit, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, - "EloqStore server max open files.", NULL, NULL, 1024, - 1, UINT_MAX, 1); -static MYSQL_SYSVAR_STR( - eloqstore_cloud_store_path, eloq_eloqstore_cloud_store_path, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_MEMALLOC, - "EloqStore cloud store path (disable cloud store if empty)", NULL, NULL, - ""); -static MYSQL_SYSVAR_UINT(eloqstore_gc_threads, eloq_eloqstore_gc_threads, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, - "EloqStore server gc threads count (Must be 0 when " - "cloud store is enabled).", - NULL, NULL, 1, 0, UINT_MAX, 1); -static MYSQL_SYSVAR_UINT(eloqstore_cloud_worker_count, - eloq_eloqstore_cloud_worker_count, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, - "EloqStore server cloud worker count", NULL, NULL, 1, - 1, UINT_MAX, 1); -static MYSQL_SYSVAR_UINT(eloqstore_data_page_restart_interval, - eloq_eloqstore_data_page_restart_interval, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, - "EloqStore data page restart interval", NULL, NULL, - 16, 1, UINT_MAX, 1); -static MYSQL_SYSVAR_UINT(eloqstore_index_page_restart_interval, - eloq_eloqstore_index_page_restart_interval, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, - "EloqStore index page restart interval", NULL, NULL, - 16, 1, UINT_MAX, 1); -static MYSQL_SYSVAR_UINT(eloqstore_init_page_count, - eloq_eloqstore_init_page_count, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, - "EloqStore init page count", NULL, NULL, 1 << 15, 1, - UINT_MAX, 1); -static MYSQL_SYSVAR_BOOL(eloqstore_skip_verify_checksum, - eloq_eloqstore_skip_verify_checksum, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, - "EloqStore skip verify checksum", NULL, NULL, FALSE); -static MYSQL_SYSVAR_UINT(eloqstore_index_buffer_pool_size, - eloq_eloqstore_index_buffer_pool_size, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, - "EloqStore index buffer pool size", NULL, NULL, - 1 << 15, 1, UINT_MAX, 1); -static MYSQL_SYSVAR_UINT(eloqstore_manifest_limit, - eloq_eloqstore_manifest_limit, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, - "EloqStore manifest limit", NULL, NULL, 8 << 20, 1, - UINT_MAX, 1); -static MYSQL_SYSVAR_UINT(eloqstore_io_queue_size, eloq_eloqstore_io_queue_size, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, - "EloqStore io queue size", NULL, NULL, 4096, 1, - UINT_MAX, 1); -static MYSQL_SYSVAR_UINT(eloqstore_max_inflight_write, - eloq_eloqstore_max_inflight_write, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, - "EloqStore max inflight write", NULL, NULL, 64 << 10, - 1, UINT_MAX, 1); -static MYSQL_SYSVAR_UINT(eloqstore_max_write_batch_pages, - eloq_eloqstore_max_write_batch_pages, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, - "EloqStore max write batch pages", NULL, NULL, 256, 1, - UINT_MAX, 1); -static MYSQL_SYSVAR_UINT(eloqstore_buf_ring_size, eloq_eloqstore_buf_ring_size, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, - "EloqStore buf ring size", NULL, NULL, 1 << 12, 1, - UINT_MAX, 1); -static MYSQL_SYSVAR_UINT(eloqstore_coroutine_stack_size, - eloq_eloqstore_coroutine_stack_size, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, - "EloqStore coroutine stack size", NULL, NULL, - 32 * 1024, 1, UINT_MAX, 1); -static MYSQL_SYSVAR_UINT(eloqstore_num_retained_archives, - eloq_eloqstore_num_retained_archives, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, - "EloqStore num retained archives", NULL, NULL, 0, 0, - UINT_MAX, 1); -static MYSQL_SYSVAR_UINT(eloqstore_archive_interval_secs, - eloq_eloqstore_archive_interval_secs, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, - "EloqStore archive interval secs", NULL, NULL, 86400, - 1, UINT_MAX, 1); -static MYSQL_SYSVAR_UINT(eloqstore_max_archive_tasks, - eloq_eloqstore_max_archive_tasks, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, - "EloqStore max archive tasks", NULL, NULL, 256, 1, - UINT_MAX, 1); -static MYSQL_SYSVAR_UINT(eloqstore_file_amplify_factor, - eloq_eloqstore_file_amplify_factor, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, - "EloqStore file amplify factor", NULL, NULL, 4, 1, - UINT_MAX, 1); -static MYSQL_SYSVAR_ULONGLONG(eloqstore_local_space_limit, - eloq_eloqstore_local_space_limit, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, - "EloqStore local space limit", NULL, NULL, - 1ULL << 40, 1, ULONGLONG_MAX, 1); -static MYSQL_SYSVAR_UINT(eloqstore_reserve_space_ratio, - eloq_eloqstore_reserve_space_ratio, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, - "EloqStore reserve space ratio", NULL, NULL, 100, 1, - UINT_MAX, 1); -static MYSQL_SYSVAR_UINT(eloqstore_data_page_size, - eloq_eloqstore_data_page_size, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, - "EloqStore data page size", NULL, NULL, 1 << 12, 1, - UINT_MAX, 1); -static MYSQL_SYSVAR_UINT(eloqstore_pages_per_file_shift, - eloq_eloqstore_pages_per_file_shift, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, - "EloqStore pages per file shift", NULL, NULL, 11, 1, - UINT_MAX, 1); -static MYSQL_SYSVAR_UINT(eloqstore_overflow_pointers, - eloq_eloqstore_overflow_pointers, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, - "EloqStore overflow pointers", NULL, NULL, 16, 1, - UINT_MAX, 1); -static MYSQL_SYSVAR_BOOL(eloqstore_data_append_mode, - eloq_eloqstore_data_append_mode, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, - "EloqStore data append mode", NULL, NULL, FALSE); -#endif - static struct st_mysql_sys_var *eloq_system_variables[]= { - MYSQL_SYSVAR(local_ip), - MYSQL_SYSVAR(ip_list), - MYSQL_SYSVAR(standby_ip_list), - MYSQL_SYSVAR(voter_ip_list), - MYSQL_SYSVAR(hm_ip), - MYSQL_SYSVAR(keyspace_name), - MYSQL_SYSVAR(dynamodb_endpoint), - MYSQL_SYSVAR(core_num), - MYSQL_SYSVAR(cc_protocol), - MYSQL_SYSVAR(enum_var), - MYSQL_SYSVAR(kv_storage), - MYSQL_SYSVAR(dynamodb_region), - MYSQL_SYSVAR(aws_access_key_id), - MYSQL_SYSVAR(aws_secret_key), - MYSQL_SYSVAR(bigtable_project_id), - MYSQL_SYSVAR(bigtable_instance_id), - MYSQL_SYSVAR(ulong_var), - MYSQL_SYSVAR(int_var), - MYSQL_SYSVAR(double_var), - MYSQL_SYSVAR(double_thdvar), - MYSQL_SYSVAR(varopt_default), - MYSQL_SYSVAR(checkpointer_interval_sec), - MYSQL_SYSVAR(checkpointer_delay_sec), - MYSQL_SYSVAR(collect_active_tx_ts_interval_sec), - MYSQL_SYSVAR(realtime_sampling), - MYSQL_SYSVAR(insert_semantic), - MYSQL_SYSVAR(ddl_skip_kv), - MYSQL_SYSVAR(skip_redo_log), - MYSQL_SYSVAR(use_key_cache), - MYSQL_SYSVAR(auto_increment), - MYSQL_SYSVAR(kickout_data_for_test), - MYSQL_SYSVAR(invalidate_cache_once), - MYSQL_SYSVAR(scan_skip_kv), - MYSQL_SYSVAR(random_scan_sort), - MYSQL_SYSVAR(report_debug_info), - MYSQL_SYSVAR(node_memory_limit_mb), - MYSQL_SYSVAR(node_log_limit_mb), - MYSQL_SYSVAR(enable_mvcc), - MYSQL_SYSVAR(metrics_port), - MYSQL_SYSVAR(deadlock_interval_sec), - MYSQL_SYSVAR(txlog_rocksdb_storage_path), - MYSQL_SYSVAR(txlog_service_list), - MYSQL_SYSVAR(txlog_group_replica_num), - MYSQL_SYSVAR(signal_monitor), - MYSQL_SYSVAR(partition_type), - MYSQL_SYSVAR(enable_metrics), - MYSQL_SYSVAR(enable_mysql_tx_metrics), - MYSQL_SYSVAR(enable_mysql_dml_metrics), - MYSQL_SYSVAR(enable_tx_metrics), - MYSQL_SYSVAR(enable_cache_hit_rate), - MYSQL_SYSVAR(enable_memory_usage), - MYSQL_SYSVAR(enable_busy_round_metrics), - MYSQL_SYSVAR(enable_remote_request_metrics), - MYSQL_SYSVAR(enable_kv_metrics), - MYSQL_SYSVAR(busy_round_threshold), - MYSQL_SYSVAR(collect_memory_usage_round), - MYSQL_SYSVAR(collect_tx_duration_round), - MYSQL_SYSVAR(enable_log_service_metrics), - MYSQL_SYSVAR(txlog_rocksdb_cloud_bucket_name), - MYSQL_SYSVAR(txlog_rocksdb_cloud_bucket_prefix), - MYSQL_SYSVAR(txlog_rocksdb_cloud_object_path), - MYSQL_SYSVAR(txlog_rocksdb_cloud_region), - MYSQL_SYSVAR(txlog_rocksdb_cloud_sst_file_cache_size), - MYSQL_SYSVAR(txlog_rocksdb_cloud_sst_file_cache_num_shard_bits), - MYSQL_SYSVAR(txlog_rocksdb_cloud_endpoint_url), - MYSQL_SYSVAR(txlog_rocksdb_target_file_size_base), - MYSQL_SYSVAR(txlog_rocksdb_sst_files_size_limit), - MYSQL_SYSVAR(txlog_rocksdb_cloud_ready_timeout), - MYSQL_SYSVAR(txlog_rocksdb_cloud_file_deletion_delay), - MYSQL_SYSVAR(node_group_replica_num), - MYSQL_SYSVAR(logserver_snapshot_interval), - MYSQL_SYSVAR(logserver_rocksdb_scan_thread_num), - MYSQL_SYSVAR(txlog_rocksdb_cloud_in_mem_log_high_watermark), - MYSQL_SYSVAR(txlog_rocksdb_max_write_buffer_number), - MYSQL_SYSVAR(txlog_rocksdb_max_background_jobs), - MYSQL_SYSVAR(enable_txlog_request_checkpoint), - MYSQL_SYSVAR(check_replay_log_size_interval_sec), - MYSQL_SYSVAR(notify_checkpointer_threshold_size), - MYSQL_SYSVAR(range_split_worker_num), - MYSQL_SYSVAR(bthread_worker_num), - MYSQL_SYSVAR(hm_bin_path), - MYSQL_SYSVAR(cluster_config_file), - MYSQL_SYSVAR(enable_heap_defragment), - MYSQL_SYSVAR(dss_config_file_path), - MYSQL_SYSVAR(dss_peer_node), - MYSQL_SYSVAR(dss_branch_name), - MYSQL_SYSVAR(dss_rocksdb_cloud_bucket_name), - MYSQL_SYSVAR(dss_rocksdb_cloud_bucket_prefix), - MYSQL_SYSVAR(dss_rocksdb_cloud_object_path), - MYSQL_SYSVAR(dss_rocksdb_cloud_region), - MYSQL_SYSVAR(dss_rocksdb_cloud_sst_file_cache_size), - MYSQL_SYSVAR(dss_rocksdb_cloud_sst_file_cache_num_shard_bits), - MYSQL_SYSVAR(dss_rocksdb_cloud_endpoint_url), - MYSQL_SYSVAR(dss_rocksdb_target_file_size_base), - MYSQL_SYSVAR(dss_rocksdb_cloud_purger_periodicity_secs), - MYSQL_SYSVAR(dss_rocksdb_cloud_file_deletion_delay), - MYSQL_SYSVAR(dss_rocksdb_max_write_buffer_number), - MYSQL_SYSVAR(dss_rocksdb_max_background_jobs), -#if defined(DATA_STORE_TYPE_ELOQDSS_ELOQSTORE) - MYSQL_SYSVAR(eloqstore_worker_num), - MYSQL_SYSVAR(eloqstore_data_path_list), - MYSQL_SYSVAR(eloqstore_open_files_limit), - MYSQL_SYSVAR(eloqstore_cloud_store_path), - MYSQL_SYSVAR(eloqstore_gc_threads), - MYSQL_SYSVAR(eloqstore_cloud_worker_count), - MYSQL_SYSVAR(eloqstore_data_page_restart_interval), - MYSQL_SYSVAR(eloqstore_index_page_restart_interval), - MYSQL_SYSVAR(eloqstore_init_page_count), - MYSQL_SYSVAR(eloqstore_skip_verify_checksum), - MYSQL_SYSVAR(eloqstore_index_buffer_pool_size), - MYSQL_SYSVAR(eloqstore_manifest_limit), - MYSQL_SYSVAR(eloqstore_io_queue_size), - MYSQL_SYSVAR(eloqstore_max_inflight_write), - MYSQL_SYSVAR(eloqstore_max_write_batch_pages), - MYSQL_SYSVAR(eloqstore_buf_ring_size), - MYSQL_SYSVAR(eloqstore_coroutine_stack_size), - MYSQL_SYSVAR(eloqstore_num_retained_archives), - MYSQL_SYSVAR(eloqstore_archive_interval_secs), - MYSQL_SYSVAR(eloqstore_max_archive_tasks), - MYSQL_SYSVAR(eloqstore_file_amplify_factor), - MYSQL_SYSVAR(eloqstore_local_space_limit), - MYSQL_SYSVAR(eloqstore_reserve_space_ratio), - MYSQL_SYSVAR(eloqstore_data_page_size), - MYSQL_SYSVAR(eloqstore_pages_per_file_shift), - MYSQL_SYSVAR(eloqstore_overflow_pointers), - MYSQL_SYSVAR(eloqstore_data_append_mode), -#endif - NULL}; + MYSQL_SYSVAR(cc_protocol), MYSQL_SYSVAR(enum_var), + MYSQL_SYSVAR(ulong_var), MYSQL_SYSVAR(int_var), + MYSQL_SYSVAR(double_var), MYSQL_SYSVAR(double_thdvar), + MYSQL_SYSVAR(varopt_default), MYSQL_SYSVAR(insert_semantic), + MYSQL_SYSVAR(auto_increment), MYSQL_SYSVAR(invalidate_cache_once), + MYSQL_SYSVAR(random_scan_sort), MYSQL_SYSVAR(report_debug_info), + MYSQL_SYSVAR(signal_monitor), NULL}; /** Structure for CREATE TABLE options (table options). @@ -1440,15 +538,9 @@ static void init_eloq_psi_keys() {} static const char *ha_eloq_exts[]= {NullS}; uint32_t node_id= 0; // node id of itself -std::unique_ptr storage_hd= nullptr; - -#if ELOQDS -std::unique_ptr data_store_service_; -#endif - -static std::unique_ptr tx_service= nullptr; -static std::unique_ptr<::txlog::LogServer> txlog_server= nullptr; -static MariaCatalogFactory maria_catalog_factory{}; +static TxService *tx_service= nullptr; +static ::txlog::LogServer *txlog_server= nullptr; +store::DataStoreHandler *storage_hd= nullptr; // insert semantic will check unique of primary key, while upsert semantic will // update the duplicated entry. bool is_insert_semantic_= true; @@ -1564,7 +656,7 @@ static bool fetch_tx_isolation_level(THD *const thd, CcProtocol cc_protocol, // eloq can distinguish there two different isolation levels. To make // it consistent with Mysql, we convert user set RepeatableRead isolation // level to Snapshot Isolation if mvcc is enabled in Eloq engine. - if (eloq_enable_mvcc) + if (DataSubstrate::GetGlobal()->GetCoreConfig().enable_mvcc) { iso_level= IsolationLevel::Snapshot; } @@ -1623,7 +715,7 @@ static bool get_or_create_myeloq_tx(THD *const thd, MyEloqTx **my_tx, int16_t thd_group_id= thd_get_group_id(thd); auto [yield_func, resume_func]= thd_get_coro_functors(thd); txservice::TransactionExecution *txm= - txservice::NewTxInit(tx_service.get(), iso_level, cc_proto, UINT32_MAX, + txservice::NewTxInit(tx_service, iso_level, cc_proto, UINT32_MAX, thd_group_id, start_now, yield_func, resume_func); (*my_tx)->Reset(txm, thd); @@ -1648,8 +740,8 @@ static bool get_or_create_myeloq_tx(THD *const thd, MyEloqTx **my_tx, // A help function to drop table, outside of MariaDB context. static void drop_table(const std::string &table_name_str) { - TransactionExecution *txm= NewTxInit( - tx_service.get(), IsolationLevel::Serializable, CcProtocol::Locking); + TransactionExecution *txm= + NewTxInit(tx_service, IsolationLevel::Serializable, CcProtocol::Locking); if (txm) { // All tables in mariadb_tables are of TableType::Primary @@ -2330,113 +1422,14 @@ static int eloq_init_abort() DBUG_RETURN(1); } -#if defined(DATA_STORE_TYPE_ELOQDSS_ELOQSTORE) -/** - * Configure EloqStore settings for the data store service - * @param eloq_store_config Reference to EloqStoreConfig to be configured - * @param dss_data_path Base data path for the data store service - */ -static void configure_eloq_store(EloqDS::EloqStoreConfig &eloq_store_config, - const std::string &dss_data_path) -{ - eloq_store_config.eloqstore_configs_.num_threads= - std::max(eloq_eloqstore_worker_num, 1U); - - std::string storage_path_list(eloq_eloqstore_data_path_list); - if (!storage_path_list.empty()) - { - EloqDS::EloqStoreConfig::ParseStoragePath( - storage_path_list, eloq_store_config.eloqstore_configs_.store_path); - } - else - { - eloq_store_config.eloqstore_configs_.store_path.emplace_back() - .append(dss_data_path) - .append("/eloqstore"); - } - - eloq_store_config.eloqstore_configs_.fd_limit= - eloq_eloqstore_open_files_limit; - eloq_store_config.eloqstore_configs_.cloud_store_path= - eloq_eloqstore_cloud_store_path; - eloq_store_config.eloqstore_configs_.num_gc_threads= - !eloq_store_config.eloqstore_configs_.cloud_store_path.empty() - ? 0 - : eloq_eloqstore_gc_threads; - eloq_store_config.eloqstore_configs_.rclone_threads= - eloq_eloqstore_cloud_worker_count; - - LOG_IF(INFO, !eloq_store_config.eloqstore_configs_.cloud_store_path.empty()) - << "EloqStore cloud store enabled."; - - eloq_store_config.eloqstore_configs_.data_page_restart_interval= - eloq_eloqstore_data_page_restart_interval; - eloq_store_config.eloqstore_configs_.index_page_restart_interval= - eloq_eloqstore_index_page_restart_interval; - eloq_store_config.eloqstore_configs_.init_page_count= - eloq_eloqstore_init_page_count; - eloq_store_config.eloqstore_configs_.skip_verify_checksum= - eloq_eloqstore_skip_verify_checksum; - eloq_store_config.eloqstore_configs_.index_buffer_pool_size= - eloq_eloqstore_index_buffer_pool_size / - eloq_store_config.eloqstore_configs_.num_threads; - eloq_store_config.eloqstore_configs_.manifest_limit= - eloq_eloqstore_manifest_limit; - eloq_store_config.eloqstore_configs_.io_queue_size= - eloq_eloqstore_io_queue_size / - eloq_store_config.eloqstore_configs_.num_threads; - eloq_store_config.eloqstore_configs_.max_inflight_write= - eloq_eloqstore_max_inflight_write / - eloq_store_config.eloqstore_configs_.num_threads; - eloq_store_config.eloqstore_configs_.max_write_batch_pages= - eloq_eloqstore_max_write_batch_pages; - eloq_store_config.eloqstore_configs_.buf_ring_size= - eloq_eloqstore_buf_ring_size; - eloq_store_config.eloqstore_configs_.coroutine_stack_size= - eloq_eloqstore_coroutine_stack_size; - eloq_store_config.eloqstore_configs_.num_retained_archives= - eloq_eloqstore_num_retained_archives; - eloq_store_config.eloqstore_configs_.archive_interval_secs= - eloq_eloqstore_archive_interval_secs; - eloq_store_config.eloqstore_configs_.max_archive_tasks= - eloq_eloqstore_max_archive_tasks; - eloq_store_config.eloqstore_configs_.file_amplify_factor= - eloq_eloqstore_file_amplify_factor; - eloq_store_config.eloqstore_configs_.local_space_limit= - eloq_eloqstore_local_space_limit / - eloq_store_config.eloqstore_configs_.num_threads; - eloq_store_config.eloqstore_configs_.reserve_space_ratio= - eloq_eloqstore_reserve_space_ratio; - eloq_store_config.eloqstore_configs_.data_page_size= - eloq_eloqstore_data_page_size; - eloq_store_config.eloqstore_configs_.pages_per_file_shift= - eloq_eloqstore_pages_per_file_shift; - eloq_store_config.eloqstore_configs_.overflow_pointers= - eloq_eloqstore_overflow_pointers; - eloq_store_config.eloqstore_configs_.data_append_mode= - eloq_eloqstore_data_append_mode; -} -#endif - static int eloq_init_func(void *p) { DBUG_ENTER_FUNC(); PrintEloqConfig(); - CatalogFactory *catalog_factory[3]{&maria_catalog_factory, nullptr, nullptr}; - sql_print_information("Eloq initializing."); -#if defined(DATA_STORE_TYPE_DYNAMODB) || defined(LOG_STATE_TYPE_RKDB_S3) || \ - defined(DATA_STORE_TYPE_ELOQDSS_ROCKSDB_CLOUD_S3) - if (aws_init()) - { - sql_print_error("Eloq failed to initialize AWS SDK."); - DBUG_RETURN(1); - } -#endif - // add mono collation and mono mem cmp mutex to psi definition init_eloq_psi_keys(); mysql_mutex_init(mono_collation_data_mutex_key, &mono_collation_data_mutex, @@ -2468,27 +1461,11 @@ static int eloq_init_func(void *p) is_insert_semantic_= false; } - node_id= 0; - uint16_t local_port= 8000; - std::string local_ip; + tx_service= DataSubstrate::GetGlobal()->GetTxService(); + txlog_server= DataSubstrate::GetGlobal()->GetLogServer(); + storage_hd= DataSubstrate::GetGlobal()->GetStoreHandler(); - // parse the eloq config. - std::string local_ip_str(eloq_local_ip); - size_t idx= local_ip_str.find_first_of(':'); - - if (idx == std::string::npos) - { - local_ip= local_ip_str; - local_port= 8000; - local_ip_str.append(":8000"); - } - else - { - local_ip= local_ip_str.substr(0, idx); - local_port= std::stoi(local_ip_str.substr(idx + 1)); - } - - sql_print_information("MariaDB Node address: %s", local_ip_str.c_str()); + node_id= DataSubstrate::GetGlobal()->GetNetworkConfig().node_id; sql_print_information("MariaDB data home: %s", mysql_real_data_home_ptr); // initialize monogrpah specific error message. @@ -2501,516 +1478,6 @@ static int eloq_init_func(void *p) DBUG_RETURN(eloq_init_abort()); } - std::string local_path("local://"); - local_path.append(mysql_real_data_home_ptr); - if (local_path.at(local_path.size() - 1) == '/') - { - local_path.erase(local_path.size() - 1); - } - - // Read host manager address - std::string hm_ip(""); - uint16_t hm_port; - std::string hm_bin_path(eloq_hm_bin_path); - - std::string cluster_config_file(eloq_cluster_config_file); - if (cluster_config_file.empty()) - { - cluster_config_file.append(mysql_real_data_home_ptr); - cluster_config_file.append("/tx_service/cluster_config"); - } - - // bootstrap is done in standalone mode and does not need host manager. - if (!opt_bootstrap) - { - std::string hm_ip_str(eloq_hm_ip); - idx= hm_ip_str.find_first_of(':'); - if (idx != std::string::npos) - { - hm_ip= hm_ip_str.substr(0, idx); - hm_port= std::stoi(hm_ip_str.substr(idx + 1)); - } -#ifdef FORK_HM_PROCESS - else - { - hm_ip= local_ip; - hm_port= local_port + 4; - } - if (hm_bin_path.empty()) - { - char path_buf[PATH_MAX]; - ssize_t len= ::readlink("/proc/self/exe", path_buf, sizeof(path_buf)); - len-= strlen("/mariadbd"); - path_buf[len]= '\0'; - hm_bin_path= std::string(path_buf, len); - hm_bin_path.append("/host_manager"); - } -#endif - } - else - { - // No need to enable log in bootstrap. - eloq_skip_redo_log= true; - } - -#ifdef ELOQ_MODULE_ENABLED - GFLAGS_NAMESPACE::SetCommandLineOption( - "bthread_concurrency", std::to_string(eloq_core_num).c_str()); - GFLAGS_NAMESPACE::SetCommandLineOption("use_pthread_event_dispatcher", - "true"); - int busy_time= 10000; - GFLAGS_NAMESPACE::SetCommandLineOption("worker_polling_time_us", - std::to_string(busy_time).c_str()); -#endif - - std::unordered_map> ng_configs; - uint64_t cluster_config_version= 2; - - std::string store_keyspace_name(eloq_keyspace_name); - - switch (eloq_kv_storage) - { -#if defined(DATA_STORE_TYPE_DYNAMODB) - case KV_DYNAMO: { - // initialize DynamoDB handler. - std::string endpoint(eloq_dynamodb_endpoint); - std::string region(eloq_dynamodb_region); - std::string access_key(eloq_aws_access_key_id); - std::string secret_key(eloq_aws_secret_key); - - storage_hd= std::make_unique( - store_keyspace_name, endpoint, region, access_key, secret_key, - opt_bootstrap, eloq_ddl_skip_kv, eloq_core_num * 2); - if (!storage_hd->Connect()) - { - sql_print_error("!!!!!!!! Failed to connect to DynamoDB server, EloqDB " - "startup is terminated !!!!!!!!"); - DBUG_RETURN(eloq_init_abort()); - } - break; - } -#elif defined(DATA_STORE_TYPE_BIGTABLE) - case KV_BIGTABLE: { - std::string project_id(eloq_bigtable_project_id); - std::string instance_id(eloq_bigtable_instance_id); - storage_hd= std::make_unique( - store_keyspace_name, project_id, instance_id, opt_bootstrap, - eloq_ddl_skip_kv); - if (!storage_hd->Connect()) - { - sql_print_error("!!!!!!!! Failed to connect BigTable server, EloqDB " - "startup is terminated !!!!!!!!"); - DBUG_RETURN(eloq_init_abort()); - } - break; - } - -#elif ELOQDS - case KV_ELOQDS: { - bool is_single_node= true; - std::string ds_peer_node= eloq_dss_peer_node; - std::string ds_branch_name= eloq_dss_branch_name; - std::string dss_data_path= mysql_real_data_home_ptr; - dss_data_path.append("/eloq_dss"); - try - { - if (!std::filesystem::exists(dss_data_path)) - { - std::filesystem::create_directories(dss_data_path); - } - } - catch (const std::filesystem::filesystem_error &e) - { - sql_print_error("Failed to create dir: %s", dss_data_path.c_str()); - DBUG_RETURN(eloq_init_abort()); - } - std::string dss_config_file_path= eloq_dss_config_file_path; - if (dss_config_file_path.empty()) - { - dss_config_file_path= dss_data_path + "/dss_config.ini"; - } - - EloqDS::DataStoreServiceClusterManager ds_config; - if (!ds_config.Load(dss_config_file_path)) - { - if (!ds_peer_node.empty()) - { - ds_config.SetThisNode(local_ip, local_port + 7); - // Fetch ds topology from peer node - if (!EloqDS::DataStoreService::FetchConfigFromPeer(ds_peer_node, - ds_config)) - { - sql_print_error("Failed to fetch config from peer node: %s", - ds_peer_node.c_str()); - DBUG_RETURN(eloq_init_abort()); - } - - // Save the fetched config to the local file - if (!ds_config.Save(dss_config_file_path)) - { - sql_print_error("Failed to save config to file: %s", - dss_config_file_path.c_str()); - DBUG_RETURN(eloq_init_abort()); - } - } - else if (opt_bootstrap || is_single_node) - { - // Initialize the data store service config - ds_config.Initialize(local_ip, local_port + 7); - if (!ds_config.Save(dss_config_file_path)) - { - sql_print_error("Failed to save config to file: %s", - dss_config_file_path.c_str()); - DBUG_RETURN(eloq_init_abort()); - } - } - else - { - sql_print_error("Failed to load data store service config file: %s", - dss_config_file_path.c_str()); - DBUG_RETURN(eloq_init_abort()); - } - } - else - { - sql_print_information("EloqDataStoreService loaded config file %s", - dss_config_file_path.c_str()); - } - -#if defined(DATA_STORE_TYPE_ELOQDSS_ROCKSDB_CLOUD_S3) || \ - defined(DATA_STORE_TYPE_ELOQDSS_ROCKSDB_CLOUD_GCS) - // std::string ds_rocksdb_config_file_path= - // "/home/lzx/test-eloqsql/eloq_ds.ini"; - INIReader fake_config_reader(nullptr, 0); - EloqDS::RocksDBConfig rocksdb_config(fake_config_reader, dss_data_path); - EloqDS::RocksDBCloudConfig rocksdb_cloud_config(fake_config_reader); -#if defined(DATA_STORE_TYPE_ELOQDSS_ROCKSDB_CLOUD_S3) - rocksdb_cloud_config.aws_access_key_id_= eloq_aws_access_key_id; - rocksdb_cloud_config.aws_secret_key_= eloq_aws_secret_key; -#endif - rocksdb_cloud_config.bucket_name_= eloq_dss_rocksdb_cloud_bucket_name; - rocksdb_cloud_config.bucket_prefix_= eloq_dss_rocksdb_cloud_bucket_prefix; - rocksdb_cloud_config.object_path_= eloq_dss_rocksdb_cloud_object_path; - rocksdb_cloud_config.region_= eloq_dss_rocksdb_cloud_region; - rocksdb_cloud_config.s3_endpoint_url_= eloq_dss_rocksdb_cloud_endpoint_url; - rocksdb_cloud_config.sst_file_cache_size_= - txlog::parse_size(eloq_dss_rocksdb_cloud_sst_file_cache_size); - rocksdb_cloud_config.sst_file_cache_num_shard_bits_= - eloq_dss_rocksdb_cloud_sst_file_cache_num_shard_bits; - rocksdb_cloud_config.db_file_deletion_delay_= - eloq_dss_rocksdb_cloud_file_deletion_delay; - rocksdb_cloud_config.purger_periodicity_millis_= - eloq_dss_rocksdb_cloud_purger_periodicity_secs * 1000; - rocksdb_cloud_config.branch_name_= ds_branch_name; - - bool enable_cache_replacement_= fake_config_reader.GetBoolean( - "local", "enable_cache_replacement", false); - auto ds_factory= std::make_unique( - rocksdb_config, rocksdb_cloud_config, enable_cache_replacement_); -#elif defined(DATA_STORE_TYPE_ELOQDSS_ROCKSDB) - INIReader fake_config_reader(nullptr, 0); - EloqDS::RocksDBConfig rocksdb_config(fake_config_reader, dss_data_path); - bool enable_cache_replacement_= fake_config_reader.GetBoolean( - "local", "enable_cache_replacement", false); - auto ds_factory= std::make_unique( - rocksdb_config, enable_cache_replacement_); -#elif defined(DATA_STORE_TYPE_ELOQDSS_ELOQSTORE) - EloqDS::EloqStoreConfig eloq_store_config; - configure_eloq_store(eloq_store_config, dss_data_path); - auto ds_factory= std::make_unique( - std::move(eloq_store_config)); -#endif - - data_store_service_= std::make_unique( - ds_config, dss_config_file_path, dss_data_path + "/DSMigrateLog", - std::move(ds_factory)); - - // setup local data store service, the data store service will start - // data store if needed. - bool ret= - data_store_service_->StartService((opt_bootstrap || is_single_node)); - if (!ret) - { - sql_print_error("Failed to start data store service"); - DBUG_RETURN(eloq_init_abort()); - } - - // setup data store service client - storage_hd= std::make_unique( - catalog_factory, ds_config, data_store_service_.get()); - - if (!storage_hd->Connect()) - { - sql_print_error("!!!!!!!! Failed to connect ELOQ_DS server, EloqDB " - "startup is terminated !!!!!!!!"); - DBUG_RETURN(eloq_init_abort()); - } - - break; - } - -#endif - - default: - DBUG_RETURN(eloq_init_abort()); - } - if (opt_bootstrap) - { - // When execute mysql_install_db.sh, eloq should run in solo mode. - std::vector solo_config; - solo_config.emplace_back(0, local_ip, local_port); - ng_configs.try_emplace(0, std::move(solo_config)); - } - else if (!txservice::ReadClusterConfigFile(cluster_config_file, ng_configs, - cluster_config_version)) - { - - // Read cluster topology from general config file in this case - auto parse_res= txservice::ParseNgConfig( - eloq_ip_list, eloq_standby_ip_list, eloq_voter_ip_list, ng_configs, - eloq_node_group_replica_num, 0); - if (!parse_res) - { - LOG(ERROR) << "Failed to extract cluster configs from ip_port_list."; - DBUG_RETURN(eloq_init_abort()); - } - } - - bool found= false; - uint32_t native_ng_id= 0; - // check whether this node is in cluster. - for (auto &pair : ng_configs) - { - auto &ng_nodes= pair.second; - for (size_t i= 0; i < ng_nodes.size(); i++) - { - if (ng_nodes[i].host_name_ == local_ip && - ng_nodes[i].port_ == local_port) - { - node_id= ng_nodes[i].node_id_; - found= true; - if (ng_nodes[i].is_candidate_) - { - // found native_ng_id. - native_ng_id= pair.first; - break; - } - } - } - } - - if (!found) - { - sql_print_error("!!!!!!!! Current node does not belong to any node " - "group, EloqDB " - "startup is terminated !!!!!!!!"); - DBUG_RETURN(eloq_init_abort()); - } - - // Set max rpc message size as 512mb. - GFLAGS_NAMESPACE::SetCommandLineOption("max_body_size", "536870912"); - // Set bthread worker number - if (eloq_bthread_worker_num != 0) - { - GFLAGS_NAMESPACE::SetCommandLineOption( - "bthread_concurrency", - std::to_string(eloq_bthread_worker_num).c_str()); - } - GFLAGS_NAMESPACE::SetCommandLineOption( - "rocksdb_scan_threads", - std::to_string(eloq_logserver_rocksdb_scan_thread_num).c_str()); - std::vector txlog_ips; - std::vector txlog_ports; - - // If eloq_txlog_service_list is empty, it means bounded txlog service - if (std::strlen(eloq_txlog_service_list) == 0) - { - sql_print_information("Stand-alone txlog service is not provided, start " - "bounded txlog service."); - // initialize log service. - std::string txlog_path(local_path); - txlog_path.append("/tx_log"); - std::string txlog_rocksdb_path; - // default rocksdb path is /tx_log/rocksdb - if (strlen(eloq_txlog_rocksdb_storage_path) == 0) - { - // remove "local://" prefix - txlog_rocksdb_path= txlog_path.substr(8) + "/rocksdb"; - } - else - { - txlog_rocksdb_path= eloq_txlog_rocksdb_storage_path; - } - sql_print_information("EloqDB txlog path: %s", txlog_path.c_str()); - sql_print_information("EloqDB txlog rocksdb path: %s", - txlog_rocksdb_path.c_str()); - - uint16_t log_server_port= local_port + 2; - for (uint32_t ng= 0; ng < ng_configs.size(); ng++) - { - // Use cc node port + 2 for log server - txlog_ports.emplace_back(ng_configs[ng][0].port_ + 2); - txlog_ips.emplace_back(ng_configs[ng][0].host_name_); - } - - bool enable_txlog_request_checkpoint= - eloq_enable_txlog_request_checkpoint ? true : false; - [[maybe_unused]] uint64_t notify_checkpointer_threshold_size= - txlog::parse_size(eloq_notify_checkpointer_threshold_size); - sql_print_information( - "eloq_enable_txlog_request_checkpoint: %s", - (eloq_enable_txlog_request_checkpoint ? "ON" : "OFF")); - if (enable_txlog_request_checkpoint) - { - sql_print_information("eloq_check_replay_log_size_interval_sec: %d", - eloq_check_replay_log_size_interval_sec); - sql_print_information("eloq_notify_checkpointer_threshold_size: %s", - eloq_notify_checkpointer_threshold_size); - } - -#if defined(LOG_STATE_TYPE_RKDB_ALL) - size_t rocksdb_target_file_size_base_val= - txlog::parse_size(eloq_txlog_rocksdb_target_file_size_base); -#if defined(LOG_STATE_TYPE_RKDB_CLOUD) - txlog::RocksDBCloudConfig rocksdb_cloud_config; -#if defined(LOG_STATE_TYPE_RKDB_S3) - rocksdb_cloud_config.aws_access_key_id_= eloq_aws_access_key_id; - rocksdb_cloud_config.aws_secret_key_= eloq_aws_secret_key; -#endif /* LOG_STATE_TYPE_RKDB_S3 */ - rocksdb_cloud_config.bucket_name_= eloq_txlog_rocksdb_cloud_bucket_name; - rocksdb_cloud_config.bucket_prefix_= - eloq_txlog_rocksdb_cloud_bucket_prefix; - rocksdb_cloud_config.object_path_= eloq_txlog_rocksdb_cloud_object_path; - rocksdb_cloud_config.region_= eloq_txlog_rocksdb_cloud_region; - rocksdb_cloud_config.endpoint_url_= eloq_txlog_rocksdb_cloud_endpoint_url; - rocksdb_cloud_config.sst_file_cache_size_= - txlog::parse_size(eloq_txlog_rocksdb_cloud_sst_file_cache_size); - rocksdb_cloud_config.sst_file_cache_num_shard_bits_= - eloq_txlog_rocksdb_cloud_sst_file_cache_num_shard_bits; - rocksdb_cloud_config.db_ready_timeout_us_= - eloq_txlog_rocksdb_cloud_ready_timeout * 1000 * 1000; - rocksdb_cloud_config.db_file_deletion_delay_= - eloq_txlog_rocksdb_cloud_file_deletion_delay; - - if (opt_bootstrap) - { - txlog_server= std::make_unique<::txlog::LogServer>( - node_id, log_server_port, txlog_ips, txlog_ports, txlog_path, 0, - eloq_txlog_group_replica_num, txlog_rocksdb_path, - eloq_logserver_rocksdb_scan_thread_num, rocksdb_cloud_config, - eloq_txlog_rocksdb_cloud_in_mem_log_size_high_watermark, - eloq_txlog_rocksdb_max_write_buffer_number, - eloq_txlog_rocksdb_max_background_jobs, - rocksdb_target_file_size_base_val, eloq_logserver_snapshot_interval); - } - else - { - txlog_server= std::make_unique<::txlog::LogServer>( - node_id, log_server_port, txlog_ips, txlog_ports, txlog_path, 0, - eloq_txlog_group_replica_num, txlog_rocksdb_path, - eloq_logserver_rocksdb_scan_thread_num, rocksdb_cloud_config, - eloq_txlog_rocksdb_cloud_in_mem_log_size_high_watermark, - eloq_txlog_rocksdb_max_write_buffer_number, - eloq_txlog_rocksdb_max_background_jobs, - rocksdb_target_file_size_base_val, eloq_logserver_snapshot_interval, - enable_txlog_request_checkpoint, - eloq_check_replay_log_size_interval_sec, - notify_checkpointer_threshold_size); - } -#else /* LOG_STATE_TYPE_RKDB_CLOUD */ - // LOG_STATE_TYPE_RKDB - size_t rocksdb_sst_files_size_limit_val= - txlog::parse_size(eloq_txlog_rocksdb_sst_files_size_limit); - - if (opt_bootstrap) - { -#if defined(OPEN_LOG_SERVICE) - txlog_server= std::make_unique<::txlog::LogServer>( - node_id, log_server_port, txlog_path, 1, - rocksdb_sst_files_size_limit_val, - eloq_txlog_rocksdb_max_write_buffer_number, - eloq_txlog_rocksdb_max_background_jobs, - rocksdb_target_file_size_base_val); -#else - txlog_server= std::make_unique<::txlog::LogServer>( - node_id, log_server_port, txlog_ips, txlog_ports, txlog_path, 0, - eloq_txlog_group_replica_num, txlog_rocksdb_path, - eloq_logserver_rocksdb_scan_thread_num, - rocksdb_sst_files_size_limit_val, - eloq_txlog_rocksdb_max_write_buffer_number, - eloq_txlog_rocksdb_max_background_jobs, - rocksdb_target_file_size_base_val, eloq_logserver_snapshot_interval); -#endif - } - else - { -#if defined(OPEN_LOG_SERVICE) - txlog_server= std::make_unique<::txlog::LogServer>( - node_id, log_server_port, txlog_path, 1, - rocksdb_sst_files_size_limit_val, - eloq_txlog_rocksdb_max_write_buffer_number, - eloq_txlog_rocksdb_max_background_jobs, - rocksdb_target_file_size_base_val); -#else - txlog_server= std::make_unique<::txlog::LogServer>( - node_id, log_server_port, txlog_ips, txlog_ports, txlog_path, 0, - eloq_txlog_group_replica_num, txlog_rocksdb_path, - eloq_logserver_rocksdb_scan_thread_num, - rocksdb_sst_files_size_limit_val, - eloq_txlog_rocksdb_max_write_buffer_number, - eloq_txlog_rocksdb_max_background_jobs, - rocksdb_target_file_size_base_val, eloq_logserver_snapshot_interval, - enable_txlog_request_checkpoint, - eloq_check_replay_log_size_interval_sec, - notify_checkpointer_threshold_size); -#endif - } - -#endif /* LOG_STATE_TYPE_RKDB_CLOUD */ -#else /* LOG_STATE_TYPE_MEM */ -#if defined(OPEN_LOG_SERVICE) - txlog_server= std::make_unique<::txlog::LogServer>( - node_id, log_server_port, txlog_path, 1); -#else - txlog_server= std::make_unique<::txlog::LogServer>( - node_id, log_server_port, txlog_ips, txlog_ports, txlog_path, 0, - eloq_txlog_group_replica_num, eloq_logserver_snapshot_interval); -#endif -#endif /* LOG_STATE_TYPE_MEM */ - err= txlog_server->Start(); - if (err != 0) - { - // TODO: append detailed error information. - sql_print_error("Failed to start the tx log service in this node."); - DBUG_RETURN(eloq_init_abort()); - } - - sql_print_information("Bounded txlog service started."); - } - else - { - sql_print_information("Stand-alone txlog service provided: %s.", - eloq_txlog_service_list); - - std::string token; - std::istringstream txlog_ip_list_stream(eloq_txlog_service_list); - while (std::getline(txlog_ip_list_stream, token, ',')) - { - size_t c_idx= token.find_first_of(':'); - if (c_idx != std::string::npos) - { - txlog_ips.emplace_back(token.substr(0, c_idx)); - uint16_t pt= std::stoi(token.substr(c_idx + 1)); - txlog_ports.emplace_back(pt); - } - else - { - sql_print_error("Port is missing in eloq_txlog_service_list"); - DBUG_RETURN(eloq_init_abort()); - } - } - } - // fetch and print the global level cc_protocol switch (fetch_tx_cc_protocol(nullptr)) { @@ -3027,7 +1494,7 @@ static int eloq_init_func(void *p) assert(false); }; - if (eloq_enable_mvcc) + if (DataSubstrate::GetGlobal()->GetCoreConfig().enable_mvcc) { sql_print_information("mvcc is enabled."); } @@ -3036,206 +1503,6 @@ static int eloq_init_func(void *p) sql_print_information("mvcc is disabled."); } - // initialize the transaction service. - std::map tx_service_conf; - tx_service_conf.insert( - std::pair("core_num", eloq_core_num)); - tx_service_conf.insert(std::pair( - "range_split_worker_num", eloq_range_split_worker_num)); - tx_service_conf.insert(std::pair( - "checkpointer_interval", eloq_checkpointer_interval_sec)); - tx_service_conf.insert(std::pair( - "node_memory_limit_mb", eloq_node_memory_limit_mb)); - tx_service_conf.insert(std::pair( - "node_log_limit_mb", eloq_node_log_limit_mb)); - tx_service_conf.insert(std::pair( - "checkpointer_delay_seconds", eloq_checkpointer_delay_sec)); - tx_service_conf.insert(std::pair( - "collect_active_tx_ts_interval_seconds", - eloq_collect_active_tx_ts_interval_sec)); - tx_service_conf.insert(std::pair( - "realtime_sampling", eloq_realtime_sampling)); - tx_service_conf.insert(std::pair( - "rep_group_cnt", eloq_node_group_replica_num)); - tx_service_conf.insert(std::pair( - "enable_key_cache", eloq_use_key_cache ? 1 : 0)); - tx_service_conf.insert(std::pair( - "enable_shard_heap_defragment", eloq_enable_heap_defragment ? 1 : 0)); - tx_service_conf.insert(std::pair( - "bthread_worker_num", eloq_bthread_worker_num)); - tx_service_conf.insert(std::pair( - "kickout_data_for_test", eloq_kickout_data_for_test ? 1 : 0)); - - auto log_agent= std::make_unique( - static_cast(eloq_txlog_group_replica_num)); - - sql_print_information("enable_metrics: %s", - eloq_enable_metrics ? "ON" : "OFF"); - - metrics::CommonLabels empty_common_labels= {}; - std::vector>> - empty_external_metrics= {}; - if (eloq_enable_metrics && !opt_bootstrap) - { - setenv("ELOQ_METRICS_PORT", std::to_string(eloq_metrics_port).c_str(), - false); - MetricsRegistryImpl::MetricsRegistryResult metrics_registry_result= - MetricsRegistryImpl::GetRegistry(); - if (metrics_registry_result.not_ok_ != nullptr) - { - // MetricsRegistry Init error will be exit(-1); - sql_print_error("EloqDB enable metrics collector but " - "MetricsRegistry Init ERR %s", - metrics_registry_result.not_ok_); - DBUG_RETURN(eloq_init_abort()); - } - else - { - /* parse metrics options */ - metrics::enable_metrics= true; - - // mysql metrics - metrics::enable_mysql_tx_metrics= - eloq_enable_mysql_tx_metrics ? true : false; - sql_print_information("enable_mysql_tx_metrics: %s", - metrics::enable_mysql_tx_metrics ? "ON" : "OFF"); - - metrics::enable_mysql_dml_metrics= - eloq_enable_mysql_dml_metrics ? true : false; - sql_print_information("enable_mysql_dml_metrics: %s", - metrics::enable_mysql_dml_metrics ? "ON" : "OFF"); - - // tx_service metrics - metrics::enable_tx_metrics= eloq_enable_tx_metrics ? true : false; - sql_print_information("enable_tx_metrics: %s", - metrics::enable_tx_metrics ? "ON" : "OFF"); - if (metrics::enable_tx_metrics) - { - metrics::collect_tx_duration_round= eloq_collect_tx_duration_round; - sql_print_information( - "collect `collect_tx_duration_round` every %d round(s).", - metrics::collect_tx_duration_round); - } - - metrics::enable_cache_hit_rate= - eloq_enable_cache_hit_rate ? true : false; - sql_print_information("enable_cache_hit_rate: %s", - metrics::enable_cache_hit_rate ? "ON" : "OFF"); - - metrics::enable_busy_round_metrics= - eloq_enable_busy_round_metrics ? true : false; - sql_print_information("enable_busy_round_metrics: %s", - metrics::enable_busy_round_metrics ? "ON" : "OFF"); - if (metrics::enable_busy_round_metrics) - { - metrics::busy_round_threshold= eloq_busy_round_threshold; - sql_print_information("busy_round_threshold %d.", - metrics::busy_round_threshold); - } - - metrics::enable_memory_usage= eloq_enable_memory_usage ? true : false; - sql_print_information("enable_memory_usage: %s", - metrics::enable_memory_usage ? "ON" : "OFF"); - if (metrics::enable_memory_usage) - { - metrics::collect_memory_usage_round= eloq_collect_memory_usage_round; - sql_print_information("collect `memory_usage` every %d round(s).", - metrics::collect_memory_usage_round); - } - - metrics::enable_remote_request_metrics= - eloq_enable_remote_request_metrics ? true : false; - sql_print_information("enable_remote_request_metrics: %s", - metrics::enable_remote_request_metrics ? "ON" - : "OFF"); - - metrics::enable_kv_metrics= eloq_enable_kv_metrics ? true : false; - sql_print_information("enable_kv_metrics: %s", - metrics::enable_kv_metrics ? "ON" : "OFF"); - - // log_service metrics - metrics::enable_log_service_metrics= - eloq_enable_log_service_metrics ? true : false; - sql_print_information("enable_log_service_metrics: %s", - metrics::enable_log_service_metrics ? "ON" - : "OFF"); - - metrics_registry= std::move(metrics_registry_result.metrics_registry_); - - metrics::CommonLabels mysql_common_labels{}; - mysql_common_labels["node_ip"]= local_ip; - mysql_common_labels["node_port"]= std::to_string(mysqld_port); - - metrics::CommonLabels kv_common_labels{}; - kv_common_labels["node_ip"]= local_ip; - kv_common_labels["node_port"]= std::to_string(mysqld_port); - - metrics::register_mysql_metrics(metrics_registry.get(), - mysql_common_labels); - storage_hd->RegisterKvMetrics(metrics_registry.get(), kv_common_labels); - - metrics::CommonLabels tx_service_common_labels{}; - tx_service_common_labels["node_ip"]= local_ip; - tx_service_common_labels["node_port"]= std::to_string(mysqld_port); - tx_service_common_labels["node_id"]= std::to_string(node_id); - - tx_service= std::make_unique( - catalog_factory, &MariaSystemHandler::Instance(), tx_service_conf, - node_id, native_ng_id, &ng_configs, cluster_config_version, - storage_hd.get(), log_agent.get(), eloq_enable_mvcc, - eloq_skip_redo_log, false /*skip kv*/, - true /*enable cache replacement*/, true /*auto redirect */, - metrics_registry.get(), tx_service_common_labels); - - sql_print_information("Eloq metrics collector bind port: %d", - eloq_metrics_port); - -#ifdef EXT_TX_PROC_ENABLED - get_tx_service_functors= tx_service->GetTxProcFunctors(); -#endif - } - } - else - { - tx_service= std::make_unique( - catalog_factory, &MariaSystemHandler::Instance(), tx_service_conf, - node_id, native_ng_id, &ng_configs, cluster_config_version, - storage_hd != nullptr ? storage_hd.get() : nullptr, log_agent.get(), - eloq_enable_mvcc, eloq_skip_redo_log, false); - -#ifdef EXT_TX_PROC_ENABLED - get_tx_service_functors= tx_service->GetTxProcFunctors(); -#endif - } - - if (tx_service->Start(node_id, native_ng_id, &ng_configs, - cluster_config_version, &txlog_ips, &txlog_ports, - &hm_ip, &hm_port, &hm_bin_path, tx_service_conf, - std::move(log_agent), local_path, - cluster_config_file) < 0) - { - sql_print_error("!!!!!!Failed to start tx service. EloqDB startup is " - "terminated!!!!!!"); - DBUG_RETURN(eloq_init_abort()); - } - - txservice::Sequences::InitSequence(tx_service.get(), storage_hd.get()); - - DeadLockCheck::SetTimeInterval(eloq_deadlock_interval_sec); - - storage_hd->SetTxService(tx_service != nullptr ? tx_service.get() : nullptr); - sql_print_information("Transaction service started."); - sql_print_information( - "Number of cores allocated for transaction service: %u", eloq_core_num); - - eloq_partition_type= 1; - // tx_service is a distributed service, should wait for all the tx_service - // nodes to finish the log recovery process and setup the cc_stream_sender. - tx_service->WaitClusterReady(); - // wait for the tx_service node to become the native group leader. - tx_service->WaitNodeBecomeNativeGroupLeader(); - drop_orphan_tmp_tables(); if (eloq_signal_monitor != 0) @@ -3245,7 +1512,8 @@ static int eloq_init_func(void *p) #else if (txlog_server) { - txlog_server->CloseBraft(); + txlog_server->CloseBraft(); // TODO(liunyl): what is this for? can we + // remove txlog_server? } #endif }; @@ -3261,31 +1529,8 @@ static int eloq_done_func(void *p) { DBUG_ENTER_FUNC(); - sql_print_information("Shutting down the tx service."); - tx_service->Shutdown(); - sql_print_information("Tx service shut down."); - - sql_print_information("Shutting down the storage handler."); - storage_hd= nullptr; // Wait for all in-fight requests complete. -#if ELOQDS - if (data_store_service_ != nullptr) - { - data_store_service_= nullptr; - } -#endif - sql_print_information("Storage handler shut down."); - - sql_print_information("Shutting down the log service."); - txlog_server= nullptr; - sql_print_information("Log service shut down."); - - txservice::Sequences::Destory(); - - // Guarantee shutdown. In some cases, like bootstrap, - // eloq_pre_shutdown() won't call. - MariaSystemHandler::Instance().Shutdown(); - - tx_service= nullptr; + sql_print_information("Shutting down the data substrate."); + // DataSubstrate::GetGlobal()->Shutdown(); mysql_mutex_destroy(&mono_collation_data_mutex); mysql_mutex_destroy(&mono_mem_cmp_space_mutex); @@ -3295,18 +1540,6 @@ static int eloq_done_func(void *p) delete it; it= nullptr; } - - if (eloq_enable_metrics) - { - metrics_registry= nullptr; - } - -#if defined(DATA_STORE_TYPE_DYNAMODB) || \ - defined(LOG_STATE_TYPE_RKDB_S3) || \ - defined(DATA_STORE_TYPE_ELOQDSS_ROCKSDB_CLOUD_S3) - aws_deinit(); -#endif - DBUG_RETURN(0); } diff --git a/storage/eloq/ha_eloq.h b/storage/eloq/ha_eloq.h index 1dc6fc1bc33..f1a8b035604 100644 --- a/storage/eloq/ha_eloq.h +++ b/storage/eloq/ha_eloq.h @@ -78,7 +78,7 @@ using namespace MyEloq; using namespace txservice; extern uint32_t node_id; // node id of itself -extern std::unique_ptr storage_hd; +extern store::DataStoreHandler *storage_hd; /** @brief Eloq_share is a class that will be shared among all open handlers. diff --git a/storage/eloq/metrics_registry_impl.cpp b/storage/eloq/metrics_registry_impl.cpp deleted file mode 100644 index c0a216b72bd..00000000000 --- a/storage/eloq/metrics_registry_impl.cpp +++ /dev/null @@ -1,80 +0,0 @@ -/** - * Copyright (C) 2025 EloqData Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the following license: - * 1. GNU General Public License as published by the Free Software - * Foundation; version 2 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License V2 - * along with this program. If not, see . - * - */ -// -// Created by pangzhenzhou on 2022/10/10. -// - -#include "metrics_registry_impl.h" -#include -#include - -namespace MyEloq -{ -MetricsRegistryImpl::MetricsRegistryResult MetricsRegistryImpl::GetRegistry() -{ - struct make_registry_shared : public MetricsRegistryImpl - { - }; - static std::unique_ptr registry_impl= - std::make_unique(); - - if (registry_impl->metrics_mgr_result_.not_ok_ == nullptr) - { - return MetricsRegistryImpl::MetricsRegistryResult{std::move(registry_impl), - nullptr}; - } - else - { - return MetricsRegistryImpl::MetricsRegistryResult{ - nullptr, - registry_impl->metrics_mgr_result_.not_ok_, - }; - } -} - -// This method is the one that needs to be extended, the open method does not -// do anything for the current implementation. -metrics::MetricsErrors MetricsRegistryImpl::Open() -{ - return metrics::MetricsErrors::Success; -} - -metrics::MetricKey MetricsRegistryImpl::Register(const metrics::Name &name, - metrics::Type type, - const metrics::Labels &labels) -{ - auto metric= metrics::Metric(name.GetName(), type, labels); - - auto metric_collector= metrics_mgr_result_.mgr_->MetricsRegistry( - std::make_unique(metric)); - - auto key= metric_collector->metric_key_; - - std::unique_lock lock(collectors_mu_); - collectors_.insert(std::make_pair(key, std::move(metric_collector))); - - return key; -} - -void MetricsRegistryImpl::Collect(metrics::MetricKey key, - const metrics::Value &val) -{ - auto collector= collectors_[key].get(); - collector->Collect(val); -} -} // namespace MyEloq diff --git a/storage/eloq/metrics_registry_impl.h b/storage/eloq/metrics_registry_impl.h deleted file mode 100644 index 9d16fa0a245..00000000000 --- a/storage/eloq/metrics_registry_impl.h +++ /dev/null @@ -1,67 +0,0 @@ -/** - * Copyright (C) 2025 EloqData Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the following license: - * 1. GNU General Public License as published by the Free Software - * Foundation; version 2 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License V2 - * along with this program. If not, see . - * - */ -// -// Created by pangzhenzhou on 2022/10/10. -// -#pragma once -#include "metrics.h" -#include "metrics_manager.h" - -#include -#include - -namespace MyEloq -{ -class MetricsRegistryImpl : public metrics::MetricsRegistry -{ - -public: - struct MetricsRegistryResult - { - std::unique_ptr metrics_registry_; - const char *not_ok_; - }; - - MetricsRegistryImpl(MetricsRegistryImpl const &)= delete; - void operator=(MetricsRegistryImpl const &)= delete; - - ~MetricsRegistryImpl()= default; - static MetricsRegistryResult GetRegistry(); - - metrics::MetricsErrors Open() override; - metrics::MetricKey Register(const metrics::Name &, metrics::Type, - const metrics::Labels &) override; - void Collect(metrics::MetricKey, const metrics::Value &) override; - -private: - MetricsRegistryImpl()= default; - - metrics::MetricsMgr::MetricsMgrResult metrics_mgr_result_= - metrics::MetricsMgr::GetMetricMgrInstance(); - std::unordered_map intervals_; - std::unordered_map> - collectors_; - - // Mutex to exclusively protect 'collectors_' during write operations in - // multi-threaded environments. Use this mutex solely for guarding - // modifications to 'collectors_'. - std::mutex collectors_mu_; -}; - -} // namespace MyEloq diff --git a/storage/eloq/store_handler b/storage/eloq/store_handler deleted file mode 160000 index cd88ee6ec2a..00000000000 --- a/storage/eloq/store_handler +++ /dev/null @@ -1 +0,0 @@ -Subproject commit cd88ee6ec2a4684a8b032b75488319f923d888e7 diff --git a/storage/eloq/tx_service b/storage/eloq/tx_service deleted file mode 160000 index a3cd3a013b0..00000000000 --- a/storage/eloq/tx_service +++ /dev/null @@ -1 +0,0 @@ -Subproject commit a3cd3a013b07a9d9be2944aac5b549e7dd334c4e From c2cb84dac1e6dfcb6ec201e4730d3dd0ef29c1f1 Mon Sep 17 00:00:00 2001 From: liunyl Date: Mon, 20 Oct 2025 11:02:31 +0000 Subject: [PATCH 02/10] fix incorrect compile macro --- CMakeLists.txt | 4 ++-- storage/eloq/CMakeLists.txt | 20 +++++++++++++++++++- storage/eloq/eloq_db.cpp | 2 ++ storage/eloq/eloq_db_dl.h | 7 +++++++ 4 files changed, 30 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4f5dbbf4389..0ec3205aacb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -455,9 +455,9 @@ CONFIGURE_PLUGINS() # Build data substrate library first (before sql/ and storage/) -IF(ELOQ_MODULE_ENABLED) +# IF(ELOQ_MODULE_ENABLED) ADD_SUBDIRECTORY(data_substrate) -ENDIF() +# ENDIF() ADD_SUBDIRECTORY(include) ADD_SUBDIRECTORY(dbug) ADD_SUBDIRECTORY(strings) diff --git a/storage/eloq/CMakeLists.txt b/storage/eloq/CMakeLists.txt index bb90f250b31..3e324fc67c8 100644 --- a/storage/eloq/CMakeLists.txt +++ b/storage/eloq/CMakeLists.txt @@ -21,6 +21,7 @@ project(eloq) set(METRICS_LIB "eloq-metrics" CACHE STRING "metrics library name.") + # add_compile_definitions(ELOQ_MODULE_ELOQSQL) # if (WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB") # find_path(ROCKSDB_INCLUDE_PATH NAMES rocksdb/db.h) @@ -206,6 +207,23 @@ set(STATISTICS ON CACHE BOOL "Turn on STATISTICS") # # INCLUDE(build_eloq_metrics.cmake) +if(ELOQ_MODULE_ENABLED) + add_compile_definitions(ELOQ_MODULE_ENABLED) + message("ADDING ELOQ MODULE ENABLED") +endif() + +if (EXT_TX_PROC_ENABLED) + add_compile_definitions(EXT_TX_PROC_ENABLED) +endif() + +if(SMALL_RANGE) + add_compile_definitions(SMALL_RANGE) +endif() + +if(STATISTICS) + add_compile_definitions(STATISTICS) +endif() + find_package(MIMALLOC REQUIRED) include_directories(${MIMALLOC_INCLUDE_DIR}) @@ -302,7 +320,7 @@ SET(MYELOQ_LIBRARY ${MYELOQ_LIBRARY} ${METRICS_LIB}) MYSQL_ADD_PLUGIN(eloq ${MYELOQ_SOURCES} STORAGE_ENGINE - #MODULE_ONLY + # MODULE_ONLY MANDATORY STATIC_ONLY MODULE_OUTPUT_NAME ha_eloq diff --git a/storage/eloq/eloq_db.cpp b/storage/eloq/eloq_db.cpp index 43c693c6994..121115ca98c 100644 --- a/storage/eloq/eloq_db.cpp +++ b/storage/eloq/eloq_db.cpp @@ -587,6 +587,7 @@ static int load_view_fetch_func() std::string schema_image; bool ok= storage_hd->FetchTable(VIEW_TABLE_NAME, schema_image, exists, version_ts); + LOG(INFO) << "load_view_fetch_func " << ok << " exists: " << exists << " version_ts: " << version_ts; if (ok) { if (exists) @@ -631,6 +632,7 @@ static int eloq_fetch_view_normal(THD *thd, LEX_CSTRING db, LEX_CSTRING view, LEX_CSTRING &frm_binary) { DBUG_ENTER_FUNC(); + LOG(INFO) << "eloq_fetch_view_normal " << db.str << " " << view.str; int ret= 0; diff --git a/storage/eloq/eloq_db_dl.h b/storage/eloq/eloq_db_dl.h index 74cd7d73824..0009cb4c504 100644 --- a/storage/eloq/eloq_db_dl.h +++ b/storage/eloq/eloq_db_dl.h @@ -53,6 +53,12 @@ static LEX_CSTRING ELOQ_ENGINE_NAME= {STRING_WITH_LEN("ELOQ")}; * These functions must be wrapped with extern "C", and function signature * must match with that defined by ELOQ_DEFINEx. */ +// #if defined(ELOQ_LINK_STATIC) +// Statically linked: declare the symbol and let the linker resolve it to the +// implementation provided in the static library (see eloq_db.cpp extern "C" block). +// #define MONOCALL_DEFINEx(x, name, ...) \ +// extern "C" int name(__SC_DECL##x(__VA_ARGS__)); +// #else #define MONOCALL_DEFINEx(x, name, ...) \ static inline int name(__SC_DECL##x(__VA_ARGS__)) \ { \ @@ -71,6 +77,7 @@ static LEX_CSTRING ELOQ_ENGINE_NAME= {STRING_WITH_LEN("ELOQ")}; \ return ret; \ } +// #endif #define MONOCALL_DEFINE1(name, ...) MONOCALL_DEFINEx(1, name, __VA_ARGS__) #define MONOCALL_DEFINE2(name, ...) MONOCALL_DEFINEx(2, name, __VA_ARGS__) From e0265af1c82eb58b21a6092718bd17e43be6af34 Mon Sep 17 00:00:00 2001 From: liunyl Date: Tue, 21 Oct 2025 10:07:14 +0000 Subject: [PATCH 03/10] move data substrate init after mysql init --- data_substrate/core/CONFIGURATION.md | 140 ------------ data_substrate/core/CONFIG_MAPPING.md | 209 ------------------ .../core/config/sample_data_substrate.ini | 41 ---- data_substrate/core/src/data_substrate.cpp | 1 - data_substrate/core/src/metrics_init.cpp | 2 +- data_substrate/core/src/storage_init.cpp | 49 +--- data_substrate/store_handler | 2 +- data_substrate/tx_service | 2 +- sql/CMakeLists.txt | 2 +- sql/main.cc | 3 +- sql/mysqld.cc | 54 ++++- storage/eloq/CMakeLists.txt | 1 - storage/eloq/eloq_db.cpp | 4 + storage/eloq/eloq_db_dl.h | 40 ++-- 14 files changed, 77 insertions(+), 473 deletions(-) delete mode 100644 data_substrate/core/CONFIGURATION.md delete mode 100644 data_substrate/core/CONFIG_MAPPING.md delete mode 100644 data_substrate/core/config/sample_data_substrate.ini diff --git a/data_substrate/core/CONFIGURATION.md b/data_substrate/core/CONFIGURATION.md deleted file mode 100644 index dfb5cc6a0d3..00000000000 --- a/data_substrate/core/CONFIGURATION.md +++ /dev/null @@ -1,140 +0,0 @@ -# Data Substrate Configuration System - -## Overview - -The Data Substrate supports configuration through both INI configuration files and command line flags (gflags). Command line flags take precedence over configuration file values, allowing for flexible deployment and testing scenarios. - -## Configuration Sources (Priority Order) - -1. **Command Line Flags** (highest priority) -2. **Configuration File** (medium priority) -3. **Default Values** (lowest priority) - -## Configuration File Format - -The configuration file uses INI format with the following sections: - -### [data_substrate] -Core data substrate configuration parameters. - -| Parameter | Type | Default | Description | -|-----------|------|---------|-------------| -| `eloq_data_path` | string | "eloq_data" | Path for cc_ng and tx_log | -| `core_num` | int | 8 | Number of cores for transaction service | -| `checkpointer_interval` | int | 10 | Interval time(seconds) of checkpoint | -| `node_memory_limit_mb` | int | 8192 | txservice node_memory_limit_mb | -| `checkpointer_delay_seconds` | int | 5 | Checkpointer delay in seconds | -| `collect_active_tx_ts_interval_seconds` | int | 2 | Collect active tx timestamp interval | -| `realtime_sampling` | bool | true | Enable realtime sampling | -| `txlog_service_list` | string | "" | Log group servers configuration | -| `enable_tx_metrics` | bool | true | Enable transaction metrics | -| `enable_log_service_metrics` | bool | true | Enable log service metrics | -| `enable_key_cache` | bool | true | Enable key cache | -| `enable_shard_heap_defragment` | bool | false | Enable shard heap defragmentation | -| `bthread_worker_num` | int | 0 | Number of bthread workers | -| `max_standby_lag` | int | 0 | Maximum standby lag | -| `kickout_data_for_test` | bool | false | Clean data for test | - -### [storage] -Storage handler configuration parameters. - -| Parameter | Type | Default | Description | -|-----------|------|---------|-------------| -| `storage_type` | string | "rocksdb" | Storage type (rocksdb, dynamodb, data_store_service) | -| `rocksdb_storage_path` | string | "" | RocksDB storage path | -| `enable_cache_replacement` | bool | true | Enable cache replacement | -| `rocksdb_max_write_buffer_number` | int | 3 | RocksDB max write buffer number | -| `rocksdb_max_background_jobs` | int | 4 | RocksDB max background jobs | -| `rocksdb_target_file_size_base` | string | "64MB" | RocksDB target file size base | - -### [dynamodb] -DynamoDB-specific configuration (when storage_type = "dynamodb"). - -| Parameter | Type | Default | Description | -|-----------|------|---------|-------------| -| `dynamodb_endpoint` | string | "" | DynamoDB endpoint | -| `dynamodb_region` | string | "ap-northeast-1" | DynamoDB region | -| `dynamodb_keyspace` | string | "eloq_data" | DynamoDB keyspace | -| `aws_access_key_id` | string | "" | AWS access key ID | -| `aws_secret_key` | string | "" | AWS secret key | - -### [log_service] -Log service configuration parameters. - -| Parameter | Type | Default | Description | -|-----------|------|---------|-------------| -| `log_service_data_path` | string | "" | Log service data path | -| `rocksdb_scan_threads` | int | 4 | RocksDB scan threads | -| `txlog_group_replica_num` | int | 3 | Replica number of one log group | -| `logserver_snapshot_interval` | int | 3600 | Log server snapshot interval | -| `enable_txlog_request_checkpoint` | bool | true | Enable txlog request checkpoint | -| `notify_checkpointer_threshold_size` | string | "1GB" | Notify checkpointer threshold size | - -### [metrics] -Metrics configuration parameters. - -| Parameter | Type | Default | Description | -|-----------|------|---------|-------------| -| `enable_metrics` | bool | true | Enable metrics | -| `metrics_port` | int | 9090 | Metrics port | - -## Command Line Flags - -All configuration parameters can be overridden using command line flags. The flag names match the parameter names in the configuration file. - -### Example Usage - -```bash -# Use configuration file only -./data_substrate --config_file=config/sample_data_substrate.ini - -# Override specific parameters via command line -./data_substrate --config_file=config/sample_data_substrate.ini \ - --core_num=16 \ - --node_memory_limit_mb=16384 \ - --storage_type=dynamodb - -# Use command line flags only (no config file) -./data_substrate \ - --eloq_data_path=/custom/path \ - --core_num=8 \ - --storage_type=rocksdb \ - --rocksdb_storage_path=/custom/rocksdb/path -``` - -## Configuration Loading Process - -1. **Parse Command Line Flags**: gflags parses command line arguments -2. **Load Configuration File**: INI file is parsed if provided -3. **Apply Overrides**: Command line flags override config file values -4. **Use Defaults**: Any unspecified parameters use default values - -## Helper Functions - -The configuration system uses helper functions to handle the priority logic: - -- `GetConfigValue()`: Gets string values with gflags override -- `GetConfigBool()`: Gets boolean values with gflags override -- `GetConfigInt()`: Gets integer values with gflags override - -## Sample Configuration File - -See `config/sample_data_substrate.ini` for a complete example configuration file with all available parameters. - -## Integration with Engines - -The configuration system is designed to work with multiple database engines: - -- **eloqsql**: Uses MariaDB-specific catalog factory and system handler -- **eloqkv**: Uses eloqkv-specific catalog factory -- **eloqdoc**: Uses eloqdoc-specific catalog factory and system handler - -Engine-specific configurations are handled through compile-time macros (`#ifdef ELOQSQL`, `#ifdef ELOQKV`, `#ifdef ELOQDOC`). - -## Best Practices - -1. **Use Configuration Files**: For production deployments, use configuration files for consistency -2. **Command Line Overrides**: Use command line flags for testing and development -3. **Default Values**: Rely on sensible defaults for optional parameters -4. **Validation**: Always validate configuration values before use -5. **Documentation**: Keep configuration documentation up to date with code changes diff --git a/data_substrate/core/CONFIG_MAPPING.md b/data_substrate/core/CONFIG_MAPPING.md deleted file mode 100644 index cd50072a022..00000000000 --- a/data_substrate/core/CONFIG_MAPPING.md +++ /dev/null @@ -1,209 +0,0 @@ -# Configuration Parameter Mapping: eloqsql, eloqkv, and Data Substrate - -## Overview - -This document provides a comprehensive mapping of all configuration parameters across the three systems: -- **eloqsql**: MySQL system variables (MYSQL_SYSVAR) -- **eloqkv**: gflags (DEFINE_*) -- **Data Substrate**: gflags (DEFINE_*) - -## Configuration Categories - -### 1. Core Data Substrate Parameters - -| Parameter | eloqsql (MYSQL_SYSVAR) | eloqkv (DEFINE_*) | Data Substrate (DEFINE_*) | Description | -|-----------|------------------------|-------------------|---------------------------|-------------| -| **Data Path** | `eloq_data_path` | `eloq_data_path` | `eloq_data_path` | Path for cc_ng and tx_log | -| **Core Number** | `eloq_core_num` | `core_number` | `core_num` | Number of cores for transaction service | -| **Checkpointer Interval** | `eloq_checkpointer_interval_sec` | `checkpoint_interval` | `checkpointer_interval` | Interval time(seconds) of checkpoint | -| **Node Memory Limit** | `eloq_node_memory_limit_mb` | `node_memory_limit_mb` | `node_memory_limit_mb` | txservice node_memory_limit_mb | -| **Checkpointer Delay** | `eloq_checkpointer_delay_sec` | - | `checkpointer_delay_seconds` | Checkpointer delay in seconds | -| **Active TX TS Interval** | `eloq_collect_active_tx_ts_interval_sec` | - | `collect_active_tx_ts_interval_seconds` | Collect active tx timestamp interval | -| **Max Standby Lag** | - | `max_standby_lag` | `max_standby_lag` | Maximum standby lag | -| **Kickout Data for Test** | `eloq_kickout_data_for_test` | `kickout_data_for_test` | `kickout_data_for_test` | Clean data for test | -| **Use Key Cache** | `eloq_use_key_cache` | - | `enable_key_cache` | Enable key cache | -| **Enable Heap Defragment** | `eloq_enable_heap_defragment` | `enable_heap_defragment` | `enable_shard_heap_defragment` | Enable heap defragmentation | -| **Enable data WAL Log** | `enable_wal` | Enable WAL | -| **Enbale Data Store** | `enable_data_store` | Enable data storage | - -### 2. Network and Cluster Configuration - -| Parameter | eloqsql (MYSQL_SYSVAR) | eloqkv (DEFINE_*) | Data Substrate (DEFINE_*) | Description | -|-----------|------------------------|-------------------|---------------------------|-------------| -| **Local IP** | `eloq_local_ip` | `ip` | - | Local IP address | -| **IP List** | `eloq_ip_list` | `ip_port_list` | - | Server cluster ip port list | -| **Standby IP List** | `eloq_standby_ip_list` | `standby_ip_port_list` | - | Standby nodes ip:port list | -| **Voter IP List** | `eloq_voter_ip_list` | `voter_ip_port_list` | - | Voter nodes ip:port list | -| **Host Manager IP** | `eloq_hm_ip` | `hm_ip` | - | Host manager ip address | -| **Host Manager Port** | - | `hm_port` | - | Host manager port | -| **Host Manager Bin** | `eloq_hm_bin_path` | `hm_bin` | - | Host manager binary path | -| **Cluster Config File** | `eloq_cluster_config_file` | `cluster_config_file` | - | Path for cluster config file | -| **Node Group Replica Num** | `eloq_node_group_replica_num` | `tx_nodegroup_replica_num` | - | Replica number of one txservice node group | - -### 3. Log Service Configuration - -| Parameter | eloqsql (MYSQL_SYSVAR) | eloqkv (DEFINE_*) | Data Substrate (DEFINE_*) | Description | -|-----------|------------------------|-------------------|---------------------------|-------------| -| **TxLog Service List** | `eloq_txlog_service_list` | `txlog_service_list` | `txlog_service_list` | Log group servers configuration | -| **TxLog Group Replica Num** | `eloq_txlog_group_replica_num` | `txlog_group_replica_num` | `txlog_group_replica_num` | Replica number of one log group | -| **TxLog RocksDB Storage Path** | `eloq_txlog_rocksdb_storage_path` | `txlog_rocksdb_storage_path` | - | Storage path for txlog rocksdb state | -| **Log Service Data Path** | - | `log_service_data_path` | `log_service_data_path` | Log service data path | -| **RocksDB Scan Threads** | `eloq_logserver_rocksdb_scan_thread_num` | `txlog_rocksdb_scan_threads` | `rocksdb_scan_threads` | RocksDB scan threads | -| **Log Server Snapshot Interval** | `eloq_logserver_snapshot_interval` | `logserver_snapshot_interval` | `logserver_snapshot_interval` | Log server snapshot interval | -| **Enable TxLog Request Checkpoint** | `eloq_enable_txlog_request_checkpoint` | `enable_txlog_request_checkpoint` | `enable_txlog_request_checkpoint` | Enable txlog request checkpoint | -| **Notify Checkpointer Threshold** | `eloq_notify_checkpointer_threshold_size` | `notify_checkpointer_threshold_size` | `notify_checkpointer_threshold_size` | Notify checkpointer threshold size | -| **Check Replay Log Size Interval** | `eloq_check_replay_log_size_interval_sec` | `check_replay_log_size_interval_sec` | - | Check replay log size interval | -| **TxLog RocksDB Max Write Buffer Number** | `eloq_txlog_rocksdb_max_write_buffer_number` | `txlog_rocksdb_max_write_buffer_number` | `txlog_rocksdb_max_write_buffer_number` | RocksDB max write buffer number for txlog | -| **TxLog RocksDB Max Background Jobs** | `eloq_txlog_rocksdb_max_background_jobs` | `txlog_rocksdb_max_background_jobs` | `txlog_rocksdb_max_background_jobs` | RocksDB max background jobs for txlog | -| **TxLog RocksDB Target File Size Base** | `eloq_txlog_rocksdb_target_file_size_base` | `txlog_rocksdb_target_file_size_base` | `txlog_rocksdb_target_file_size_base` | RocksDB target file size base for txlog | -| **TxLog RocksDB SST Files Size Limit** | `eloq_txlog_rocksdb_sst_files_size_limit` | `txlog_rocksdb_sst_files_size_limit` | - | RocksDB SST files size limit for txlog | -| **TxLog RocksDB Cloud Region** | `eloq_txlog_rocksdb_cloud_region` | `txlog_rocksdb_cloud_region` | `txlog_rocksdb_cloud_region` | Cloud service region for txlog | -| **TxLog RocksDB Cloud Bucket Name** | `eloq_txlog_rocksdb_cloud_bucket_name` | `txlog_rocksdb_cloud_bucket_name` | `txlog_rocksdb_cloud_bucket_name` | Cloud storage bucket name for txlog | -| **TxLog RocksDB Cloud Bucket Prefix** | `eloq_txlog_rocksdb_cloud_bucket_prefix` | `txlog_rocksdb_cloud_bucket_prefix` | `txlog_rocksdb_cloud_bucket_prefix` | Cloud storage bucket prefix for txlog | -| **TxLog RocksDB Cloud Object Path** | `eloq_txlog_rocksdb_cloud_object_path` | `txlog_rocksdb_cloud_object_path` | `txlog_rocksdb_cloud_object_path` | Cloud object path for txlog | -| **TxLog RocksDB Cloud Endpoint URL** | `eloq_txlog_rocksdb_cloud_endpoint_url` | `txlog_rocksdb_cloud_endpoint_url` | `txlog_rocksdb_cloud_endpoint_url` | Cloud storage endpoint URL for txlog | -| **TxLog RocksDB Cloud SST File Cache Size** | `eloq_txlog_rocksdb_cloud_sst_file_cache_size` | `txlog_rocksdb_cloud_sst_file_cache_size` | `txlog_rocksdb_cloud_sst_file_cache_size` | Cloud SST file cache size for txlog | -| **TxLog RocksDB Cloud Ready Timeout** | `eloq_txlog_rocksdb_cloud_ready_timeout` | `txlog_rocksdb_cloud_ready_timeout` | `txlog_rocksdb_cloud_ready_timeout` | Cloud ready timeout for txlog | -| **TxLog RocksDB Cloud File Deletion Delay** | `eloq_txlog_rocksdb_cloud_file_deletion_delay` | `txlog_rocksdb_cloud_file_deletion_delay` | `txlog_rocksdb_cloud_file_deletion_delay` | Cloud file deletion delay for txlog | - -### 4. Storage Configuration - -| Parameter | eloqsql (MYSQL_SYSVAR) | eloqkv (DEFINE_*) | Data Substrate (DEFINE_*) | Description | -|-----------|------------------------|-------------------|---------------------------|-------------| -| **Enable Cache Replacement** | - | `enable_cache_replacement` | `enable_cache_replacement` | Enable cache replacement | -| **RocksDB Storage Path** | - | - | `rocksdb_storage_path` | RocksDB storage path | - -#### 4.1. DynamoDB Configuration - -| Parameter | eloqsql (MYSQL_SYSVAR) | eloqkv (DEFINE_*) | Data Substrate (DEFINE_*) | Description | -|-----------|------------------------|-------------------|---------------------------|-------------| -| **DynamoDB Endpoint** | `eloq_dynamodb_endpoint` | `dynamodb_endpoint` | `dynamodb_endpoint` | DynamoDB endpoint | -| **DynamoDB Region** | `eloq_dynamodb_region` | `dynamodb_region` | `dynamodb_region` | DynamoDB region | -| **DynamoDB Keyspace** | - | `dynamodb_keyspace` | `dynamodb_keyspace` | DynamoDB keyspace | -| **AWS Access Key ID** | `eloq_aws_access_key_id` | `aws_access_key_id` | `aws_access_key_id` | AWS access key ID | -| **AWS Secret Key** | `eloq_aws_secret_key` | `aws_secret_key` | `aws_secret_key` | AWS secret key | - -#### 4.2. BigTable Configuration (eloqsql only) - -| Parameter | eloqsql (MYSQL_SYSVAR) | Description | -|-----------|------------------------|-------------| -| **BigTable Project ID** | `eloq_bigtable_project_id` | BigTable project ID | -| **BigTable Instance ID** | `eloq_bigtable_instance_id` | BigTable instance ID | - -#### 4.3. Data Store Service (ELOQDS) -| Parameter | eloqsql (MYSQL_SYSVAR) | eloqkv (DEFINE_*) | Description | -|-----------|------------------------|-------------------|-------------| -| **DSS Config File Path** | `eloq_dss_config_file_path` | - | DSS config file path | -| **DSS Peer Node** | `eloq_dss_peer_node` | `eloq_dss_peer_node` | DSS peer node | -| **DSS Branch Name** | `eloq_dss_branch_name` | `eloq_dss_branch_name` | DSS branch name | -| **DSS RocksDB Cloud Config** | Multiple `eloq_dss_rocksdb_cloud_*` | - | DSS RocksDB cloud configuration | - -#### 4.4. EloqStore (DATA_STORE_TYPE_ELOQDSS_ELOQSTORE) -| Parameter | eloqsql (MYSQL_SYSVAR) | Description | -|-----------|------------------------|-------------| -| **EloqStore Worker Num** | `eloq_eloqstore_worker_num` | EloqStore worker number | -| **EloqStore Data Path List** | `eloq_eloqstore_data_path_list` | EloqStore data path list | -| **EloqStore Open Files Limit** | `eloq_eloqstore_open_files_limit` | EloqStore open files limit | -| **EloqStore Cloud Store Path** | `eloq_eloqstore_cloud_store_path` | EloqStore cloud store path | -| **EloqStore GC Threads** | `eloq_eloqstore_gc_threads` | EloqStore GC threads | -| **EloqStore Cloud Worker Count** | `eloq_eloqstore_cloud_worker_count` | EloqStore cloud worker count | -| **EloqStore Data Page Restart Interval** | `eloq_eloqstore_data_page_restart_interval` | EloqStore data page restart interval | -| **EloqStore Index Page Restart Interval** | `eloq_eloqstore_index_page_restart_interval` | EloqStore index page restart interval | -| **EloqStore Init Page Count** | `eloq_eloqstore_init_page_count` | EloqStore init page count | -| **EloqStore Skip Verify Checksum** | `eloq_eloqstore_skip_verify_checksum` | EloqStore skip verify checksum | -| **EloqStore Index Buffer Pool Size** | `eloq_eloqstore_index_buffer_pool_size` | EloqStore index buffer pool size | -| **EloqStore Manifest Limit** | `eloq_eloqstore_manifest_limit` | EloqStore manifest limit | -| **EloqStore IO Queue Size** | `eloq_eloqstore_io_queue_size` | EloqStore IO queue size | -| **EloqStore Max Inflight Write** | `eloq_eloqstore_max_inflight_write` | EloqStore max inflight write | -| **EloqStore Max Write Batch Pages** | `eloq_eloqstore_max_write_batch_pages` | EloqStore max write batch pages | -| **EloqStore Buf Ring Size** | `eloq_eloqstore_buf_ring_size` | EloqStore buf ring size | -| **EloqStore Coroutine Stack Size** | `eloq_eloqstore_coroutine_stack_size` | EloqStore coroutine stack size | -| **EloqStore Num Retained Archives** | `eloq_eloqstore_num_retained_archives` | EloqStore num retained archives | -| **EloqStore Archive Interval** | `eloq_eloqstore_archive_interval_secs` | EloqStore archive interval | -| **EloqStore Max Archive Tasks** | `eloq_eloqstore_max_archive_tasks` | EloqStore max archive tasks | -| **EloqStore File Amplify Factor** | `eloq_eloqstore_file_amplify_factor` | EloqStore file amplify factor | -| **EloqStore Local Space Limit** | `eloq_eloqstore_local_space_limit` | EloqStore local space limit | -| **EloqStore Reserve Space Ratio** | `eloq_eloqstore_reserve_space_ratio` | EloqStore reserve space ratio | -| **EloqStore Data Page Size** | `eloq_eloqstore_data_page_size` | EloqStore data page size | -| **EloqStore Pages Per File Shift** | `eloq_eloqstore_pages_per_file_shift` | EloqStore pages per file shift | -| **EloqStore Overflow Pointers** | `eloq_eloqstore_overflow_pointers` | EloqStore overflow pointers | -| **EloqStore Data Append Mode** | `eloq_e - -### 5. Metrics Configuration - -| Parameter | eloqsql (MYSQL_SYSVAR) | eloqkv (DEFINE_*) | Data Substrate (DEFINE_*) | Description | -|-----------|------------------------|-------------------|---------------------------|-------------| -| **Enable Metrics** | `eloq_enable_metrics` | - | `enable_metrics` | Enable metrics | -| **Metrics Port** | `eloq_metrics_port` | - | `metrics_port` | Metrics port | -| **Enable TX Metrics** | `eloq_enable_tx_metrics` | - | `enable_tx_metrics` | Enable transaction metrics | -| **Enable Log Service Metrics** | `eloq_enable_log_service_metrics` | - | `enable_log_service_metrics` | Enable log service metrics | -| **Enable Cache Hit Rate** | `eloq_enable_cache_hit_rate` | - | `enable_cache_hit_rate` | Enable cache hit rate metrics | -| **Enable KV Metrics** | `eloq_enable_kv_metrics` | - | `enable_kv_metrics` | Enable KV store metrics | -| **Enable Remote Request Metrics** | `eloq_enable_remote_request_metrics` | - | `enable_remote_request_metrics` | Enable remote request metrics | -| **Collect TX Duration Round** | `eloq_collect_tx_duration_round` | - | `collect_tx_duration_round` | Collect transaction duration round | -| **Busy Round Threshold** | `eloq_busy_round_threshold` | - | `busy_round_threshold` | Busy round threshold | -| **Collect Memory Usage Round** | `eloq_collect_memory_usage_round` | - | - | Collect memory usage round | -| **Enable Busy Round Metrics** | `eloq_enable_busy_round_metrics` | - | - | Enable busy round metrics | -| **Enable Memory Usage** | `eloq_enable_memory_usage` | - | - | Enable memory usage metrics | -| **Realtime Sampling** | `eloq_realtime_sampling` | - | `realtime_sampling` | Enable realtime sampling | - -### 6. Engine-Specific Configuration - -#### eloqsql-specific (MySQL system variables) -| Parameter | Description | -|-----------|-------------| -| `eloq_kv_storage` | KV storage type | -| `eloq_insert_semantic` | Insert semantic (insert/upsert) | -| `eloq_auto_increment` | Auto increment configuration | -| `eloq_invalidate_cache_once` | Invalidate cache once | -| `eloq_ddl_skip_kv` | Skip KV for DDL operations | -| `eloq_skip_redo_log` | Skip redo log | -| `eloq_scan_skip_kv` | Skip KV for scan operations | -| `eloq_random_scan_sort` | Random scan sort | -| `eloq_report_debug_info` | Report debug info | -| `eloq_deadlock_interval_sec` | Deadlock check interval | -| `eloq_signal_monitor` | Signal monitor | -| `eloq_partition_type` | Partition type | -| `eloq_range_split_worker_num` | Range split worker number | - -#### eloqkv-specific (gflags) -| Parameter | Description | -|-----------|-------------| -| `auto_redirect` | Auto redirect request to remote node | -| `cc_notify` | Notify txrequest sender when cc request finishes | -| `enable_io_uring` | Enable io_uring as IO engine | -| `raft_log_async_fsync` | Raft log fsync performed asynchronously | -| `bind_all` | Listen on all interfaces | -| `port` | Redis port | -| `bootstrap` | Init system tables and exit | -| `maxclients` | Maximum clients | -| `slow_log_threshold` | Slow log threshold | -| `slow_log_max_length` | Slow log max length | -| `enable_redis_stats` | Enable redis statistics | -| `enable_cmd_sort` | Enable command sort in Multi-Exec | -| `isolation_level` | Isolation level of simple commands | -| `protocol` | Concurrency control protocol | -| `txn_isolation_level` | Isolation level of MULTI/EXEC | -| `txn_protocol` | Concurrency control protocol of MULTI/EXEC | -| `snapshot_sync_worker_num` | Snapshot sync worker num | -| `retry_on_occ_error` | Retry transaction on OCC error | -| `fork_host_manager` | Fork host manager process | - -### Total Configuration Parameters: -- **eloqsql**: ~150+ parameters (including compile-time macro parameters) -- **eloqkv**: ~75+ parameters (including compile-time macro parameters) -- **Data Substrate**: ~40+ parameters (core unified parameters) - -### Key Observations: -1. **Naming Inconsistencies**: Different naming conventions across systems (e.g., `core_num` vs `core_number`) -2. **Missing Parameters**: Some parameters exist in one system but not others -3. **Compile-time Macros**: Many parameters are wrapped in compile-time macros for conditional compilation -4. **Engine-Specific**: Each system has unique parameters for its specific functionality -5. **Unified Approach**: Data Substrate provides a unified configuration system that can support all three engines - -### Recommendations: -1. **Standardize Naming**: Use consistent parameter names across all systems -2. **Complete Coverage**: Ensure all necessary parameters are available in Data Substrate -3. **Documentation**: Maintain this mapping as systems evolve -4. **Validation**: Implement parameter validation and type checking -5. **Migration Path**: Provide clear migration path from existing systems to Data Substrate diff --git a/data_substrate/core/config/sample_data_substrate.ini b/data_substrate/core/config/sample_data_substrate.ini deleted file mode 100644 index 7e071a9674c..00000000000 --- a/data_substrate/core/config/sample_data_substrate.ini +++ /dev/null @@ -1,41 +0,0 @@ -[data_substrate] -eloq_data_path = /tmp/eloq_data -core_num = 8 -checkpointer_interval = 10 -node_memory_limit_mb = 8192 -checkpointer_delay_seconds = 5 -collect_active_tx_ts_interval_seconds = 2 -realtime_sampling = true -txlog_service_list = 127.0.0.1:8001,127.0.0.1:8002,127.0.0.1:8003 -txlog_group_replica_num = 3 -enable_tx_metrics = true -enable_log_service_metrics = true -enable_cache_hit_rate = true -enable_kv_metrics = true -enable_remote_request_metrics = true -collect_tx_duration_round = 1 -busy_round_threshold = 10 -enable_key_cache = true -enable_shard_heap_defragment = false -bthread_worker_num = 0 -max_standby_lag = 0 -kickout_data_for_test = false - -[storage] -storage_type = rocksdb -rocksdb_storage_path = /tmp/eloq_data/rocksdb -enable_cache_replacement = true -rocksdb_max_write_buffer_number = 3 -rocksdb_max_background_jobs = 4 -rocksdb_target_file_size_base = 67108864 - -[log_service] -log_service_data_path = /tmp/eloq_data/log_service -rocksdb_scan_threads = 4 -logserver_snapshot_interval = 3600 -enable_txlog_request_checkpoint = true -notify_checkpointer_threshold_size = 1GB - -[metrics] -enable_metrics = true -metrics_port = 9090 diff --git a/data_substrate/core/src/data_substrate.cpp b/data_substrate/core/src/data_substrate.cpp index fe5473360ab..4bd08cb604c 100644 --- a/data_substrate/core/src/data_substrate.cpp +++ b/data_substrate/core/src/data_substrate.cpp @@ -185,7 +185,6 @@ void DataSubstrate::Shutdown() #if ELOQDS if (data_store_service_ != nullptr) { - data_store_service_->DisconnectDataStore(); data_store_service_= nullptr; } #endif diff --git a/data_substrate/core/src/metrics_init.cpp b/data_substrate/core/src/metrics_init.cpp index 20cdb15ce7f..a638c4f8ca4 100644 --- a/data_substrate/core/src/metrics_init.cpp +++ b/data_substrate/core/src/metrics_init.cpp @@ -11,7 +11,7 @@ #include "log_service_metrics.h" #endif #ifdef ELOQ_MODULE_ELOQSQL -#include "mysql_metrics.h" +#include "mysql_metrics.h" //TDOO(liunyl): move this file to correct location #endif DEFINE_bool(enable_metrics, false, "Enable metrics"); diff --git a/data_substrate/core/src/storage_init.cpp b/data_substrate/core/src/storage_init.cpp index f4a32fb894f..d33ec1a5854 100644 --- a/data_substrate/core/src/storage_init.cpp +++ b/data_substrate/core/src/storage_init.cpp @@ -283,60 +283,13 @@ bool DataSubstrate::InitializeStorageHandler(const INIReader &config_reader) dss_config_file_path, eloq_dss_data_path + "/DSMigrateLog", std::move(ds_factory)); - std::vector dss_shards = ds_config.GetShardsForThisNode(); - std::unordered_map > - dss_shards_map; - // setup rocksdb cloud data store - for (int shard_id : dss_shards) - { -#if defined(DATA_STORE_TYPE_ELOQDSS_ROCKSDB_CLOUD_S3) || \ - defined(DATA_STORE_TYPE_ELOQDSS_ROCKSDB_CLOUD_GCS) - // TODO(lzx): move setup datastore to data_store_service - auto ds = std::make_unique( - rocksdb_cloud_config, - rocksdb_config, - (core_config_.bootstrap || is_single_node), - core_config_.enable_cache_replacement, - shard_id, - data_store_service_.get()); -#elif defined(DATA_STORE_TYPE_ELOQDSS_ROCKSDB) - // TODO(lzx): move setup datastore to data_store_service - auto ds = std::make_unique( - rocksdb_config, - (core_config_.bootstrap || is_single_node), - core_config_.enable_cache_replacement, - shard_id, - data_store_service_.get()); - -#elif defined(DATA_STORE_TYPE_ELOQDSS_ELOQSTORE) - auto ds = std::make_unique( - shard_id, data_store_service_.get()); -#endif - ds->Initialize(); - - // Start db if the shard status is not closed - if (ds_config.FetchDSShardStatus(shard_id) != - EloqDS::DSShardStatus::Closed) - { - bool ret = ds->StartDB(); - if (!ret) - { - LOG(ERROR) - << "Failed to start db instance in data store service"; - return false; - } - } - dss_shards_map[shard_id] = std::move(ds); - } - // setup local data store service - bool ret = data_store_service_->StartService(); + bool ret = data_store_service_->StartService(core_config_.bootstrap || is_single_node); if (!ret) { LOG(ERROR) << "Failed to start data store service"; return false; } - data_store_service_->ConnectDataStore(std::move(dss_shards_map)); // setup data store service client txservice::CatalogFactory *catalog_factory[NUM_EXTERNAL_ENGINES] = {nullptr, nullptr, nullptr}; diff --git a/data_substrate/store_handler b/data_substrate/store_handler index d64612f04ed..60176c420db 160000 --- a/data_substrate/store_handler +++ b/data_substrate/store_handler @@ -1 +1 @@ -Subproject commit d64612f04ed7256fe97fa1558a4ca53372114a73 +Subproject commit 60176c420db7178f878939ff6d1103203da17cc2 diff --git a/data_substrate/tx_service b/data_substrate/tx_service index 734a724c359..b69746c2eec 160000 --- a/data_substrate/tx_service +++ b/data_substrate/tx_service @@ -1 +1 @@ -Subproject commit 734a724c3593a08a3e14c7e5882040f8e45f8894 +Subproject commit b69746c2eec7c8ac9b4c48c3626fc589068aa617 diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt index 5d7402df1c8..be3c91bd6e9 100644 --- a/sql/CMakeLists.txt +++ b/sql/CMakeLists.txt @@ -287,7 +287,7 @@ IF(COROUTINE_ENABLED) ENDIF() IF(ELOQ_MODULE_ENABLED) - SET(SQL_DEP_LIBS ${SQL_DEP_LIBS} ${BRPC_LIB}) + SET(SQL_DEP_LIBS ${SQL_DEP_LIBS} ${BRPC_LIB} ${DATA_SUBSTRATE_LIB}) set(SQL_INCLUDE_DIR ${BRPC_INCLUDE_PATH}) ENDIF() diff --git a/sql/main.cc b/sql/main.cc index eb52420eb52..962f805a6de 100644 --- a/sql/main.cc +++ b/sql/main.cc @@ -40,7 +40,6 @@ extern int mysqld_win_main(int argc, char **argv); extern int mysqld_main(int argc, char **argv); #endif -extern txservice::CatalogFactory *eloqsql_catalog_factory; int main(int argc, char **argv) { @@ -54,7 +53,7 @@ int main(int argc, char **argv) gflags::ParseCommandLineFlags(&argc, &argv, true); - DataSubstrate::InitializeGlobal(FLAGS_data_substrate_config); + // DataSubstrate::InitializeGlobal(FLAGS_data_substrate_config); #ifdef WITH_GLOG diff --git a/sql/mysqld.cc b/sql/mysqld.cc index dce912a542e..648bf03a10a 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -113,6 +113,9 @@ #include #endif +#include "glog/logging.h" +#include "data_substrate.h" + #include #include #include @@ -641,6 +644,8 @@ const char *current_dbug_option=""; // Declare gflags DECLARE_string(eloqsql_config); +DECLARE_bool(bootstrap); +DECLARE_string(data_substrate_config); // Global string to hold defaults file argument static std::string g_defaults_file_arg; @@ -5041,6 +5046,10 @@ static int init_server_components() /* It's now safe to use thread specific memory */ mysqld_server_initialized= 1; + // Wait for mysqld to initialize before initializing data substrate + // as data substrate may need to access some mysqld variables during replay. + DataSubstrate::InitializeGlobal(FLAGS_data_substrate_config); + #ifndef EMBEDDED_LIBRARY wsrep_thr_init(); #endif @@ -5556,18 +5565,44 @@ int mysqld_main(int argc, char **argv) my_defaults_mark_files= TRUE; // Build argv for load_defaults - // If --eloqsql_config specified, pass as --defaults-file - char *load_default_argv[3]; - int load_default_argc = 1; - load_default_argv[0] = argv[0]; + // Copy all original arguments first, then append --eloqsql_config and --bootstrap if needed + int additional_args = 0; + if (!FLAGS_eloqsql_config.empty()) + additional_args++; + if (FLAGS_bootstrap) + additional_args++; + + char **load_default_argv = (char**)malloc((argc + additional_args + 1) * sizeof(char*)); + if (!load_default_argv) + { + fprintf(stderr, "Failed to allocate memory for load_default_argv\n"); + return 1; + } + // Copy all original arguments + int load_default_argc = argc; + for (int i = 0; i < argc; i++) + { + load_default_argv[i] = argv[i]; + } + + // Append --defaults-file if --eloqsql_config specified if (!FLAGS_eloqsql_config.empty()) { - // Pass as --defaults-file=path g_defaults_file_arg = std::string("--defaults-file=") + FLAGS_eloqsql_config; - load_default_argv[1] = const_cast(g_defaults_file_arg.c_str()); - load_default_argc = 2; + load_default_argv[load_default_argc] = const_cast(g_defaults_file_arg.c_str()); + load_default_argc++; } + + // Append --bootstrap if FLAGS_bootstrap is true + if (FLAGS_bootstrap) + { + load_default_argv[load_default_argc] = const_cast("--bootstrap"); + load_default_argc++; + } + + // Null-terminate the array + load_default_argv[load_default_argc] = nullptr; char **load_default_argv_ptr = load_default_argv; load_defaults_or_exit(MYSQL_CONFIG_NAME, load_default_groups, @@ -5968,6 +6003,9 @@ int mysqld_main(int argc, char **argv) close_connections(); ha_pre_shutdown(); clean_up(1); + + // Free the temporary argv array we allocated + free(load_default_argv); sd_notify(0, "STATUS=MariaDB server is down"); /* (void) pthread_attr_destroy(&connection_attrib); */ @@ -7800,6 +7838,7 @@ static int mysql_init_variables(void) mysql_real_data_home_len= (uint)(strmake_buf(mysql_real_data_home, get_relative_path(MYSQL_DATADIR)) - mysql_real_data_home); + LOG(INFO) << "default mysql data home " << mysql_real_data_home; /* Replication parameters */ master_info_file= (char*) "master.info", relay_log_info_file= (char*) "relay-log.info"; @@ -7986,6 +8025,7 @@ mysqld_get_one_option(const struct my_option *opt, const char *argument, break; case 'h': strmake_buf(mysql_real_data_home, argument); + LOG(INFO) << "mysql data homne " << mysql_real_data_home; /* Correct pointer set by my_getopt (for embedded library) */ mysql_real_data_home_ptr= mysql_real_data_home; break; diff --git a/storage/eloq/CMakeLists.txt b/storage/eloq/CMakeLists.txt index 3e324fc67c8..8888b94ed58 100644 --- a/storage/eloq/CMakeLists.txt +++ b/storage/eloq/CMakeLists.txt @@ -209,7 +209,6 @@ set(STATISTICS ON CACHE BOOL "Turn on STATISTICS") if(ELOQ_MODULE_ENABLED) add_compile_definitions(ELOQ_MODULE_ENABLED) - message("ADDING ELOQ MODULE ENABLED") endif() if (EXT_TX_PROC_ENABLED) diff --git a/storage/eloq/eloq_db.cpp b/storage/eloq/eloq_db.cpp index 121115ca98c..da13ed801ab 100644 --- a/storage/eloq/eloq_db.cpp +++ b/storage/eloq/eloq_db.cpp @@ -593,17 +593,20 @@ static int load_view_fetch_func() if (exists) { eloq_fetch_view_impl= eloq_fetch_view_normal; + LOG(INFO) << "eloq_fetch_view_impl= eloq_fetch_view_normal"; ret= 0; } else { eloq_fetch_view_impl= eloq_fetch_view_bootstrap; + LOG(INFO) << "eloq_fetch_view_impl= eloq_fetch_view_bootstrap"; ret= HA_ERR_KEY_NOT_FOUND; } } else { eloq_fetch_view_impl= eloq_fetch_view_error; + LOG(INFO) << "eloq_fetch_view_impl= eloq_fetch_view_error"; ret= HA_ERR_INTERNAL_ERROR; } return ret; @@ -613,6 +616,7 @@ static int eloq_fetch_view_bootstrap(THD *thd, LEX_CSTRING db, LEX_CSTRING view, LEX_CSTRING &frm_binary) { DBUG_ENTER_FUNC(); + LOG(INFO) << "eloq_fetch_view_bootstrap " << db.str << " " << view.str << ", opt bootstrap: " << opt_bootstrap; int err= load_view_fetch_func(); if (err == 0) { diff --git a/storage/eloq/eloq_db_dl.h b/storage/eloq/eloq_db_dl.h index 0009cb4c504..86387af1561 100644 --- a/storage/eloq/eloq_db_dl.h +++ b/storage/eloq/eloq_db_dl.h @@ -56,27 +56,27 @@ static LEX_CSTRING ELOQ_ENGINE_NAME= {STRING_WITH_LEN("ELOQ")}; // #if defined(ELOQ_LINK_STATIC) // Statically linked: declare the symbol and let the linker resolve it to the // implementation provided in the static library (see eloq_db.cpp extern "C" block). -// #define MONOCALL_DEFINEx(x, name, ...) \ -// extern "C" int name(__SC_DECL##x(__VA_ARGS__)); +#define MONOCALL_DEFINEx(x, name, ...) \ + extern "C" int name(__SC_DECL##x(__VA_ARGS__)); // #else -#define MONOCALL_DEFINEx(x, name, ...) \ - static inline int name(__SC_DECL##x(__VA_ARGS__)) \ - { \ - using T= decltype(&name); \ - \ - int ret= 0; \ - \ - static_assert(std::is_same::value); \ - static THD *t= __ARG1(__VA_ARGS__); \ - \ - static plugin_ref pi= ha_resolve_by_name(t, &ELOQ_ENGINE_NAME, false); \ - \ - static void *handle= plugin_handle(pi); \ - static T fp= reinterpret_cast(dlsym(handle, __FUNCTION__)); \ - ret= fp(__SC_CAST##x(__VA_ARGS__)); \ - \ - return ret; \ - } +// #define MONOCALL_DEFINEx(x, name, ...) \ +// static inline int name(__SC_DECL##x(__VA_ARGS__)) \ +// { \ +// using T= decltype(&name); \ +// \ +// int ret= 0; \ +// \ +// static_assert(std::is_same::value); \ +// static THD *t= __ARG1(__VA_ARGS__); \ +// \ +// static plugin_ref pi= ha_resolve_by_name(t, &ELOQ_ENGINE_NAME, false); \ +// \ +// static void *handle= plugin_handle(pi); \ +// static T fp= reinterpret_cast(dlsym(handle, __FUNCTION__)); \ +// ret= fp(__SC_CAST##x(__VA_ARGS__)); \ +// \ +// return ret; \ +// } // #endif #define MONOCALL_DEFINE1(name, ...) MONOCALL_DEFINEx(1, name, __VA_ARGS__) From f0a77a25f9d5b23ec8a2c1a60fec3272f1ea533e Mon Sep 17 00:00:00 2001 From: liunyl Date: Wed, 22 Oct 2025 07:19:48 +0000 Subject: [PATCH 04/10] rename eloq files, add build as lib option --- CMakeLists.txt | 27 ++++++++-- data_substrate/core/src/data_substrate.cpp | 6 +-- sql/CMakeLists.txt | 20 +++++++- sql/glog_error_logging.cc | 6 +-- sql/mysqld.cc | 35 +++++++++++++ sql/mysqld_init.h | 25 ++++++++++ storage/eloq/CMakeLists.txt | 50 +++++++++---------- storage/eloq/eloq_key_def.cc | 2 +- ...actory.cpp => eloqsql_catalog_factory.cpp} | 4 +- ...og_factory.h => eloqsql_catalog_factory.h} | 4 +- storage/eloq/{eloq_key.h => eloqsql_key.h} | 0 .../{eloq_schema.cpp => eloqsql_schema.cpp} | 4 +- .../eloq/{eloq_schema.h => eloqsql_schema.h} | 0 storage/eloq/ha_eloq.cc | 2 +- storage/eloq/ha_eloq.h | 6 +-- storage/innobase/CMakeLists.txt | 2 +- 16 files changed, 144 insertions(+), 49 deletions(-) create mode 100644 sql/mysqld_init.h rename storage/eloq/{eloq_catalog_factory.cpp => eloqsql_catalog_factory.cpp} (99%) rename storage/eloq/{eloq_catalog_factory.h => eloqsql_catalog_factory.h} (99%) rename storage/eloq/{eloq_key.h => eloqsql_key.h} (100%) rename storage/eloq/{eloq_schema.cpp => eloqsql_schema.cpp} (99%) rename storage/eloq/{eloq_schema.h => eloqsql_schema.h} (100%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0ec3205aacb..5e3d3f270bf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -94,10 +94,18 @@ ELSE() SET(CMAKE_CXX_EXTENSIONS OFF) ENDIF() +# Option to build as library for converged binary +option(BUILD_ELOQSQL_AS_LIBRARY "Build eloqsql as library instead of executable" OFF) + +IF(BUILD_ELOQSQL_AS_LIBRARY) + SET(CMAKE_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) +ENDIF() + SET(CPACK_PACKAGE_NAME "MariaDB") SET(CPACK_PACKAGE_DESCRIPTION_SUMMARY "MariaDB: a very fast and robust SQL database server") SET(CPACK_PACKAGE_URL "http://mariadb.org") +option(WITH_DYNAMO_DB "With DynamoDB" OFF) IF(BUILD_CONFIG) INCLUDE( ${CMAKE_SOURCE_DIR}/cmake/build_configurations/${BUILD_CONFIG}.cmake) @@ -150,6 +158,8 @@ IF (NOT CPACK_GENERATOR) ENDIF(WIN32) ENDIF(NOT CPACK_GENERATOR) +SET(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_CURRENT_SOURCE_DIR}/cmake) + INCLUDE(FeatureSummary) INCLUDE(misc) INCLUDE(mysql_version) @@ -455,9 +465,16 @@ CONFIGURE_PLUGINS() # Build data substrate library first (before sql/ and storage/) -# IF(ELOQ_MODULE_ENABLED) - ADD_SUBDIRECTORY(data_substrate) -# ENDIF() +if(NOT BUILD_ELOQSQL_AS_LIBRARY) + # Build data substrate library first (standalone mode) + ADD_SUBDIRECTORY(data_substrate) +else() + # Use parent data_substrate (converged mode) + message(STATUS "EloqSQL: Using parent data_substrate") + # When building as library, mysqld will NOT initialize data substrate + # It will use synchronization with converged main instead + add_compile_definitions(MYSQLD_LIBRARY_MODE) +endif() ADD_SUBDIRECTORY(include) ADD_SUBDIRECTORY(dbug) ADD_SUBDIRECTORY(strings) @@ -625,4 +642,6 @@ IF(NOT WITHOUT_SERVER) ADD_DEPENDENCIES(smoketest minbuild) ENDIF() -option(WITH_DYNAMO_DB "With DynamoDB" OFF) +IF(BUILD_ELOQSQL_AS_LIBRARY) + SET(CMAKE_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../) +ENDIF() diff --git a/data_substrate/core/src/data_substrate.cpp b/data_substrate/core/src/data_substrate.cpp index 4bd08cb604c..47c67fdfd42 100644 --- a/data_substrate/core/src/data_substrate.cpp +++ b/data_substrate/core/src/data_substrate.cpp @@ -66,9 +66,9 @@ DEFINE_bool( bind_all, false, "Listen on all interfaces if enabled, otherwise listen on local.ip"); DEFINE_uint32(maxclients, 10000, "maxclients"); -// DEFINE_string(log_file_name_prefix, -// "eloqdb.log", -// "Sets the prefix for log files. Default is 'eloqdb.log'"); +DEFINE_string(log_file_name_prefix, + "eloqdb.log", + "Sets the prefix for log files. Default is 'eloqdb.log'"); // Declare as weak symbols to allow optional linking extern txservice::CatalogFactory *eloqsql_catalog_factory __attribute__((weak)); extern txservice::CatalogFactory *eloqkv_catalog_factory __attribute__((weak)); diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt index be3c91bd6e9..842a3106542 100644 --- a/sql/CMakeLists.txt +++ b/sql/CMakeLists.txt @@ -302,7 +302,7 @@ IF(MARIA_WITH_GLOG) endif() include_directories(${GLOG_INCLUDE_PATH}) set(SQL_DEP_LIBS ${SQL_DEP_LIBS} ${GLOG_LIB} gflags) - set(SQL_SOURCE ${SQL_SOURCE} glog_error_logging.cc) + #set(SQL_SOURCE ${SQL_SOURCE} glog_error_logging.cc) ENDIF() ADD_LIBRARY(sql STATIC ${SQL_SOURCE}) @@ -384,7 +384,22 @@ ENDIF() ADD_LIBRARY(sql_builtins STATIC ${CMAKE_CURRENT_BINARY_DIR}/sql_builtin.cc) TARGET_LINK_LIBRARIES(sql_builtins ${MYSQLD_STATIC_PLUGIN_LIBS}) -MYSQL_ADD_EXECUTABLE(mariadbd ${MYSQLD_SOURCE} DESTINATION ${INSTALL_SBINDIR} COMPONENT Server) +# Export libraries for converged build +if(BUILD_ELOQSQL_AS_LIBRARY) + # Make sql and sql_builtins available to parent + set(ELOQSQL_SQL_LIBRARY sql PARENT_SCOPE) + set(ELOQSQL_SQL_BUILTINS_LIBRARY sql_builtins PARENT_SCOPE) + set(ELOQSQL_INCLUDE_DIRS + ${CMAKE_SOURCE_DIR}/include + ${CMAKE_SOURCE_DIR}/sql + ${CMAKE_BINARY_DIR}/include + PARENT_SCOPE + ) +endif() + +# Make executable optional +if(NOT BUILD_ELOQSQL_AS_LIBRARY) + MYSQL_ADD_EXECUTABLE(mariadbd ${MYSQLD_SOURCE} DESTINATION ${INSTALL_SBINDIR} COMPONENT Server) IF(APPLE) # Add CoreServices framework since some dloadable plugins may need it @@ -432,6 +447,7 @@ IF(WITH_MYSQLD_LDFLAGS) SET_TARGET_PROPERTIES(mariadbd PROPERTIES LINK_FLAGS "${MYSQLD_LINK_FLAGS} ${WITH_MYSQLD_LDFLAGS}") ENDIF() +endif() # End of BUILD_ELOQSQL_AS_LIBRARY conditional # Handle out-of-source build from source package with possibly broken diff --git a/sql/glog_error_logging.cc b/sql/glog_error_logging.cc index e44a718c509..12da20ce85a 100644 --- a/sql/glog_error_logging.cc +++ b/sql/glog_error_logging.cc @@ -1,6 +1,6 @@ #if WITH_GLOG -#include +// #include -DEFINE_string(log_file_name_prefix, "mysqld.log", - "Sets the prefix for log files. Default is 'mysqld.log'"); +// DEFINE_string(log_file_name_prefix, "mysqld.log", +// "Sets the prefix for log files. Default is 'mysqld.log'"); #endif diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 648bf03a10a..e37163f9c9f 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -107,6 +107,19 @@ #include "semisync_master.h" #include "semisync_slave.h" +#ifdef MYSQLD_LIBRARY_MODE +#include "mysqld_init.h" + +// Define synchronization variables +namespace mysqld_converged_sync { + std::mutex init_mutex; + std::condition_variable mysqld_basic_init_done_cv; + std::condition_variable data_substrate_init_done_cv; + bool mysqld_basic_init_done = false; + bool data_substrate_init_done = false; +} +#endif + #include "transaction.h" #ifdef HAVE_SYS_PRCTL_H @@ -5046,9 +5059,31 @@ static int init_server_components() /* It's now safe to use thread specific memory */ mysqld_server_initialized= 1; +#ifdef MYSQLD_LIBRARY_MODE + // In library mode (converged binary), use synchronization: + // 1. Signal that MySQL basic initialization is done + // 2. Wait for data substrate to be initialized by converged main + { + std::unique_lock lock(mysqld_converged_sync::init_mutex); + mysqld_converged_sync::mysqld_basic_init_done = true; + mysqld_converged_sync::mysqld_basic_init_done_cv.notify_one(); + + LOG(INFO) << "MySQL basic initialization complete, waiting for data substrate..."; + + // Wait for data substrate initialization to complete + mysqld_converged_sync::data_substrate_init_done_cv.wait(lock, [] { + return mysqld_converged_sync::data_substrate_init_done; + }); + + LOG(INFO) << "Data substrate initialized, MySQL continuing..."; + } +#else + // In standalone mode, initialize data substrate here // Wait for mysqld to initialize before initializing data substrate // as data substrate may need to access some mysqld variables during replay. + LOG(INFO) << "Standalone mode: Initializing data substrate..."; DataSubstrate::InitializeGlobal(FLAGS_data_substrate_config); +#endif #ifndef EMBEDDED_LIBRARY wsrep_thr_init(); diff --git a/sql/mysqld_init.h b/sql/mysqld_init.h new file mode 100644 index 00000000000..b23f540945d --- /dev/null +++ b/sql/mysqld_init.h @@ -0,0 +1,25 @@ +#ifndef MYSQLD_INIT_INCLUDED +#define MYSQLD_INIT_INCLUDED + +#include +#include + +// Synchronization for converged binary initialization sequence +// This ensures proper order: MySQL basic init → data substrate init → MySQL rest +namespace mysqld_converged_sync { + // Mutex and condition variables for initialization synchronization + extern std::mutex init_mutex; + extern std::condition_variable mysqld_basic_init_done_cv; + extern std::condition_variable data_substrate_init_done_cv; + extern bool mysqld_basic_init_done; + extern bool data_substrate_init_done; +} + +// Function to initialize and run mysqld in library mode +// Returns 0 on success, non-zero on error +extern "C" int mysqld_main(int argc, char **argv); + +// Function to shutdown mysqld gracefully +extern "C" void mysqld_shutdown(); + +#endif // MYSQLD_INIT_INCLUDED diff --git a/storage/eloq/CMakeLists.txt b/storage/eloq/CMakeLists.txt index 8888b94ed58..25341cca3f9 100644 --- a/storage/eloq/CMakeLists.txt +++ b/storage/eloq/CMakeLists.txt @@ -226,27 +226,27 @@ endif() find_package(MIMALLOC REQUIRED) include_directories(${MIMALLOC_INCLUDE_DIR}) -include_directories( - ${CMAKE_CURRENT_SOURCE_DIR} - ${CMAKE_SOURCE_DIR}/data_substrate/ - ${CMAKE_SOURCE_DIR}/data_substrate/core/include - ${CMAKE_SOURCE_DIR}/data_substrate/tx_service/include - ${CMAKE_SOURCE_DIR}/data_substrate/tx_service/include/cc - ${CMAKE_SOURCE_DIR}/data_substrate/tx_service/include/remote - ${CMAKE_SOURCE_DIR}/data_substrate/tx_service/include/fault - ${CMAKE_SOURCE_DIR}/data_substrate/tx_service/include/store - ${CMAKE_SOURCE_DIR}/data_substrate/tx_service/tx-log-protos - ${CMAKE_SOURCE_DIR}/data_substrate/eloq_metrics/include - ${CMAKE_SOURCE_DIR}/data_substrate/store_handler -) +# include_directories( +# ${CMAKE_CURRENT_SOURCE_DIR} +# ${CMAKE_SOURCE_DIR}/data_substrate/ +# ${CMAKE_SOURCE_DIR}/data_substrate/core/include +# ${CMAKE_SOURCE_DIR}/data_substrate/tx_service/include +# ${CMAKE_SOURCE_DIR}/data_substrate/tx_service/include/cc +# ${CMAKE_SOURCE_DIR}/data_substrate/tx_service/include/remote +# ${CMAKE_SOURCE_DIR}/data_substrate/tx_service/include/fault +# ${CMAKE_SOURCE_DIR}/data_substrate/tx_service/include/store +# ${CMAKE_SOURCE_DIR}/data_substrate/tx_service/tx-log-protos +# ${CMAKE_SOURCE_DIR}/data_substrate/eloq_metrics/include +# ${CMAKE_SOURCE_DIR}/data_substrate/store_handler +# ) -if(WITH_LOG_SERVICE) - if (OPEN_LOG_SERVICE) - include_directories(${CMAKE_SOURCE_DIR}/data_substrate/log_service/include) - else() - include_directories(${CMAKE_SOURCE_DIR}/data_substrate/eloq_log_service/include) - endif() -endif() +# if(WITH_LOG_SERVICE) +# if (OPEN_LOG_SERVICE) +# include_directories(${CMAKE_SOURCE_DIR}/data_substrate/log_service/include) +# else() +# include_directories(${CMAKE_SOURCE_DIR}/data_substrate/eloq_log_service/include) +# endif() +# endif() option(SMALL_RANGE "Whether enable small range" OFF) message(NOTICE "SMALL_RANGE : ${SMALL_RANGE}") @@ -259,11 +259,11 @@ SET(MYELOQ_SOURCES eloq_i_s.cc eloq_errors.h eloq_errors.cpp - eloq_key.h - eloq_schema.h - eloq_schema.cpp - eloq_catalog_factory.h - eloq_catalog_factory.cpp + eloqsql_key.h + eloqsql_schema.h + eloqsql_schema.cpp + eloqsql_catalog_factory.h + eloqsql_catalog_factory.cpp eloq_db.cpp eloq_buff.h eloq_key_def.h diff --git a/storage/eloq/eloq_key_def.cc b/storage/eloq/eloq_key_def.cc index 23c59f41c8e..76a9d57d631 100644 --- a/storage/eloq/eloq_key_def.cc +++ b/storage/eloq/eloq_key_def.cc @@ -64,7 +64,7 @@ #include "sql_table.h" #include "log.h" #include "mysqld_error.h" -#include "eloq_key.h" +#include "eloqsql_key.h" #define HA_EXIT_FAILURE 1 #define HA_EXIT_SUCCESS 0 diff --git a/storage/eloq/eloq_catalog_factory.cpp b/storage/eloq/eloqsql_catalog_factory.cpp similarity index 99% rename from storage/eloq/eloq_catalog_factory.cpp rename to storage/eloq/eloqsql_catalog_factory.cpp index 23f1a619e3b..2dce4fe528e 100644 --- a/storage/eloq/eloq_catalog_factory.cpp +++ b/storage/eloq/eloqsql_catalog_factory.cpp @@ -18,7 +18,7 @@ #include "type.h" #define MYSQL_SERVER 1 -#include "eloq_catalog_factory.h" +#include "eloqsql_catalog_factory.h" #include #include @@ -29,7 +29,7 @@ #include "sql_class.h" #include "store_handler/kv_store.h" -#include "eloq_key.h" +#include "eloqsql_key.h" #include "cc/template_cc_map.h" #include "constants.h" diff --git a/storage/eloq/eloq_catalog_factory.h b/storage/eloq/eloqsql_catalog_factory.h similarity index 99% rename from storage/eloq/eloq_catalog_factory.h rename to storage/eloq/eloqsql_catalog_factory.h index fa7ba37ed43..6d71258e029 100644 --- a/storage/eloq/eloq_catalog_factory.h +++ b/storage/eloq/eloqsql_catalog_factory.h @@ -24,8 +24,8 @@ #include "tx_service/include/catalog_factory.h" #include "tx_service/include/store/data_store_handler.h" -#include "eloq_key.h" -#include "eloq_schema.h" +#include "eloqsql_key.h" +#include "eloqsql_schema.h" #include "ha_eloq_macro.h" #include "tx_service/include/table_statistics.h" #include "tx_service/include/type.h" // TableName diff --git a/storage/eloq/eloq_key.h b/storage/eloq/eloqsql_key.h similarity index 100% rename from storage/eloq/eloq_key.h rename to storage/eloq/eloqsql_key.h diff --git a/storage/eloq/eloq_schema.cpp b/storage/eloq/eloqsql_schema.cpp similarity index 99% rename from storage/eloq/eloq_schema.cpp rename to storage/eloq/eloqsql_schema.cpp index 37ac5d629b3..635f851541b 100644 --- a/storage/eloq/eloq_schema.cpp +++ b/storage/eloq/eloqsql_schema.cpp @@ -22,9 +22,9 @@ #include #include "eloq_key_def.h" -#include "eloq_schema.h" +#include "eloqsql_schema.h" #include "date.h" -#include "eloq_key.h" +#include "eloqsql_key.h" #include "tx_record.h" #include "butil/logging.h" diff --git a/storage/eloq/eloq_schema.h b/storage/eloq/eloqsql_schema.h similarity index 100% rename from storage/eloq/eloq_schema.h rename to storage/eloq/eloqsql_schema.h diff --git a/storage/eloq/ha_eloq.cc b/storage/eloq/ha_eloq.cc index 4b6f2ba6d26..32d2be7d4ce 100644 --- a/storage/eloq/ha_eloq.cc +++ b/storage/eloq/ha_eloq.cc @@ -141,7 +141,7 @@ #include "mysql/service_thd_wait.h" #include "eloq_i_s.h" -#include "eloq_catalog_factory.h" +#include "eloqsql_catalog_factory.h" #include "eloq_catalog_name.h" #include "eloq_system_handler.h" #include "eloq_errors.h" diff --git a/storage/eloq/ha_eloq.h b/storage/eloq/ha_eloq.h index f1a8b035604..029bddbbc32 100644 --- a/storage/eloq/ha_eloq.h +++ b/storage/eloq/ha_eloq.h @@ -62,14 +62,14 @@ #include #include "ha_eloq_macro.h" -#include "eloq_key.h" -#include "eloq_schema.h" +#include "eloqsql_key.h" +#include "eloqsql_schema.h" #include "tx_service/include/statistics.h" #include "tx_service/include/tx_request.h" #include "tx_service/include/tx_execution.h" #include "tx_service/include/tx_util.h" #include "tx_service/include/store/data_store_scanner.h" -#include "eloq_catalog_factory.h" +#include "eloqsql_catalog_factory.h" #include "eloq_buff.h" #include "eloq_key_def.h" #include "tx_service/include/cc/reader_writer_cntl.h" diff --git a/storage/innobase/CMakeLists.txt b/storage/innobase/CMakeLists.txt index cc1ec1e302a..35fcf1bf6d8 100644 --- a/storage/innobase/CMakeLists.txt +++ b/storage/innobase/CMakeLists.txt @@ -400,7 +400,7 @@ ENDIF() IF(NOT (PLUGIN_INNOBASE STREQUAL DYNAMIC)) TARGET_LINK_LIBRARIES(innobase tpool mysys) - ADD_SUBDIRECTORY(${CMAKE_SOURCE_DIR}/extra/mariabackup ${CMAKE_BINARY_DIR}/extra/mariabackup) + ADD_SUBDIRECTORY(${CURRENT_SOURCE_DIR}/extra/mariabackup ${CMAKE_BINARY_DIR}/extra/mariabackup) ENDIF() IF(WITH_UNIT_TESTS) From ca5aab8742380e8b949906094168a23b48895726 Mon Sep 17 00:00:00 2001 From: liunyl Date: Wed, 22 Oct 2025 10:56:10 +0000 Subject: [PATCH 05/10] Override cmake source dir with my source dir --- CMakeLists.txt | 63 ++--- client/CMakeLists.txt | 18 +- cmake/abi_check.cmake | 40 +-- .../build_configurations/mysql_release.cmake | 2 +- cmake/cpack_deb.cmake | 4 +- cmake/cpack_rpm.cmake | 32 +-- cmake/ctest.cmake | 4 +- cmake/dtrace.cmake | 34 +-- cmake/info_macros.cmake.in | 22 +- cmake/install_macros.cmake | 10 +- cmake/libutils.cmake | 2 +- cmake/make_dist.cmake.in | 58 ++-- cmake/mysql_add_executable.cmake | 2 +- cmake/mysql_version.cmake | 14 +- cmake/pcre.cmake | 2 +- cmake/plugin.cmake | 14 +- cmake/readline.cmake | 4 +- cmake/sign.cmake.in | 2 +- cmake/ssl.cmake | 6 +- cmake/submodules.cmake | 12 +- cmake/tags.cmake | 4 +- cmake/wsrep.cmake | 8 +- cmake/zlib.cmake | 2 +- configure.cmake | 6 +- dbug/CMakeLists.txt | 4 +- extra/CMakeLists.txt | 24 +- extra/mariabackup/CMakeLists.txt | 4 +- extra/readline/CMakeLists.txt | 2 +- libmysqld/CMakeLists.txt | 22 +- libmysqld/examples/CMakeLists.txt | 6 +- libservices/CMakeLists.txt | 2 +- mysql-test/CMakeLists.txt | 4 +- mysql-test/lib/My/SafeProcess/CMakeLists.txt | 4 +- mysql-test/mtr.out-of-source | 6 +- mysys/CMakeLists.txt | 2 +- mysys_ssl/CMakeLists.txt | 4 +- plugin/auth_dialog/CMakeLists.txt | 2 +- plugin/disks/CMakeLists.txt | 2 +- plugin/feedback/CMakeLists.txt | 2 +- plugin/file_key_management/CMakeLists.txt | 2 +- plugin/locale_info/CMakeLists.txt | 2 +- plugin/qc_info/CMakeLists.txt | 2 +- plugin/type_geom/CMakeLists.txt | 2 +- plugin/userstat/CMakeLists.txt | 2 +- plugin/wsrep_info/CMakeLists.txt | 4 +- scripts/CMakeLists.txt | 6 +- scripts/sys_schema/CMakeLists.txt | 2 +- sql/CMakeLists.txt | 56 ++-- sql/circular_queue.h | 258 ++++++++++++++++++ sql/mdl.h | 2 +- sql/mysql_metrics.h | 4 +- sql/mysqld.cc | 8 +- sql/mysqld_init.h | 25 -- sql/sql_class.h | 3 +- sql/threadpool_generic.h | 2 +- storage/eloq/CMakeLists.txt | 24 +- storage/eloq/eloq_key_def.cc | 2 +- storage/eloq/eloq_key_def.h | 2 +- storage/innobase/CMakeLists.txt | 2 +- storage/innobase/innodb.cmake | 10 +- storage/innobase/unittest/CMakeLists.txt | 8 +- storage/maria/unittest/CMakeLists.txt | 4 +- storage/mroonga/CMakeLists.txt | 12 +- storage/mroonga/vendor/groonga/CMakeLists.txt | 2 +- .../groonga-normalizer-mysql/CMakeLists.txt | 4 +- storage/perfschema/CMakeLists.txt | 6 +- storage/perfschema/unittest/CMakeLists.txt | 10 +- storage/rocksdb/unittest/CMakeLists.txt | 6 +- storage/spider/CMakeLists.txt | 8 +- strings/CMakeLists.txt | 2 +- tests/CMakeLists.txt | 8 +- unittest/embedded/CMakeLists.txt | 6 +- unittest/json_lib/CMakeLists.txt | 8 +- unittest/my_decimal/CMakeLists.txt | 8 +- unittest/mytap/CMakeLists.txt | 2 +- unittest/sql/CMakeLists.txt | 8 +- vio/CMakeLists.txt | 2 +- win/packaging/CMakeLists.txt | 10 +- win/packaging/ca/CMakeLists.txt | 4 +- win/upgrade_wizard/CMakeLists.txt | 4 +- 80 files changed, 614 insertions(+), 383 deletions(-) create mode 100644 sql/circular_queue.h delete mode 100644 sql/mysqld_init.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 5e3d3f270bf..fd19baa01a9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -40,9 +40,16 @@ FOREACH(p CMP0022 CMP0046 CMP0040 CMP0048 CMP0054 CMP0075 CMP0069 CMP0135) ENDFOREACH() MESSAGE(STATUS "Running cmake version ${CMAKE_VERSION}") +# Option to build as library for converged binary +option(BUILD_ELOQSQL_AS_LIBRARY "Build eloqsql as library instead of executable" OFF) + +IF(BUILD_ELOQSQL_AS_LIBRARY) + SET(ELOQSQL_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) + SET(ELOQSQL_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}) +ENDIF() SET(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} - ${CMAKE_SOURCE_DIR}/cmake ${CMAKE_SOURCE_DIR}/cmake/Internal/CPack) + ${ELOQSQL_SOURCE_DIR}/cmake ${ELOQSQL_SOURCE_DIR}/cmake/Internal/CPack) # Use a default manufacturer if no manufacturer was identified. IF(NOT DEFINED MANUFACTURER) @@ -94,13 +101,6 @@ ELSE() SET(CMAKE_CXX_EXTENSIONS OFF) ENDIF() -# Option to build as library for converged binary -option(BUILD_ELOQSQL_AS_LIBRARY "Build eloqsql as library instead of executable" OFF) - -IF(BUILD_ELOQSQL_AS_LIBRARY) - SET(CMAKE_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) -ENDIF() - SET(CPACK_PACKAGE_NAME "MariaDB") SET(CPACK_PACKAGE_DESCRIPTION_SUMMARY "MariaDB: a very fast and robust SQL database server") SET(CPACK_PACKAGE_URL "http://mariadb.org") @@ -108,7 +108,7 @@ SET(CPACK_PACKAGE_URL "http://mariadb.org") option(WITH_DYNAMO_DB "With DynamoDB" OFF) IF(BUILD_CONFIG) INCLUDE( - ${CMAKE_SOURCE_DIR}/cmake/build_configurations/${BUILD_CONFIG}.cmake) + ${ELOQSQL_SOURCE_DIR}/cmake/build_configurations/${BUILD_CONFIG}.cmake) ENDIF() # Include the platform-specific file. To allow exceptions, this code @@ -121,7 +121,7 @@ FOREACH(_base ${CMAKE_SYSTEM_NAME}-${CMAKE_SYSTEM_VERSION}-${CMAKE_SYSTEM_PROCESSOR} ${CMAKE_SYSTEM_NAME}-${CMAKE_SYSTEM_VERSION} ${CMAKE_SYSTEM_NAME}) - SET(_file ${CMAKE_SOURCE_DIR}/cmake/os/${_base}.cmake) + SET(_file ${ELOQSQL_SOURCE_DIR}/cmake/os/${_base}.cmake) IF(EXISTS ${_file}) INCLUDE(${_file}) BREAK() @@ -158,8 +158,6 @@ IF (NOT CPACK_GENERATOR) ENDIF(WIN32) ENDIF(NOT CPACK_GENERATOR) -SET(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_CURRENT_SOURCE_DIR}/cmake) - INCLUDE(FeatureSummary) INCLUDE(misc) INCLUDE(mysql_version) @@ -526,15 +524,15 @@ IF(NOT CMAKE_CROSSCOMPILING OR DEFINED CMAKE_CROSSCOMPILING_EMULATOR) ENDIF() # minimal target to build only binaries for export ADD_CUSTOM_TARGET(import_executables DEPENDS ${EXPORTED}) - EXPORT(TARGETS ${EXPORTED} FILE ${CMAKE_BINARY_DIR}/import_executables.cmake) + EXPORT(TARGETS ${EXPORTED} FILE ${ELOQSQL_BINARY_DIR}/import_executables.cmake) ENDIF() -CONFIGURE_FILE(config.h.cmake ${CMAKE_BINARY_DIR}/include/my_config.h) -CONFIGURE_FILE(config.h.cmake ${CMAKE_BINARY_DIR}/include/config.h) -CONFIGURE_FILE(${CMAKE_SOURCE_DIR}/include/mysql_version.h.in - ${CMAKE_BINARY_DIR}/include/mysql_version.h ) -CONFIGURE_FILE(${CMAKE_SOURCE_DIR}/sql/sql_builtin.cc.in - ${CMAKE_BINARY_DIR}/sql/sql_builtin.cc) +CONFIGURE_FILE(config.h.cmake ${ELOQSQL_BINARY_DIR}/include/my_config.h) +CONFIGURE_FILE(config.h.cmake ${ELOQSQL_BINARY_DIR}/include/config.h) +CONFIGURE_FILE(${ELOQSQL_SOURCE_DIR}/include/mysql_version.h.in + ${ELOQSQL_BINARY_DIR}/include/mysql_version.h ) +CONFIGURE_FILE(${ELOQSQL_SOURCE_DIR}/sql/sql_builtin.cc.in + ${ELOQSQL_BINARY_DIR}/sql/sql_builtin.cc) IF(GIT_EXECUTABLE AND EXISTS ${PROJECT_SOURCE_DIR}/.git) EXECUTE_PROCESS( @@ -552,30 +550,30 @@ IF(SOURCE_REVISION OR ENDIF() CONFIGURE_FILE( - ${CMAKE_SOURCE_DIR}/cmake/info_macros.cmake.in - ${CMAKE_BINARY_DIR}/info_macros.cmake @ONLY) + ${ELOQSQL_SOURCE_DIR}/cmake/info_macros.cmake.in + ${ELOQSQL_BINARY_DIR}/info_macros.cmake @ONLY) # Handle the "INFO_*" files. -INCLUDE(${CMAKE_BINARY_DIR}/info_macros.cmake) +INCLUDE(${ELOQSQL_BINARY_DIR}/info_macros.cmake) # Source: This can be done during the cmake phase, all information is # available, but should be repeated on each "make" just in case someone # does "cmake ; make ; git pull ; make". -CREATE_INFO_SRC(${CMAKE_BINARY_DIR}/Docs) +CREATE_INFO_SRC(${ELOQSQL_BINARY_DIR}/Docs) ADD_CUSTOM_TARGET(INFO_SRC ALL - COMMAND ${CMAKE_COMMAND} -P ${CMAKE_SOURCE_DIR}/cmake/info_src.cmake - WORKING_DIRECTORY ${CMAKE_BINARY_DIR} + COMMAND ${CMAKE_COMMAND} -P ${ELOQSQL_SOURCE_DIR}/cmake/info_src.cmake + WORKING_DIRECTORY ${ELOQSQL_BINARY_DIR} ) # Build flags: This must be postponed to the make phase. ADD_CUSTOM_TARGET(INFO_BIN ALL - COMMAND ${CMAKE_COMMAND} -P ${CMAKE_SOURCE_DIR}/cmake/info_bin.cmake - WORKING_DIRECTORY ${CMAKE_BINARY_DIR} + COMMAND ${CMAKE_COMMAND} -P ${ELOQSQL_SOURCE_DIR}/cmake/info_bin.cmake + WORKING_DIRECTORY ${ELOQSQL_BINARY_DIR} ) INSTALL_DOCUMENTATION(README.md CREDITS COPYING THIRDPARTY COMPONENT Readme) # MDEV-6526 these files are not installed anymore -#INSTALL_DOCUMENTATION(${CMAKE_BINARY_DIR}/Docs/INFO_SRC -# ${CMAKE_BINARY_DIR}/Docs/INFO_BIN) +#INSTALL_DOCUMENTATION(${ELOQSQL_BINARY_DIR}/Docs/INFO_SRC +# ${ELOQSQL_BINARY_DIR}/Docs/INFO_BIN) IF(UNIX) INSTALL_DOCUMENTATION(Docs/INSTALL-BINARY Docs/README-wsrep COMPONENT Readme) @@ -604,7 +602,7 @@ IF(NON_DISTRIBUTABLE_WARNING) You have linked MariaDB with ${NON_DISTRIBUTABLE_WARNING} libraries! You may not distribute the resulting binary. If you do, you will put yourself into a legal problem with the Free Software Foundation.") ENDIF() -IF(NOT WITHOUT_SERVER) +IF(NOT WITHOUT_SERVER AND NOT BUILD_ELOQSQL_AS_LIBRARY) # Define target for minimal mtr-testable build ADD_CUSTOM_TARGET(minbuild) ADD_DEPENDENCIES(minbuild @@ -638,10 +636,7 @@ IF(NOT WITHOUT_SERVER) ENDIF() ADD_CUSTOM_TARGET(smoketest COMMAND perl ./mysql-test-run.pl main.1st - WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/mysql-test) + WORKING_DIRECTORY ${ELOQSQL_BINARY_DIR}/mysql-test) ADD_DEPENDENCIES(smoketest minbuild) ENDIF() -IF(BUILD_ELOQSQL_AS_LIBRARY) - SET(CMAKE_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../) -ENDIF() diff --git a/client/CMakeLists.txt b/client/CMakeLists.txt index ac718ea8e26..8b91bca8e70 100644 --- a/client/CMakeLists.txt +++ b/client/CMakeLists.txt @@ -15,20 +15,20 @@ # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA INCLUDE_DIRECTORIES( - ${CMAKE_SOURCE_DIR}/include + ${ELOQSQL_SOURCE_DIR}/include ${PCRE_INCLUDES} - ${CMAKE_SOURCE_DIR}/mysys_ssl + ${ELOQSQL_SOURCE_DIR}/mysys_ssl ${ZLIB_INCLUDE_DIR} ${SSL_INCLUDE_DIRS} - ${CMAKE_SOURCE_DIR}/sql - ${CMAKE_SOURCE_DIR}/strings + ${ELOQSQL_SOURCE_DIR}/sql + ${ELOQSQL_SOURCE_DIR}/strings ${MY_READLINE_INCLUDE_DIR} ${CMAKE_CURRENT_BINARY_DIR} ) INCLUDE_DIRECTORIES(BEFORE - ${CMAKE_BINARY_DIR}/libmariadb/include - ${CMAKE_SOURCE_DIR}/libmariadb/include) + ${ELOQSQL_BINARY_DIR}/libmariadb/include + ${ELOQSQL_SOURCE_DIR}/libmariadb/include) ## We will need libeay32.dll and ssleay32.dll when running client executables. COPY_OPENSSL_DLLS(copy_openssl_client) @@ -37,7 +37,7 @@ SET(CLIENT_LIB mariadbclient mysys) ADD_DEFINITIONS(${SSL_DEFINES}) MYSQL_ADD_EXECUTABLE(mariadb completion_hash.cc mysql.cc readline.cc - ${CMAKE_SOURCE_DIR}/sql/sql_string.cc) + ${ELOQSQL_SOURCE_DIR}/sql/sql_string.cc) TARGET_LINK_LIBRARIES(mariadb ${CLIENT_LIB}) IF(UNIX) TARGET_LINK_LIBRARIES(mariadb ${MY_READLINE_LIBRARY}) @@ -48,7 +48,7 @@ ENDIF(UNIX) ADD_DEFINITIONS(-DWITH_ELOQ_STORAGE_ENGINE) # ENDIF() -MYSQL_ADD_EXECUTABLE(mariadb-test mysqltest.cc ${CMAKE_SOURCE_DIR}/sql/sql_string.cc COMPONENT Test) +MYSQL_ADD_EXECUTABLE(mariadb-test mysqltest.cc ${ELOQSQL_SOURCE_DIR}/sql/sql_string.cc COMPONENT Test) SET_SOURCE_FILES_PROPERTIES(mysqltest.cc PROPERTIES COMPILE_FLAGS "-DTHREADS ${PCRE2_DEBIAN_HACK}") TARGET_LINK_LIBRARIES(mariadb-test ${CLIENT_LIB} pcre2-posix pcre2-8) SET_TARGET_PROPERTIES(mariadb-test PROPERTIES ENABLE_EXPORTS TRUE) @@ -85,7 +85,7 @@ SET_SOURCE_FILES_PROPERTIES(mysqlslap.c PROPERTIES COMPILE_FLAGS "-DTHREADS") TARGET_LINK_LIBRARIES(mariadb-slap ${CLIENT_LIB}) MYSQL_ADD_EXECUTABLE(mariadb-conv mariadb-conv.cc - ${CMAKE_SOURCE_DIR}/sql/sql_string.cc) + ${ELOQSQL_SOURCE_DIR}/sql/sql_string.cc) TARGET_LINK_LIBRARIES(mariadb-conv mysys strings) IF(WIN32) diff --git a/cmake/abi_check.cmake b/cmake/abi_check.cmake index 98877be9351..e98fbe039d7 100644 --- a/cmake/abi_check.cmake +++ b/cmake/abi_check.cmake @@ -36,26 +36,26 @@ IF(CMAKE_C_COMPILER_ID MATCHES "GNU|Clang" AND RUN_ABI_CHECK) SET(COMPILER ${CMAKE_C_COMPILER}) ENDIF() SET(API_PREPROCESSOR_HEADER - ${CMAKE_SOURCE_DIR}/include/mysql/plugin_audit.h - ${CMAKE_SOURCE_DIR}/include/mysql/plugin_ftparser.h - ${CMAKE_SOURCE_DIR}/include/mysql.h - ${CMAKE_SOURCE_DIR}/include/mysql/psi/psi_abi_v1.h - ${CMAKE_SOURCE_DIR}/include/mysql/psi/psi_abi_v2.h - ${CMAKE_SOURCE_DIR}/include/mysql/client_plugin.h - ${CMAKE_SOURCE_DIR}/include/mysql/plugin_auth.h - ${CMAKE_SOURCE_DIR}/include/mysql/plugin_password_validation.h - ${CMAKE_SOURCE_DIR}/include/mysql/plugin_encryption.h - ${CMAKE_SOURCE_DIR}/include/mysql/plugin_data_type.h - ${CMAKE_SOURCE_DIR}/include/mysql/plugin_function.h + ${ELOQSQL_SOURCE_DIR}/include/mysql/plugin_audit.h + ${ELOQSQL_SOURCE_DIR}/include/mysql/plugin_ftparser.h + ${ELOQSQL_SOURCE_DIR}/include/mysql.h + ${ELOQSQL_SOURCE_DIR}/include/mysql/psi/psi_abi_v1.h + ${ELOQSQL_SOURCE_DIR}/include/mysql/psi/psi_abi_v2.h + ${ELOQSQL_SOURCE_DIR}/include/mysql/client_plugin.h + ${ELOQSQL_SOURCE_DIR}/include/mysql/plugin_auth.h + ${ELOQSQL_SOURCE_DIR}/include/mysql/plugin_password_validation.h + ${ELOQSQL_SOURCE_DIR}/include/mysql/plugin_encryption.h + ${ELOQSQL_SOURCE_DIR}/include/mysql/plugin_data_type.h + ${ELOQSQL_SOURCE_DIR}/include/mysql/plugin_function.h ) ADD_CUSTOM_TARGET(abi_check ALL COMMAND ${CMAKE_COMMAND} -DCOMPILER=${COMPILER} - -DSOURCE_DIR=${CMAKE_SOURCE_DIR} - -DBINARY_DIR=${CMAKE_BINARY_DIR} + -DSOURCE_DIR=${ELOQSQL_SOURCE_DIR} + -DBINARY_DIR=${ELOQSQL_BINARY_DIR} "-DABI_HEADERS=${API_PREPROCESSOR_HEADER}" - -P ${CMAKE_SOURCE_DIR}/cmake/do_abi_check.cmake + -P ${ELOQSQL_SOURCE_DIR}/cmake/do_abi_check.cmake VERBATIM ) @@ -63,20 +63,20 @@ IF(CMAKE_C_COMPILER_ID MATCHES "GNU|Clang" AND RUN_ABI_CHECK) COMMAND ${CMAKE_COMMAND} -DCOMPILER=${COMPILER} -DABI_UPDATE=1 - -DSOURCE_DIR=${CMAKE_SOURCE_DIR} - -DBINARY_DIR=${CMAKE_BINARY_DIR} + -DSOURCE_DIR=${ELOQSQL_SOURCE_DIR} + -DBINARY_DIR=${ELOQSQL_BINARY_DIR} "-DABI_HEADERS=${API_PREPROCESSOR_HEADER}" - -P ${CMAKE_SOURCE_DIR}/cmake/do_abi_check.cmake + -P ${ELOQSQL_SOURCE_DIR}/cmake/do_abi_check.cmake VERBATIM ) ADD_CUSTOM_TARGET(abi_check_all COMMAND ${CMAKE_COMMAND} -DCOMPILER=${COMPILER} - -DSOURCE_DIR=${CMAKE_SOURCE_DIR} - -DBINARY_DIR=${CMAKE_BINARY_DIR} + -DSOURCE_DIR=${ELOQSQL_SOURCE_DIR} + -DBINARY_DIR=${ELOQSQL_BINARY_DIR} "-DABI_HEADERS=${API_PREPROCESSOR_HEADER}" - -P ${CMAKE_SOURCE_DIR}/cmake/do_abi_check.cmake + -P ${ELOQSQL_SOURCE_DIR}/cmake/do_abi_check.cmake VERBATIM ) ENDIF() diff --git a/cmake/build_configurations/mysql_release.cmake b/cmake/build_configurations/mysql_release.cmake index 3a081073f72..93d8ee32c23 100644 --- a/cmake/build_configurations/mysql_release.cmake +++ b/cmake/build_configurations/mysql_release.cmake @@ -86,7 +86,7 @@ SET(WITH_INNODB_SNAPPY OFF CACHE STRING "") SET(WITH_NUMA 0 CACHE BOOL "") SET(CPU_LEVEL1_DCACHE_LINESIZE 0) -IF(NOT EXISTS ${CMAKE_SOURCE_DIR}/.git) +IF(NOT EXISTS ${ELOQSQL_SOURCE_DIR}/.git) SET(GIT_EXECUTABLE GIT_EXECUTABLE-NOTFOUND CACHE FILEPATH "") ENDIF() diff --git a/cmake/cpack_deb.cmake b/cmake/cpack_deb.cmake index b253f8f86c7..a3a01b8d3db 100644 --- a/cmake/cpack_deb.cmake +++ b/cmake/cpack_deb.cmake @@ -12,9 +12,9 @@ FUNCTION(SET_PLUGIN_DEB_VERSION plugin ver) STRING(REPLACE "_" "-" plugin ${plugin}) STRING(REPLACE "-" "." serverver ${SERVER_VERSION}) STRING(REPLACE ${SERVER_VERSION} ${serverver} ver ${ver}) - FILE(READ ${CMAKE_SOURCE_DIR}/debian/changelog changelog) + FILE(READ ${ELOQSQL_SOURCE_DIR}/debian/changelog changelog) STRING(REPLACE ${serverver} ${ver} changelog "${changelog}") - FILE(WRITE ${CMAKE_SOURCE_DIR}/debian/mariadb-plugin-${plugin}.changelog "${changelog}") + FILE(WRITE ${ELOQSQL_SOURCE_DIR}/debian/mariadb-plugin-${plugin}.changelog "${changelog}") ENDFUNCTION() ELSE(DEB) diff --git a/cmake/cpack_rpm.cmake b/cmake/cpack_rpm.cmake index f21c6325ff3..e07646c4d14 100644 --- a/cmake/cpack_rpm.cmake +++ b/cmake/cpack_rpm.cmake @@ -228,15 +228,15 @@ IF(WITH_WSREP) "coreutils" "findutils" "tar") ENDIF() -SET(CPACK_RPM_server_PRE_INSTALL_SCRIPT_FILE ${CMAKE_SOURCE_DIR}/support-files/rpm/server-prein.sh) -SET(CPACK_RPM_server_PRE_UNINSTALL_SCRIPT_FILE ${CMAKE_SOURCE_DIR}/support-files/rpm/server-preun.sh) -SET(CPACK_RPM_server_POST_INSTALL_SCRIPT_FILE ${CMAKE_SOURCE_DIR}/support-files/rpm/server-postin.sh) -SET(CPACK_RPM_server_POST_UNINSTALL_SCRIPT_FILE ${CMAKE_SOURCE_DIR}/support-files/rpm/server-postun.sh) -SET(CPACK_RPM_server_POST_TRANS_SCRIPT_FILE ${CMAKE_SOURCE_DIR}/support-files/rpm/server-posttrans.sh) -SET(CPACK_RPM_shared_POST_INSTALL_SCRIPT_FILE ${CMAKE_SOURCE_DIR}/support-files/rpm/shared-post.sh) -SET(CPACK_RPM_shared_POST_UNINSTALL_SCRIPT_FILE ${CMAKE_SOURCE_DIR}/support-files/rpm/shared-post.sh) -SET(CPACK_RPM_compat_POST_INSTALL_SCRIPT_FILE ${CMAKE_SOURCE_DIR}/support-files/rpm/shared-post.sh) -SET(CPACK_RPM_compat_POST_UNINSTALL_SCRIPT_FILE ${CMAKE_SOURCE_DIR}/support-files/rpm/shared-post.sh) +SET(CPACK_RPM_server_PRE_INSTALL_SCRIPT_FILE ${ELOQSQL_SOURCE_DIR}/support-files/rpm/server-prein.sh) +SET(CPACK_RPM_server_PRE_UNINSTALL_SCRIPT_FILE ${ELOQSQL_SOURCE_DIR}/support-files/rpm/server-preun.sh) +SET(CPACK_RPM_server_POST_INSTALL_SCRIPT_FILE ${ELOQSQL_SOURCE_DIR}/support-files/rpm/server-postin.sh) +SET(CPACK_RPM_server_POST_UNINSTALL_SCRIPT_FILE ${ELOQSQL_SOURCE_DIR}/support-files/rpm/server-postun.sh) +SET(CPACK_RPM_server_POST_TRANS_SCRIPT_FILE ${ELOQSQL_SOURCE_DIR}/support-files/rpm/server-posttrans.sh) +SET(CPACK_RPM_shared_POST_INSTALL_SCRIPT_FILE ${ELOQSQL_SOURCE_DIR}/support-files/rpm/shared-post.sh) +SET(CPACK_RPM_shared_POST_UNINSTALL_SCRIPT_FILE ${ELOQSQL_SOURCE_DIR}/support-files/rpm/shared-post.sh) +SET(CPACK_RPM_compat_POST_INSTALL_SCRIPT_FILE ${ELOQSQL_SOURCE_DIR}/support-files/rpm/shared-post.sh) +SET(CPACK_RPM_compat_POST_UNINSTALL_SCRIPT_FILE ${ELOQSQL_SOURCE_DIR}/support-files/rpm/shared-post.sh) MACRO(ALTERNATIVE_NAME real alt) IF(${ARGC} GREATER 2) @@ -303,15 +303,15 @@ SET(PYTHON_SHEBANG "/usr/bin/python3" CACHE STRING "python shebang") # If we want to build build MariaDB-shared-compat, # extract compat libraries from MariaDB-shared-5.3 rpm -FILE(GLOB compat53 RELATIVE ${CMAKE_SOURCE_DIR} - "${CMAKE_SOURCE_DIR}/../MariaDB-shared-5.3.*.rpm") -FILE(GLOB compat101 RELATIVE ${CMAKE_SOURCE_DIR} - "${CMAKE_SOURCE_DIR}/../MariaDB-shared-10.1.*.rpm") +FILE(GLOB compat53 RELATIVE ${ELOQSQL_SOURCE_DIR} + "${ELOQSQL_SOURCE_DIR}/../MariaDB-shared-5.3.*.rpm") +FILE(GLOB compat101 RELATIVE ${ELOQSQL_SOURCE_DIR} + "${ELOQSQL_SOURCE_DIR}/../MariaDB-shared-10.1.*.rpm") IF(compat53 AND compat101) FOREACH(compat_rpm "${compat53}" "${compat101}") MESSAGE(STATUS "Using ${compat_rpm} to build MariaDB-compat") INSTALL(CODE "EXECUTE_PROCESS( - COMMAND rpm2cpio ${CMAKE_SOURCE_DIR}/${compat_rpm} + COMMAND rpm2cpio ${ELOQSQL_SOURCE_DIR}/${compat_rpm} COMMAND cpio --extract --make-directories */libmysqlclient*.so.* - WORKING_DIRECTORY \$ENV{DESTDIR}) EXECUTE_PROCESS( @@ -321,11 +321,11 @@ IF(compat53 AND compat101) ENDFOREACH() EXECUTE_PROCESS( - COMMAND rpm -q --provides -p "${CMAKE_SOURCE_DIR}/${compat101}" + COMMAND rpm -q --provides -p "${ELOQSQL_SOURCE_DIR}/${compat101}" ERROR_QUIET OUTPUT_VARIABLE compat_provides) EXECUTE_PROCESS( - COMMAND rpm -q --obsoletes -p "${CMAKE_SOURCE_DIR}/${compat101}" + COMMAND rpm -q --obsoletes -p "${ELOQSQL_SOURCE_DIR}/${compat101}" ERROR_QUIET OUTPUT_VARIABLE compat_obsoletes) diff --git a/cmake/ctest.cmake b/cmake/ctest.cmake index 5bc1ce5f832..aed255cce89 100644 --- a/cmake/ctest.cmake +++ b/cmake/ctest.cmake @@ -8,8 +8,8 @@ ENDMACRO() MACRO(MY_ADD_TESTS) CMAKE_PARSE_ARGUMENTS(ARG "" "EXT" "LINK_LIBRARIES" ${ARGN}) - INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include - ${CMAKE_SOURCE_DIR}/unittest/mytap) + INCLUDE_DIRECTORIES(${ELOQSQL_SOURCE_DIR}/include + ${ELOQSQL_SOURCE_DIR}/unittest/mytap) IF (NOT ARG_EXT) SET(ARG_EXT "c") diff --git a/cmake/dtrace.cmake b/cmake/dtrace.cmake index dae3b56b584..731fb624722 100644 --- a/cmake/dtrace.cmake +++ b/cmake/dtrace.cmake @@ -70,7 +70,7 @@ MACRO (DTRACE_HEADER provider header header_no_dtrace) ADD_CUSTOM_COMMAND( OUTPUT ${header} ${header_no_dtrace} COMMAND ${DTRACE} -h -s ${provider} -o ${header} - COMMAND perl ${CMAKE_SOURCE_DIR}/scripts/dheadgen.pl -f ${provider} > ${header_no_dtrace} + COMMAND perl ${ELOQSQL_SOURCE_DIR}/scripts/dheadgen.pl -f ${provider} > ${header_no_dtrace} DEPENDS ${provider} ) ENDIF() @@ -79,22 +79,22 @@ ENDMACRO() # Create provider headers IF(ENABLE_DTRACE) - CONFIGURE_FILE(${CMAKE_SOURCE_DIR}/include/probes_mysql.d.base - ${CMAKE_BINARY_DIR}/include/probes_mysql.d COPYONLY) + CONFIGURE_FILE(${ELOQSQL_SOURCE_DIR}/include/probes_mysql.d.base + ${ELOQSQL_BINARY_DIR}/include/probes_mysql.d COPYONLY) DTRACE_HEADER( - ${CMAKE_BINARY_DIR}/include/probes_mysql.d - ${CMAKE_BINARY_DIR}/include/probes_mysql_dtrace.h - ${CMAKE_BINARY_DIR}/include/probes_mysql_nodtrace.h + ${ELOQSQL_BINARY_DIR}/include/probes_mysql.d + ${ELOQSQL_BINARY_DIR}/include/probes_mysql_dtrace.h + ${ELOQSQL_BINARY_DIR}/include/probes_mysql_nodtrace.h ) ADD_CUSTOM_TARGET(gen_dtrace_header DEPENDS - ${CMAKE_BINARY_DIR}/include/probes_mysql.d - ${CMAKE_BINARY_DIR}/include/probes_mysql_dtrace.h - ${CMAKE_BINARY_DIR}/include/probes_mysql_nodtrace.h + ${ELOQSQL_BINARY_DIR}/include/probes_mysql.d + ${ELOQSQL_BINARY_DIR}/include/probes_mysql_dtrace.h + ${ELOQSQL_BINARY_DIR}/include/probes_mysql_nodtrace.h ) ELSE() - CONFIGURE_FILE(${CMAKE_SOURCE_DIR}/include/probes_mysql_nodtrace.h.in - ${CMAKE_BINARY_DIR}/include/probes_mysql_nodtrace.h COPYONLY) + CONFIGURE_FILE(${ELOQSQL_SOURCE_DIR}/include/probes_mysql_nodtrace.h.in + ${ELOQSQL_BINARY_DIR}/include/probes_mysql_nodtrace.h COPYONLY) ENDIF() FUNCTION(DTRACE_INSTRUMENT target) @@ -117,21 +117,21 @@ FUNCTION(DTRACE_INSTRUMENT target) COMMAND ${CMAKE_COMMAND} -DDTRACE=${DTRACE} -DOUTFILE=${outfile} - -DDFILE=${CMAKE_BINARY_DIR}/include/probes_mysql.d + -DDFILE=${ELOQSQL_BINARY_DIR}/include/probes_mysql.d -DDTRACE_FLAGS=${DTRACE_FLAGS} -DDIRS=. -DTYPE=${target_type} - -P ${CMAKE_SOURCE_DIR}/cmake/dtrace_prelink.cmake + -P ${ELOQSQL_SOURCE_DIR}/cmake/dtrace_prelink.cmake WORKING_DIRECTORY ${objdir} ) ELSEIF(CMAKE_SYSTEM_NAME MATCHES "Linux") # dtrace on Linux runs gcc and uses flags from environment SET(CFLAGS_SAVED $ENV{CFLAGS}) SET(ENV{CFLAGS} ${CMAKE_C_FLAGS}) - SET(outfile "${CMAKE_BINARY_DIR}/probes_mysql.o") + SET(outfile "${ELOQSQL_BINARY_DIR}/probes_mysql.o") # Systemtap object EXECUTE_PROCESS( - COMMAND ${DTRACE} -G -s ${CMAKE_SOURCE_DIR}/include/probes_mysql.d.base + COMMAND ${DTRACE} -G -s ${ELOQSQL_SOURCE_DIR}/include/probes_mysql.d.base -o ${outfile} ) SET(ENV{CFLAGS} ${CFLAGS_SAVED}) @@ -190,11 +190,11 @@ IF(CMAKE_SYSTEM_NAME MATCHES "SunOS" AND ENABLE_DTRACE) COMMAND ${CMAKE_COMMAND} -DDTRACE=${DTRACE} -DOUTFILE=${obj} - -DDFILE=${CMAKE_BINARY_DIR}/include/probes_mysql.d + -DDFILE=${ELOQSQL_BINARY_DIR}/include/probes_mysql.d -DDTRACE_FLAGS=${DTRACE_FLAGS} "-DDIRS=${dirs}" -DTYPE=MERGE - -P ${CMAKE_SOURCE_DIR}/cmake/dtrace_prelink.cmake + -P ${ELOQSQL_SOURCE_DIR}/cmake/dtrace_prelink.cmake VERBATIM ) ADD_CUSTOM_TARGET(${target}_dtrace_all DEPENDS ${obj}) diff --git a/cmake/info_macros.cmake.in b/cmake/info_macros.cmake.in index 89ca1ec79dc..93fae001051 100644 --- a/cmake/info_macros.cmake.in +++ b/cmake/info_macros.cmake.in @@ -26,8 +26,8 @@ SET(VERSION "@VERSION@") SET(MAJOR_VERSION "@MAJOR_VERSION@") SET(MINOR_VERSION "@MINOR_VERSION@") SET(PATCH_VERSION "@PATCH_VERSION@") -SET(CMAKE_SOURCE_DIR "@CMAKE_SOURCE_DIR@") -SET(CMAKE_BINARY_DIR "@CMAKE_BINARY_DIR@") +SET(ELOQSQL_SOURCE_DIR "@ELOQSQL_SOURCE_DIR@") +SET(ELOQSQL_BINARY_DIR "@ELOQSQL_BINARY_DIR@") SET(CMAKE_GENERATOR "@CMAKE_GENERATOR@") SET(CMAKE_SIZEOF_VOID_P "@CMAKE_SIZEOF_VOID_P@") SET(GIT_EXECUTABLE "@GIT_EXECUTABLE@") @@ -60,18 +60,18 @@ MACRO(CREATE_INFO_SRC target_dir) MESSAGE(STATUS "Could not determine build-date: <${error}>") ENDIF() - IF(GIT_EXECUTABLE AND EXISTS ${CMAKE_SOURCE_DIR}/.git) + IF(GIT_EXECUTABLE AND EXISTS ${ELOQSQL_SOURCE_DIR}/.git) # Sources are in a GIT repository: Always update. EXECUTE_PROCESS( COMMAND ${GIT_EXECUTABLE} rev-parse --abbrev-ref HEAD - WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} + WORKING_DIRECTORY ${ELOQSQL_SOURCE_DIR} OUTPUT_VARIABLE bname ) EXECUTE_PROCESS( COMMAND ${GIT_EXECUTABLE} log -1 --pretty="commit: %H%ndate: %ci%nbuild-date: ${bdate} %nshort: %h%nbranch: ${bname}" - WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} + WORKING_DIRECTORY ${ELOQSQL_SOURCE_DIR} OUTPUT_VARIABLE VERSION_INFO ) @@ -85,14 +85,14 @@ MACRO(CREATE_INFO_SRC target_dir) ELSEIF(EXISTS ${INFO_SRC}) # Outside a git tree, there is no need to change an existing "INFO_SRC", # it cannot be improved. - ELSEIF(EXISTS ${CMAKE_SOURCE_DIR}/Docs/INFO_SRC) + ELSEIF(EXISTS ${ELOQSQL_SOURCE_DIR}/Docs/INFO_SRC) # If we are building from a source distribution, it also contains "INFO_SRC". # Similar, the export used for a release build already has the file. - FILE(READ ${CMAKE_SOURCE_DIR}/Docs/INFO_SRC SOURCE_INFO) + FILE(READ ${ELOQSQL_SOURCE_DIR}/Docs/INFO_SRC SOURCE_INFO) FILE(WRITE ${INFO_SRC} "${SOURCE_INFO}\n") - ELSEIF(EXISTS ${CMAKE_SOURCE_DIR}/INFO_SRC) + ELSEIF(EXISTS ${ELOQSQL_SOURCE_DIR}/INFO_SRC) # This is not the proper location, but who knows ... - FILE(READ ${CMAKE_SOURCE_DIR}/INFO_SRC SOURCE_INFO) + FILE(READ ${ELOQSQL_SOURCE_DIR}/INFO_SRC SOURCE_INFO) FILE(WRITE ${INFO_SRC} "${SOURCE_INFO}\n") ELSE() # This is a fall-back. @@ -148,9 +148,9 @@ MACRO(CREATE_INFO_BIN) FILE(APPEND ${INFO_BIN} "Pointer size: ${CMAKE_SIZEOF_VOID_P}\n\n") FILE(APPEND ${INFO_BIN} "===== Feature flags used: =====\n") - IF(EXISTS ${CMAKE_BINARY_DIR}/CMakeCache.txt) + IF(EXISTS ${ELOQSQL_BINARY_DIR}/CMakeCache.txt) # Attention: "-N" prevents cmake from entering a recursion, and it must be a separate flag from "-L". - EXECUTE_PROCESS(COMMAND ${CMAKE_COMMAND} -N -L ${CMAKE_BINARY_DIR} OUTPUT_VARIABLE FEATURE_FLAGS) + EXECUTE_PROCESS(COMMAND ${CMAKE_COMMAND} -N -L ${ELOQSQL_BINARY_DIR} OUTPUT_VARIABLE FEATURE_FLAGS) FILE(APPEND ${INFO_BIN} ${FEATURE_FLAGS} "\n") ELSE() FILE(APPEND ${INFO_BIN} "File 'CMakeCache.txt' is not yet found.\n\n") diff --git a/cmake/install_macros.cmake b/cmake/install_macros.cmake index 765f3be23e0..e5cedffc3e7 100644 --- a/cmake/install_macros.cmake +++ b/cmake/install_macros.cmake @@ -59,10 +59,10 @@ FUNCTION(INSTALL_MANPAGE file) ENDIF() GET_FILENAME_COMPONENT(file_name "${file}" NAME) SET(GLOB_EXPR - ${CMAKE_SOURCE_DIR}/man/*${file}man.1* - ${CMAKE_SOURCE_DIR}/man/*${file}man.8* - ${CMAKE_BINARY_DIR}/man/*${file}man.1* - ${CMAKE_BINARY_DIR}/man/*${file}man.8* + ${ELOQSQL_SOURCE_DIR}/man/*${file}man.1* + ${ELOQSQL_SOURCE_DIR}/man/*${file}man.8* + ${ELOQSQL_BINARY_DIR}/man/*${file}man.1* + ${ELOQSQL_BINARY_DIR}/man/*${file}man.8* ) IF(MYSQL_DOC_DIR) SET(GLOB_EXPR @@ -260,7 +260,7 @@ ENDFUNCTION() # Optionally install mysqld/client/embedded from debug build run. outside of the current build dir # (unless multi-config generator is used like Visual Studio or Xcode). # For Makefile generators we default Debug build directory to ${buildroot}/../debug. -GET_FILENAME_COMPONENT(BINARY_PARENTDIR ${CMAKE_BINARY_DIR} PATH) +GET_FILENAME_COMPONENT(BINARY_PARENTDIR ${ELOQSQL_BINARY_DIR} PATH) SET(DEBUGBUILDDIR "${BINARY_PARENTDIR}/debug" CACHE INTERNAL "Directory of debug build") FUNCTION(INSTALL_MYSQL_TEST from to) diff --git a/cmake/libutils.cmake b/cmake/libutils.cmake index 74853c36a74..8e24cd5a107 100644 --- a/cmake/libutils.cmake +++ b/cmake/libutils.cmake @@ -231,7 +231,7 @@ MACRO(MERGE_STATIC_LIBS TARGET OUTPUT_NAME LIBS_TO_MERGE) COMMAND ${CMAKE_COMMAND} ARGS -DTARGET_SCRIPT="${MRI_SCRIPT}" - -DTOP_DIR="${CMAKE_BINARY_DIR}" + -DTOP_DIR="${ELOQSQL_BINARY_DIR}" -DCMAKE_AR="${CMAKE_AR}" -P "${MYSQL_CMAKE_SCRIPT_DIR}/merge_archives_unix.cmake" COMMAND ${CMAKE_RANLIB} diff --git a/cmake/make_dist.cmake.in b/cmake/make_dist.cmake.in index 4f6988cfed2..d912f792bf0 100644 --- a/cmake/make_dist.cmake.in +++ b/cmake/make_dist.cmake.in @@ -17,8 +17,8 @@ # If bzr is present, run bzr export. # Otherwise, just run cpack with source configuration. -SET(CMAKE_SOURCE_DIR "@CMAKE_SOURCE_DIR@") -SET(CMAKE_BINARY_DIR "@CMAKE_BINARY_DIR@") +SET(ELOQSQL_SOURCE_DIR "@ELOQSQL_SOURCE_DIR@") +SET(ELOQSQL_BINARY_DIR "@ELOQSQL_BINARY_DIR@") SET(CPACK_SOURCE_PACKAGE_FILE_NAME "@CPACK_SOURCE_PACKAGE_FILE_NAME@") SET(CMAKE_CPACK_COMMAND "@CMAKE_CPACK_COMMAND@") SET(CMAKE_COMMAND "@CMAKE_COMMAND@") @@ -34,7 +34,7 @@ SET(VERSION "@VERSION@") SET(MYSQL_DOCS_LOCATION "@MYSQL_DOCS_LOCATION@") -SET(PACKAGE_DIR ${CMAKE_BINARY_DIR}/${CPACK_SOURCE_PACKAGE_FILE_NAME}) +SET(PACKAGE_DIR ${ELOQSQL_BINARY_DIR}/${CPACK_SOURCE_PACKAGE_FILE_NAME}) FILE(REMOVE_RECURSE ${PACKAGE_DIR}) FILE(REMOVE ${PACKAGE_DIR}.tar.gz ) @@ -43,7 +43,7 @@ IF(GIT_EXECUTABLE) MESSAGE(STATUS "Running git checkout-index") EXECUTE_PROCESS( COMMAND "${GIT_EXECUTABLE}" checkout-index --all --prefix=${PACKAGE_DIR}/ - WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} + WORKING_DIRECTORY ${ELOQSQL_SOURCE_DIR} RESULT_VARIABLE RESULT ) IF(NOT RESULT EQUAL 0) @@ -51,7 +51,7 @@ IF(GIT_EXECUTABLE) ENDIF() EXECUTE_PROCESS( COMMAND "${GIT_EXECUTABLE}" submodule foreach "${GIT_EXECUTABLE} checkout-index --all --prefix=${PACKAGE_DIR}/$path/" - WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} + WORKING_DIRECTORY ${ELOQSQL_SOURCE_DIR} RESULT_VARIABLE RESULT ) IF(NOT RESULT EQUAL 0) @@ -59,7 +59,7 @@ IF(GIT_EXECUTABLE) ENDIF() EXECUTE_PROCESS( COMMAND "${GIT_EXECUTABLE}" submodule foreach "${GIT_EXECUTABLE} checkout-index --all --prefix=${PACKAGE_DIR}/wsrep-lib/$path/" - WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/wsrep-lib + WORKING_DIRECTORY ${ELOQSQL_SOURCE_DIR}/wsrep-lib RESULT_VARIABLE RESULT ) IF(NOT RESULT EQUAL 0) @@ -67,7 +67,7 @@ IF(GIT_EXECUTABLE) ENDIF() EXECUTE_PROCESS( COMMAND "${GIT_EXECUTABLE}" submodule foreach "${GIT_EXECUTABLE} checkout-index --all --prefix=${PACKAGE_DIR}/storage/columnstore/columnstore/$path/" - WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/storage/columnstore/columnstore + WORKING_DIRECTORY ${ELOQSQL_SOURCE_DIR}/storage/columnstore/columnstore RESULT_VARIABLE RESULT ) IF(NOT RESULT EQUAL 0) @@ -75,64 +75,64 @@ IF(GIT_EXECUTABLE) ENDIF() ENDIF() -CONFIGURE_FILE(${CMAKE_BINARY_DIR}/include/source_revision.h +CONFIGURE_FILE(${ELOQSQL_BINARY_DIR}/include/source_revision.h ${PACKAGE_DIR}/include/source_revision.h COPYONLY) -CONFIGURE_FILE(${CMAKE_BINARY_DIR}/storage/rocksdb/rdb_source_revision.h +CONFIGURE_FILE(${ELOQSQL_BINARY_DIR}/storage/rocksdb/rdb_source_revision.h ${PACKAGE_DIR}/storage/rocksdb/rdb_source_revision.h COPYONLY) IF(NOT GIT_EXECUTABLE) MESSAGE(STATUS "git not found or source dir is not a repo, use CPack") - IF(CMAKE_SOURCE_DIR STREQUAL CMAKE_BINARY_DIR) + IF(ELOQSQL_SOURCE_DIR STREQUAL ELOQSQL_BINARY_DIR) # In-source build is the worst option, we have to cleanup source tree. # Save bison output first. - CONFIGURE_FILE(${CMAKE_BINARY_DIR}/sql/sql_yacc.cc - ${CMAKE_BINARY_DIR}/sql_yacc.cc COPYONLY) - CONFIGURE_FILE(${CMAKE_BINARY_DIR}/sql/sql_yacc.hh - ${CMAKE_BINARY_DIR}/sql_yacc.hh COPYONLY) + CONFIGURE_FILE(${ELOQSQL_BINARY_DIR}/sql/sql_yacc.cc + ${ELOQSQL_BINARY_DIR}/sql_yacc.cc COPYONLY) + CONFIGURE_FILE(${ELOQSQL_BINARY_DIR}/sql/sql_yacc.hh + ${ELOQSQL_BINARY_DIR}/sql_yacc.hh COPYONLY) IF(CMAKE_GENERATOR MATCHES "Makefiles") # make clean EXECUTE_PROCESS( COMMAND ${CMAKE_MAKE_PROGRAM} clean - WORKING_DIRECTORY ${CMAKE_BINARY_DIR} + WORKING_DIRECTORY ${ELOQSQL_BINARY_DIR} ) ENDIF() # Restore bison output - CONFIGURE_FILE(${CMAKE_BINARY_DIR}/sql_yacc.cc - ${CMAKE_BINARY_DIR}/sql/sql_yacc.cc COPYONLY) - CONFIGURE_FILE(${CMAKE_BINARY_DIR}/sql_yacc.hh - ${CMAKE_BINARY_DIR}/sql/sql_yacc.h COPYONLY) - FILE(REMOVE ${CMAKE_BINARY_DIR}/sql_yacc.cc) - FILE(REMOVE ${CMAKE_BINARY_DIR}/sql_yacc.hh) + CONFIGURE_FILE(${ELOQSQL_BINARY_DIR}/sql_yacc.cc + ${ELOQSQL_BINARY_DIR}/sql/sql_yacc.cc COPYONLY) + CONFIGURE_FILE(${ELOQSQL_BINARY_DIR}/sql_yacc.hh + ${ELOQSQL_BINARY_DIR}/sql/sql_yacc.h COPYONLY) + FILE(REMOVE ${ELOQSQL_BINARY_DIR}/sql_yacc.cc) + FILE(REMOVE ${ELOQSQL_BINARY_DIR}/sql_yacc.hh) ENDIF() EXECUTE_PROCESS( COMMAND ${CMAKE_CPACK_COMMAND} -G TGZ --config ./CPackSourceConfig.cmake - ${CMAKE_BINARY_DIR}/CPackSourceConfig.cmake + ${ELOQSQL_BINARY_DIR}/CPackSourceConfig.cmake - WORKING_DIRECTORY ${CMAKE_BINARY_DIR} + WORKING_DIRECTORY ${ELOQSQL_BINARY_DIR} ) EXECUTE_PROCESS( COMMAND ${CMAKE_COMMAND} -E tar xzf ${CPACK_SOURCE_PACKAGE_FILE_NAME}.tar.gz ${PACK_SOURCE_PACKAGE_FILE_NAME} - WORKING_DIRECTORY ${CMAKE_BINARY_DIR} + WORKING_DIRECTORY ${ELOQSQL_BINARY_DIR} ) ENDIF() # Copy bison output -CONFIGURE_FILE(${CMAKE_BINARY_DIR}/sql/yy_mariadb.hh +CONFIGURE_FILE(${ELOQSQL_BINARY_DIR}/sql/yy_mariadb.hh ${PACKAGE_DIR}/sql/yy_mariadb.hh COPYONLY) -CONFIGURE_FILE(${CMAKE_BINARY_DIR}/sql/yy_mariadb.cc +CONFIGURE_FILE(${ELOQSQL_BINARY_DIR}/sql/yy_mariadb.cc ${PACKAGE_DIR}/sql/yy_mariadb.cc COPYONLY) # Copy bison output -CONFIGURE_FILE(${CMAKE_BINARY_DIR}/sql/yy_oracle.hh +CONFIGURE_FILE(${ELOQSQL_BINARY_DIR}/sql/yy_oracle.hh ${PACKAGE_DIR}/sql/yy_oracle.hh COPYONLY) -CONFIGURE_FILE(${CMAKE_BINARY_DIR}/sql/yy_oracle.cc +CONFIGURE_FILE(${ELOQSQL_BINARY_DIR}/sql/yy_oracle.cc ${PACKAGE_DIR}/sql/yy_oracle.cc COPYONLY) # Add documentation, if user has specified where to find them @@ -142,7 +142,7 @@ IF(MYSQL_DOCS_LOCATION) ENDIF() # Ensure there is an "INFO_SRC" file. -INCLUDE(${CMAKE_BINARY_DIR}/info_macros.cmake) +INCLUDE(${ELOQSQL_BINARY_DIR}/info_macros.cmake) IF(NOT EXISTS ${PACKAGE_DIR}/Docs/INFO_SRC) CREATE_INFO_SRC(${PACKAGE_DIR}/Docs) ENDIF() diff --git a/cmake/mysql_add_executable.cmake b/cmake/mysql_add_executable.cmake index 06bdfce14de..df43f7ed220 100644 --- a/cmake/mysql_add_executable.cmake +++ b/cmake/mysql_add_executable.cmake @@ -43,7 +43,7 @@ FUNCTION (MYSQL_ADD_EXECUTABLE) IF(MSVC) # Add compatibility manifest, to fix GetVersionEx on Windows 8.1 and later IF (CMAKE_VERSION VERSION_GREATER 3.3) - SET(sources ${sources} ${PROJECT_SOURCE_DIR}/cmake/win_compatibility.manifest) + SET(sources ${sources} ${ELOQSQL_SOURCE_DIR}/cmake/win_compatibility.manifest) ENDIF() ENDIF() diff --git a/cmake/mysql_version.cmake b/cmake/mysql_version.cmake index a55c27b4d4c..a8eb4fdd089 100644 --- a/cmake/mysql_version.cmake +++ b/cmake/mysql_version.cmake @@ -23,15 +23,15 @@ SET(DOT_FRM_VERSION "6") # Generate "something" to trigger cmake rerun when VERSION changes CONFIGURE_FILE( - ${CMAKE_SOURCE_DIR}/VERSION - ${CMAKE_BINARY_DIR}/VERSION.dep + ${ELOQSQL_SOURCE_DIR}/VERSION + ${ELOQSQL_BINARY_DIR}/VERSION.dep ) # Read value for a variable from VERSION. MACRO(MYSQL_GET_CONFIG_VALUE keyword var) IF(NOT ${var}) - FILE (STRINGS ${CMAKE_SOURCE_DIR}/VERSION str REGEX "^[ ]*${keyword}=") + FILE (STRINGS ${ELOQSQL_SOURCE_DIR}/VERSION str REGEX "^[ ]*${keyword}=") IF(str) STRING(REPLACE "${keyword}=" "" str ${str}) STRING(REGEX REPLACE "[ ].*" "" str "${str}") @@ -127,17 +127,17 @@ IF(MSVC) SET(FILETYPE VFT_APP) CONFIGURE_FILE(${MYSQL_CMAKE_SCRIPT_DIR}/versioninfo.rc.in - ${CMAKE_BINARY_DIR}/versioninfo_exe.rc) + ${ELOQSQL_BINARY_DIR}/versioninfo_exe.rc) SET(FILETYPE VFT_DLL) CONFIGURE_FILE(${MYSQL_CMAKE_SCRIPT_DIR}/versioninfo.rc.in - ${CMAKE_BINARY_DIR}/versioninfo_dll.rc) + ${ELOQSQL_BINARY_DIR}/versioninfo_dll.rc) FUNCTION(ADD_VERSION_INFO target target_type sources_var) IF("${target_type}" MATCHES "SHARED" OR "${target_type}" MATCHES "MODULE") - SET(rcfile ${CMAKE_BINARY_DIR}/versioninfo_dll.rc) + SET(rcfile ${ELOQSQL_BINARY_DIR}/versioninfo_dll.rc) ELSEIF("${target_type}" MATCHES "EXE") - SET(rcfile ${CMAKE_BINARY_DIR}/versioninfo_exe.rc) + SET(rcfile ${ELOQSQL_BINARY_DIR}/versioninfo_exe.rc) ENDIF() SET(${sources_var} ${${sources_var}} ${rcfile} PARENT_SCOPE) ENDFUNCTION() diff --git a/cmake/pcre.cmake b/cmake/pcre.cmake index 0ac834f44a6..93707061289 100644 --- a/cmake/pcre.cmake +++ b/cmake/pcre.cmake @@ -5,7 +5,7 @@ SET(WITH_PCRE "auto" CACHE STRING "Which pcre to use (possible values are 'bundled', 'system', or 'auto')") MACRO(BUNDLE_PCRE2) - SET(dir "${CMAKE_BINARY_DIR}/extra/pcre2") + SET(dir "${ELOQSQL_BINARY_DIR}/extra/pcre2") SET(PCRE_INCLUDES ${dir}/src/pcre2-build ${dir}/src/pcre2/src) SET(byproducts) FOREACH(lib pcre2-posix pcre2-8) diff --git a/cmake/plugin.cmake b/cmake/plugin.cmake index 813d8ef6e42..eb228f509a5 100644 --- a/cmake/plugin.cmake +++ b/cmake/plugin.cmake @@ -42,8 +42,8 @@ MACRO(MYSQL_ADD_PLUGIN) IF(NOT WITHOUT_SERVER OR ARG_CLIENT) # Add common include directories - INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include - ${CMAKE_SOURCE_DIR}/sql + INCLUDE_DIRECTORIES(${ELOQSQL_SOURCE_DIR}/include + ${ELOQSQL_SOURCE_DIR}/sql ${PCRE_INCLUDES} ${SSL_INCLUDE_DIRS} ${ZLIB_INCLUDE_DIR}) @@ -215,7 +215,7 @@ MACRO(MYSQL_ADD_PLUGIN) TARGET_LINK_LIBRARIES (${target} mysqlservices ${ARG_LINK_LIBRARIES}) IF(CMAKE_SYSTEM_NAME MATCHES AIX) - TARGET_LINK_OPTIONS(${target} PRIVATE "-Wl,-bE:${CMAKE_SOURCE_DIR}/libservices/mysqlservices_aix.def") + TARGET_LINK_OPTIONS(${target} PRIVATE "-Wl,-bE:${ELOQSQL_SOURCE_DIR}/libservices/mysqlservices_aix.def") ENDIF() # Server plugins use symbols defined in mysqld executable. @@ -267,8 +267,8 @@ MACRO(MYSQL_ADD_PLUGIN) FILE(WRITE ${ARG_CONFIG} "[mariadb]\nplugin-load-add=${ARG_MODULE_OUTPUT_NAME}.so\n") ENDIF() SET(CPACK_RPM_${ARG_COMPONENT}_USER_FILELIST ${ignored} "%config(noreplace) ${INSTALL_SYSCONF2DIR}/*" PARENT_SCOPE) - SET(CPACK_RPM_${ARG_COMPONENT}_POST_INSTALL_SCRIPT_FILE ${CMAKE_SOURCE_DIR}/support-files/rpm/plugin-postin.sh PARENT_SCOPE) - SET(CPACK_RPM_${ARG_COMPONENT}_POST_TRANS_SCRIPT_FILE ${CMAKE_SOURCE_DIR}/support-files/rpm/server-posttrans.sh PARENT_SCOPE) + SET(CPACK_RPM_${ARG_COMPONENT}_POST_INSTALL_SCRIPT_FILE ${ELOQSQL_SOURCE_DIR}/support-files/rpm/plugin-postin.sh PARENT_SCOPE) + SET(CPACK_RPM_${ARG_COMPONENT}_POST_TRANS_SCRIPT_FILE ${ELOQSQL_SOURCE_DIR}/support-files/rpm/server-posttrans.sh PARENT_SCOPE) ENDIF() ENDIF() ELSE() @@ -308,10 +308,10 @@ ENDMACRO() # subdirectories, configure sql_builtins.cc MACRO(CONFIGURE_PLUGINS) IF(NOT WITHOUT_SERVER) - FILE(GLOB dirs_storage ${CMAKE_SOURCE_DIR}/storage/*) + FILE(GLOB dirs_storage ${ELOQSQL_SOURCE_DIR}/storage/*) ENDIF() - FILE(GLOB dirs_plugin ${CMAKE_SOURCE_DIR}/plugin/*) + FILE(GLOB dirs_plugin ${ELOQSQL_SOURCE_DIR}/plugin/*) FOREACH(dir ${dirs_storage} ${dirs_plugin}) IF (EXISTS ${dir}/CMakeLists.txt) ADD_SUBDIRECTORY(${dir}) diff --git a/cmake/readline.cmake b/cmake/readline.cmake index 9c35d8c7d22..39b8cbae527 100644 --- a/cmake/readline.cmake +++ b/cmake/readline.cmake @@ -93,9 +93,9 @@ ENDMACRO() MACRO (MYSQL_USE_BUNDLED_READLINE) SET(USE_NEW_READLINE_INTERFACE 1) SET(HAVE_HIST_ENTRY 0 CACHE INTERNAL "" FORCE) - SET(MY_READLINE_INCLUDE_DIR ${CMAKE_SOURCE_DIR}/extra/readline) + SET(MY_READLINE_INCLUDE_DIR ${ELOQSQL_SOURCE_DIR}/extra/readline) SET(MY_READLINE_LIBRARY readline) - ADD_SUBDIRECTORY(${CMAKE_SOURCE_DIR}/extra/readline) + ADD_SUBDIRECTORY(${ELOQSQL_SOURCE_DIR}/extra/readline) ENDMACRO() MACRO (MYSQL_FIND_SYSTEM_READLINE) diff --git a/cmake/sign.cmake.in b/cmake/sign.cmake.in index 6b0fa8b128f..ac1c540cf0a 100644 --- a/cmake/sign.cmake.in +++ b/cmake/sign.cmake.in @@ -13,7 +13,7 @@ ENDIF() GET_FILENAME_COMPONENT(SIGNTOOL_DIR "@SIGNTOOL_EXECUTABLE@" DIRECTORY) GET_FILENAME_COMPONENT(SIGNTOOL_NAME "@SIGNTOOL_EXECUTABLE@" NAME) -FILE(GLOB_RECURSE files "@CMAKE_BINARY_DIR@/*.signme") +FILE(GLOB_RECURSE files "@ELOQSQL_BINARY_DIR@/*.signme") MESSAGE(STATUS "signing files") diff --git a/cmake/ssl.cmake b/cmake/ssl.cmake index 646aa37a91c..2361173669b 100644 --- a/cmake/ssl.cmake +++ b/cmake/ssl.cmake @@ -49,9 +49,9 @@ ENDMACRO() MACRO (MYSQL_USE_BUNDLED_SSL) SET(INC_DIRS - ${CMAKE_BINARY_DIR}/extra/wolfssl - ${CMAKE_SOURCE_DIR}/extra/wolfssl/wolfssl - ${CMAKE_SOURCE_DIR}/extra/wolfssl/wolfssl/wolfssl + ${ELOQSQL_BINARY_DIR}/extra/wolfssl + ${ELOQSQL_SOURCE_DIR}/extra/wolfssl/wolfssl + ${ELOQSQL_SOURCE_DIR}/extra/wolfssl/wolfssl/wolfssl ) SET(SSL_LIBRARIES wolfssl wolfcrypt) SET(SSL_INCLUDE_DIRS ${INC_DIRS}) diff --git a/cmake/submodules.cmake b/cmake/submodules.cmake index ddebb777ce9..f06ae0f9308 100644 --- a/cmake/submodules.cmake +++ b/cmake/submodules.cmake @@ -5,9 +5,9 @@ IF(NOT UPDATE_SUBMODULES) RETURN() ENDIF() -IF(GIT_EXECUTABLE AND EXISTS "${CMAKE_SOURCE_DIR}/.git") +IF(GIT_EXECUTABLE AND EXISTS "${ELOQSQL_SOURCE_DIR}/.git") EXECUTE_PROCESS(COMMAND "${GIT_EXECUTABLE}" config --get cmake.update-submodules - WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" + WORKING_DIRECTORY "${ELOQSQL_SOURCE_DIR}" OUTPUT_VARIABLE cmake_update_submodules RESULT_VARIABLE git_config_get_result) IF(cmake_update_submodules MATCHES no) @@ -28,22 +28,22 @@ IF(GIT_EXECUTABLE AND EXISTS "${CMAKE_SOURCE_DIR}/.git") IF(cmake_update_submodules MATCHES force) MESSAGE(STATUS "Updating submodules (forced)") EXECUTE_PROCESS(COMMAND ${UPDATE_SUBMODULES_COMMAND} --force - WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" + WORKING_DIRECTORY "${ELOQSQL_SOURCE_DIR}" RESULT_VARIABLE update_result) ELSEIF(cmake_update_submodules MATCHES yes) EXECUTE_PROCESS(COMMAND ${UPDATE_SUBMODULES_COMMAND} - WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" + WORKING_DIRECTORY "${ELOQSQL_SOURCE_DIR}" RESULT_VARIABLE update_result) ELSE() MESSAGE(STATUS "Updating submodules") EXECUTE_PROCESS(COMMAND ${UPDATE_SUBMODULES_COMMAND} - WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" + WORKING_DIRECTORY "${ELOQSQL_SOURCE_DIR}" RESULT_VARIABLE update_result) ENDIF() ENDIF() ENDIF() -IF(update_result OR NOT EXISTS ${CMAKE_SOURCE_DIR}/libmariadb/CMakeLists.txt) +IF(update_result OR NOT EXISTS ${ELOQSQL_SOURCE_DIR}/libmariadb/CMakeLists.txt) MESSAGE(FATAL_ERROR "No MariaDB Connector/C! Run ${GIT_EXECUTABLE} submodule update --init --recursive Then restart the build.${SUBMODULE_UPDATE_CONFIG_MESSAGE}") diff --git a/cmake/tags.cmake b/cmake/tags.cmake index 309bfae9832..b78ea466dae 100644 --- a/cmake/tags.cmake +++ b/cmake/tags.cmake @@ -17,10 +17,10 @@ IF(UNIX) ADD_CUSTOM_TARGET (tags COMMAND support-files/build-tags - WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} + WORKING_DIRECTORY ${ELOQSQL_SOURCE_DIR} ) ADD_CUSTOM_TARGET (ctags COMMAND ctags -R -f CTAGS - WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} + WORKING_DIRECTORY ${ELOQSQL_SOURCE_DIR} ) ENDIF() diff --git a/cmake/wsrep.cmake b/cmake/wsrep.cmake index 368ae61d1d3..04ad9a7ae77 100644 --- a/cmake/wsrep.cmake +++ b/cmake/wsrep.cmake @@ -33,14 +33,14 @@ IF(WITH_WSREP) # Set the patch version SET(WSREP_PATCH_VERSION "22") - IF(NOT EXISTS "${CMAKE_SOURCE_DIR}/wsrep-lib/wsrep-API/v26/wsrep_api.h") + IF(NOT EXISTS "${ELOQSQL_SOURCE_DIR}/wsrep-lib/wsrep-API/v26/wsrep_api.h") MESSAGE(FATAL_ERROR "No MariaDB wsrep-API code! Run ${GIT_EXECUTABLE} submodule update --init --recursive Then restart the build. ") ENDIF() # Obtain wsrep API version - FILE(STRINGS "${CMAKE_SOURCE_DIR}/wsrep-lib/wsrep-API/v26/wsrep_api.h" WSREP_API_VERSION + FILE(STRINGS "${ELOQSQL_SOURCE_DIR}/wsrep-lib/wsrep-API/v26/wsrep_api.h" WSREP_API_VERSION LIMIT_COUNT 1 REGEX "WSREP_INTERFACE_VERSION") STRING(REGEX MATCH "([0-9]+)" WSREP_API_VERSION "${WSREP_API_VERSION}") @@ -56,8 +56,8 @@ Then restart the build. SET(WSREP_LIB_WITH_DBSIM OFF CACHE BOOL "Disable building dbsim for wsrep-lib") endif() - INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/wsrep-lib/include) - INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/wsrep-lib/wsrep-API/v26) + INCLUDE_DIRECTORIES(${ELOQSQL_SOURCE_DIR}/wsrep-lib/include) + INCLUDE_DIRECTORIES(${ELOQSQL_SOURCE_DIR}/wsrep-lib/wsrep-API/v26) SET(old_BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS}) SET(BUILD_SHARED_LIBS OFF) diff --git a/cmake/zlib.cmake b/cmake/zlib.cmake index 9e085189e4f..09fe71cb03e 100644 --- a/cmake/zlib.cmake +++ b/cmake/zlib.cmake @@ -14,7 +14,7 @@ # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA MACRO (MYSQL_USE_BUNDLED_ZLIB) - SET(ZLIB_INCLUDE_DIR ${CMAKE_SOURCE_DIR}/zlib ${CMAKE_BINARY_DIR}/zlib) + SET(ZLIB_INCLUDE_DIR ${ELOQSQL_SOURCE_DIR}/zlib ${ELOQSQL_BINARY_DIR}/zlib) SET(BUILD_BUNDLED_ZLIB 1) SET(ZLIB_LIBRARY zlib CACHE INTERNAL "Bundled zlib library") SET(ZLIB_FOUND TRUE) diff --git a/configure.cmake b/configure.cmake index 5b49a065b5a..f36c77b411f 100644 --- a/configure.cmake +++ b/configure.cmake @@ -168,7 +168,7 @@ IF(UNIX) HAVE_LIBWRAP) SET(CMAKE_REQUIRED_LIBRARIES ${SAVE_CMAKE_REQUIRED_LIBRARIES}) IF(HAVE_LIBWRAP) - SET(MYSYS_LIBWRAP_SOURCE ${CMAKE_SOURCE_DIR}/mysys/my_libwrap.c) + SET(MYSYS_LIBWRAP_SOURCE ${ELOQSQL_SOURCE_DIR}/mysys/my_libwrap.c) SET(LIBWRAP "wrap") ENDIF() ENDIF() @@ -679,8 +679,8 @@ IF(NOT STACK_DIRECTION) "or -DSTACK_DIRECTION=-1 when calling cmake.") ELSE() TRY_RUN(STACKDIR_RUN_RESULT STACKDIR_COMPILE_RESULT - ${CMAKE_BINARY_DIR} - ${CMAKE_SOURCE_DIR}/cmake/stack_direction.c + ${ELOQSQL_BINARY_DIR} + ${ELOQSQL_SOURCE_DIR}/cmake/stack_direction.c ) # Test program returns 0 (down) or 1 (up). # Convert to -1 or 1 diff --git a/dbug/CMakeLists.txt b/dbug/CMakeLists.txt index d938788aded..ce397518f9f 100644 --- a/dbug/CMakeLists.txt +++ b/dbug/CMakeLists.txt @@ -14,8 +14,8 @@ # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA INCLUDE_DIRECTORIES( - ${CMAKE_SOURCE_DIR}/dbug - ${CMAKE_SOURCE_DIR}/include + ${ELOQSQL_SOURCE_DIR}/dbug + ${ELOQSQL_SOURCE_DIR}/include ) SET(DBUG_SOURCES dbug.c) ADD_CONVENIENCE_LIBRARY(dbug ${DBUG_SOURCES}) diff --git a/extra/CMakeLists.txt b/extra/CMakeLists.txt index 5021128ed35..74dacada37c 100644 --- a/extra/CMakeLists.txt +++ b/extra/CMakeLists.txt @@ -13,7 +13,7 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA -INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include ${ZLIB_INCLUDE_DIR}) +INCLUDE_DIRECTORIES(${ELOQSQL_SOURCE_DIR}/include ${ZLIB_INCLUDE_DIR}) # Default install component for the files is Server here SET(MYSQL_INSTALL_COMPONENT Server) @@ -28,23 +28,23 @@ ENDIF() # mysqld_error.h results in rebuild of the almost whole server) # To preserve timestamp, first generate a temp header file, then copy it # to mysqld_error.h using cmake -E copy_if_different -ADD_CUSTOM_COMMAND(OUTPUT ${CMAKE_BINARY_DIR}/include/mysqld_error.h.tmp +ADD_CUSTOM_COMMAND(OUTPUT ${ELOQSQL_BINARY_DIR}/include/mysqld_error.h.tmp COMMAND comp_err --charset=${PROJECT_SOURCE_DIR}/sql/share/charsets - --out-dir=${CMAKE_BINARY_DIR}/sql/share/ - --header_file=${CMAKE_BINARY_DIR}/include/mysqld_error.h.tmp - --name_file=${CMAKE_BINARY_DIR}/include/mysqld_ername.h.tmp - --state_file=${CMAKE_BINARY_DIR}/include/sql_state.h.tmp + --out-dir=${ELOQSQL_BINARY_DIR}/sql/share/ + --header_file=${ELOQSQL_BINARY_DIR}/include/mysqld_error.h.tmp + --name_file=${ELOQSQL_BINARY_DIR}/include/mysqld_ername.h.tmp + --state_file=${ELOQSQL_BINARY_DIR}/include/sql_state.h.tmp --in_file=${PROJECT_SOURCE_DIR}/sql/share/errmsg-utf8.txt - COMMAND ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_BINARY_DIR}/include/mysqld_error.h.tmp ${CMAKE_BINARY_DIR}/include/mysqld_error.h - COMMAND ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_BINARY_DIR}/include/mysqld_ername.h.tmp ${CMAKE_BINARY_DIR}/include/mysqld_ername.h - COMMAND ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_BINARY_DIR}/include/sql_state.h.tmp ${CMAKE_BINARY_DIR}/include/sql_state.h + COMMAND ${CMAKE_COMMAND} -E copy_if_different ${ELOQSQL_BINARY_DIR}/include/mysqld_error.h.tmp ${ELOQSQL_BINARY_DIR}/include/mysqld_error.h + COMMAND ${CMAKE_COMMAND} -E copy_if_different ${ELOQSQL_BINARY_DIR}/include/mysqld_ername.h.tmp ${ELOQSQL_BINARY_DIR}/include/mysqld_ername.h + COMMAND ${CMAKE_COMMAND} -E copy_if_different ${ELOQSQL_BINARY_DIR}/include/sql_state.h.tmp ${ELOQSQL_BINARY_DIR}/include/sql_state.h DEPENDS comp_err ${PROJECT_SOURCE_DIR}/sql/share/errmsg-utf8.txt) ADD_CUSTOM_TARGET(GenError ALL DEPENDS - ${CMAKE_BINARY_DIR}/include/mysqld_error.h.tmp) + ${ELOQSQL_BINARY_DIR}/include/mysqld_error.h.tmp) MYSQL_ADD_EXECUTABLE(my_print_defaults my_print_defaults.c COMPONENT Client) TARGET_LINK_LIBRARIES(my_print_defaults mysys) @@ -67,8 +67,8 @@ ENDIF() IF(WITH_INNOBASE_STORAGE_ENGINE) # Add path to the InnoDB headers INCLUDE_DIRECTORIES( - ${CMAKE_SOURCE_DIR}/storage/innobase/include - ${CMAKE_SOURCE_DIR}/sql) + ${ELOQSQL_SOURCE_DIR}/storage/innobase/include + ${ELOQSQL_SOURCE_DIR}/sql) # We use the InnoDB code directly in case the code changes. ADD_DEFINITIONS("-DUNIV_INNOCHECKSUM") diff --git a/extra/mariabackup/CMakeLists.txt b/extra/mariabackup/CMakeLists.txt index a7a35c58ac3..ecd3322f53e 100644 --- a/extra/mariabackup/CMakeLists.txt +++ b/extra/mariabackup/CMakeLists.txt @@ -29,8 +29,8 @@ IF(NOT WIN32) ENDIF() INCLUDE_DIRECTORIES( - ${CMAKE_SOURCE_DIR}/include - ${CMAKE_SOURCE_DIR}/sql + ${ELOQSQL_SOURCE_DIR}/include + ${ELOQSQL_SOURCE_DIR}/sql ${CMAKE_CURRENT_SOURCE_DIR}/quicklz ${CMAKE_CURRENT_SOURCE_DIR} ) diff --git a/extra/readline/CMakeLists.txt b/extra/readline/CMakeLists.txt index 9a02d9b8247..687a88d3ecd 100644 --- a/extra/readline/CMakeLists.txt +++ b/extra/readline/CMakeLists.txt @@ -13,7 +13,7 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA -INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include ${CMAKE_CURRENT_SOURCE_DIR}) +INCLUDE_DIRECTORIES(${ELOQSQL_SOURCE_DIR}/include ${CMAKE_CURRENT_SOURCE_DIR}) ADD_DEFINITIONS(-DHAVE_CONFIG_H -DNO_KILL_INTR) diff --git a/libmysqld/CMakeLists.txt b/libmysqld/CMakeLists.txt index 85456f91500..6413e5dfc1f 100644 --- a/libmysqld/CMakeLists.txt +++ b/libmysqld/CMakeLists.txt @@ -18,11 +18,11 @@ ADD_DEFINITIONS(-DMYSQL_SERVER -DEMBEDDED_LIBRARY ${SSL_DEFINES}) INCLUDE_DIRECTORIES( -${CMAKE_SOURCE_DIR}/include -${CMAKE_SOURCE_DIR}/libmysqld -${CMAKE_SOURCE_DIR}/sql -${CMAKE_SOURCE_DIR}/tpool -${CMAKE_BINARY_DIR}/sql +${ELOQSQL_SOURCE_DIR}/include +${ELOQSQL_SOURCE_DIR}/libmysqld +${ELOQSQL_SOURCE_DIR}/sql +${ELOQSQL_SOURCE_DIR}/tpool +${ELOQSQL_BINARY_DIR}/sql ${PCRE_INCLUDES} ${ZLIB_INCLUDE_DIR} ${SSL_INCLUDE_DIRS} @@ -30,11 +30,11 @@ ${SSL_INTERNAL_INCLUDE_DIRS} ) SET(GEN_SOURCES -${CMAKE_BINARY_DIR}/sql/sql_yacc.hh -${CMAKE_BINARY_DIR}/sql/yy_mariadb.cc -${CMAKE_BINARY_DIR}/sql/yy_oracle.hh -${CMAKE_BINARY_DIR}/sql/yy_oracle.cc -${CMAKE_BINARY_DIR}/sql/lex_hash.h +${ELOQSQL_BINARY_DIR}/sql/sql_yacc.hh +${ELOQSQL_BINARY_DIR}/sql/yy_mariadb.cc +${ELOQSQL_BINARY_DIR}/sql/yy_oracle.hh +${ELOQSQL_BINARY_DIR}/sql/yy_oracle.cc +${ELOQSQL_BINARY_DIR}/sql/lex_hash.h ) SET_SOURCE_FILES_PROPERTIES(${GEN_SOURCES} PROPERTIES GENERATED TRUE) @@ -99,7 +99,7 @@ SET(SQL_EMBEDDED_SOURCES emb_qcache.cc libmysqld.c lib_sql.cc ../sql/event_parse_data.cc ../sql/sql_signal.cc ../sql/sys_vars.cc - ${CMAKE_BINARY_DIR}/sql/sql_builtin.cc + ${ELOQSQL_BINARY_DIR}/sql/sql_builtin.cc ../sql/mdl.cc ../sql/transaction.cc ../sql/sql_join_cache.cc ../sql/multi_range_read.cc diff --git a/libmysqld/examples/CMakeLists.txt b/libmysqld/examples/CMakeLists.txt index 2a10def8e2e..ef964053444 100644 --- a/libmysqld/examples/CMakeLists.txt +++ b/libmysqld/examples/CMakeLists.txt @@ -13,10 +13,10 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA -INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include - ${CMAKE_SOURCE_DIR}/libmysqld/include +INCLUDE_DIRECTORIES(${ELOQSQL_SOURCE_DIR}/include + ${ELOQSQL_SOURCE_DIR}/libmysqld/include ${PCRE_INCLUDES} - ${CMAKE_SOURCE_DIR}/sql + ${ELOQSQL_SOURCE_DIR}/sql ${MY_READLINE_INCLUDE_DIR} ) diff --git a/libservices/CMakeLists.txt b/libservices/CMakeLists.txt index 274c8ce6dac..9f4ab13087b 100644 --- a/libservices/CMakeLists.txt +++ b/libservices/CMakeLists.txt @@ -13,7 +13,7 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA -INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include) +INCLUDE_DIRECTORIES(${ELOQSQL_SOURCE_DIR}/include) SET(MYSQLSERVICES_SOURCES base64_service.c diff --git a/mysql-test/CMakeLists.txt b/mysql-test/CMakeLists.txt index 2c040fc0e32..377bdc10865 100644 --- a/mysql-test/CMakeLists.txt +++ b/mysql-test/CMakeLists.txt @@ -16,7 +16,7 @@ INSTALL_MYSQL_TEST("." ".") -IF(NOT ${CMAKE_SOURCE_DIR} STREQUAL ${CMAKE_BINARY_DIR}) +IF(NOT ${ELOQSQL_SOURCE_DIR} STREQUAL ${ELOQSQL_BINARY_DIR}) # Enable running mtr from build directory CONFIGURE_FILE( ${CMAKE_CURRENT_SOURCE_DIR}/mtr.out-of-source @@ -106,7 +106,7 @@ ENDIF() SET(MTR_FORCE perl ./mysql-test-run.pl --force) -IF(EXISTS ${CMAKE_SOURCE_DIR}/mysql-test/suite/nist) +IF(EXISTS ${ELOQSQL_SOURCE_DIR}/mysql-test/suite/nist) SET(TEST_NIST ${MTR_FORCE} --comment=nist suite=nist ${EXP} && ${MTR_FORCE} --comment=nist --force --suite=nist+ps ${EXP}) ELSE() diff --git a/mysql-test/lib/My/SafeProcess/CMakeLists.txt b/mysql-test/lib/My/SafeProcess/CMakeLists.txt index 0004a44982d..ad49834349d 100644 --- a/mysql-test/lib/My/SafeProcess/CMakeLists.txt +++ b/mysql-test/lib/My/SafeProcess/CMakeLists.txt @@ -14,12 +14,12 @@ # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA -INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include) +INCLUDE_DIRECTORIES(${ELOQSQL_SOURCE_DIR}/include) IF (WIN32) ADD_EXECUTABLE(my_safe_process safe_process_win.cc) ADD_EXECUTABLE(my_safe_kill safe_kill_win.cc) - TARGET_INCLUDE_DIRECTORIES(my_safe_kill PRIVATE ${CMAKE_SOURCE_DIR}/include) + TARGET_INCLUDE_DIRECTORIES(my_safe_kill PRIVATE ${ELOQSQL_SOURCE_DIR}/include) TARGET_LINK_LIBRARIES(my_safe_kill mysys psapi) ELSE() ADD_EXECUTABLE(my_safe_process safe_process.cc) diff --git a/mysql-test/mtr.out-of-source b/mysql-test/mtr.out-of-source index abab0be90df..be6c28be567 100644 --- a/mysql-test/mtr.out-of-source +++ b/mysql-test/mtr.out-of-source @@ -1,5 +1,5 @@ #!/usr/bin/env perl # Call mtr in out-of-source build -$ENV{MTR_BINDIR} = '@CMAKE_BINARY_DIR@'; -chdir('@CMAKE_SOURCE_DIR@/mysql-test'); -exit(system($^X, '@CMAKE_SOURCE_DIR@/mysql-test/mariadb-test-run.pl', @ARGV) >> 8); +$ENV{MTR_BINDIR} = '@ELOQSQL_BINARY_DIR@'; +chdir('@ELOQSQL_SOURCE_DIR@/mysql-test'); +exit(system($^X, '@ELOQSQL_SOURCE_DIR@/mysql-test/mariadb-test-run.pl', @ARGV) >> 8); diff --git a/mysys/CMakeLists.txt b/mysys/CMakeLists.txt index 760c3c1475d..146e33a4dcd 100644 --- a/mysys/CMakeLists.txt +++ b/mysys/CMakeLists.txt @@ -14,7 +14,7 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA -INCLUDE_DIRECTORIES(${ZLIB_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/include ${CMAKE_SOURCE_DIR}/mysys) +INCLUDE_DIRECTORIES(${ZLIB_INCLUDE_DIR} ${ELOQSQL_SOURCE_DIR}/include ${ELOQSQL_SOURCE_DIR}/mysys) SET(MYSYS_SOURCES array.c charset-def.c charset.c my_default.c get_password.c diff --git a/mysys_ssl/CMakeLists.txt b/mysys_ssl/CMakeLists.txt index 1c3f60b5bb0..81cedff4b83 100644 --- a/mysys_ssl/CMakeLists.txt +++ b/mysys_ssl/CMakeLists.txt @@ -13,8 +13,8 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA -INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include - ${CMAKE_SOURCE_DIR}/mysys_ssl +INCLUDE_DIRECTORIES(${ELOQSQL_SOURCE_DIR}/include + ${ELOQSQL_SOURCE_DIR}/mysys_ssl ${SSL_INCLUDE_DIRS}) IF(SSL_DEFINES) diff --git a/plugin/auth_dialog/CMakeLists.txt b/plugin/auth_dialog/CMakeLists.txt index b5e1b2beca0..7c2fcac3f00 100644 --- a/plugin/auth_dialog/CMakeLists.txt +++ b/plugin/auth_dialog/CMakeLists.txt @@ -16,5 +16,5 @@ # disabled in favor of libmariadb/plugins/auth/dialog.c # -#MYSQL_ADD_PLUGIN(dialog dialog.c ${CMAKE_SOURCE_DIR}/libmysql/get_password.c +#MYSQL_ADD_PLUGIN(dialog dialog.c ${ELOQSQL_SOURCE_DIR}/libmysql/get_password.c # MODULE_ONLY CLIENT COMPONENT ClientPlugins) diff --git a/plugin/disks/CMakeLists.txt b/plugin/disks/CMakeLists.txt index 4e40842cad0..3c4897a44cf 100644 --- a/plugin/disks/CMakeLists.txt +++ b/plugin/disks/CMakeLists.txt @@ -19,6 +19,6 @@ int main() ENDIF() IF (HAVE_GETMNTENT OR HAVE_GETMNTENT_IN_SYS_MNTAB OR HAVE_GETMNTINFO OR HAVE_GETMNTINFO64) - INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/sql) + INCLUDE_DIRECTORIES(${ELOQSQL_SOURCE_DIR}/sql) MYSQL_ADD_PLUGIN(DISKS information_schema_disks.cc MODULE_ONLY RECOMPILE_FOR_EMBEDDED) ENDIF() diff --git a/plugin/feedback/CMakeLists.txt b/plugin/feedback/CMakeLists.txt index 2103250e5a6..059f490e8ae 100644 --- a/plugin/feedback/CMakeLists.txt +++ b/plugin/feedback/CMakeLists.txt @@ -1,4 +1,4 @@ -INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/sql +INCLUDE_DIRECTORIES(${ELOQSQL_SOURCE_DIR}/sql ${PCRE_INCLUDES} ${SSL_INCLUDE_DIRS}) diff --git a/plugin/file_key_management/CMakeLists.txt b/plugin/file_key_management/CMakeLists.txt index 9b09da9bf20..d91b2916062 100644 --- a/plugin/file_key_management/CMakeLists.txt +++ b/plugin/file_key_management/CMakeLists.txt @@ -1,4 +1,4 @@ SET(FILE_KEY_MANAGEMENT_PLUGIN_SOURCES file_key_management_plugin.cc parser.cc) -INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/sql) +INCLUDE_DIRECTORIES(${ELOQSQL_SOURCE_DIR}/sql) MYSQL_ADD_PLUGIN(FILE_KEY_MANAGEMENT ${FILE_KEY_MANAGEMENT_PLUGIN_SOURCES} MODULE_ONLY) diff --git a/plugin/locale_info/CMakeLists.txt b/plugin/locale_info/CMakeLists.txt index c988d652d40..58f60de9a93 100644 --- a/plugin/locale_info/CMakeLists.txt +++ b/plugin/locale_info/CMakeLists.txt @@ -1,4 +1,4 @@ -INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/sql ${CMAKE_SOURCE_DIR}/regex) +INCLUDE_DIRECTORIES(${ELOQSQL_SOURCE_DIR}/sql ${ELOQSQL_SOURCE_DIR}/regex) MYSQL_ADD_PLUGIN(LOCALES locale_info.cc RECOMPILE_FOR_EMBEDDED) diff --git a/plugin/qc_info/CMakeLists.txt b/plugin/qc_info/CMakeLists.txt index b8c5f926cff..8f043b32aa8 100644 --- a/plugin/qc_info/CMakeLists.txt +++ b/plugin/qc_info/CMakeLists.txt @@ -1,4 +1,4 @@ -INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/sql +INCLUDE_DIRECTORIES(${ELOQSQL_SOURCE_DIR}/sql ${PCRE_INCLUDES}) MYSQL_ADD_PLUGIN(QUERY_CACHE_INFO qc_info.cc RECOMPILE_FOR_EMBEDDED) diff --git a/plugin/type_geom/CMakeLists.txt b/plugin/type_geom/CMakeLists.txt index 2b0c84f6fa0..b82019fecd8 100644 --- a/plugin/type_geom/CMakeLists.txt +++ b/plugin/type_geom/CMakeLists.txt @@ -1,3 +1,3 @@ -INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/sql) +INCLUDE_DIRECTORIES(${ELOQSQL_SOURCE_DIR}/sql) MYSQL_ADD_PLUGIN(TYPE_GEOM plugin.cc MANDATORY RECOMPILE_FOR_EMBEDDED) diff --git a/plugin/userstat/CMakeLists.txt b/plugin/userstat/CMakeLists.txt index 5daa4f59577..82bc05d60f3 100644 --- a/plugin/userstat/CMakeLists.txt +++ b/plugin/userstat/CMakeLists.txt @@ -1,4 +1,4 @@ -INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/sql) +INCLUDE_DIRECTORIES(${ELOQSQL_SOURCE_DIR}/sql) MYSQL_ADD_PLUGIN(USERSTAT userstat.cc MANDATORY) diff --git a/plugin/wsrep_info/CMakeLists.txt b/plugin/wsrep_info/CMakeLists.txt index 34aee9fba2c..0427450e8ad 100644 --- a/plugin/wsrep_info/CMakeLists.txt +++ b/plugin/wsrep_info/CMakeLists.txt @@ -1,5 +1,5 @@ IF (WITH_WSREP) - INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/sql - ${CMAKE_SOURCE_DIR}/wsrep) + INCLUDE_DIRECTORIES(${ELOQSQL_SOURCE_DIR}/sql + ${ELOQSQL_SOURCE_DIR}/wsrep) MYSQL_ADD_PLUGIN(WSREP_INFO plugin.cc MODULE_ONLY RECOMPILE_FOR_EMBEDDED) ENDIF() diff --git a/scripts/CMakeLists.txt b/scripts/CMakeLists.txt index 9eec793c9fb..1208d8ed8fd 100644 --- a/scripts/CMakeLists.txt +++ b/scripts/CMakeLists.txt @@ -29,7 +29,7 @@ MACRO(INSTALL_LINK old new destination component) ENDMACRO() IF(NOT CMAKE_CROSSCOMPILING OR DEFINED CMAKE_CROSSCOMPILING_EMULATOR) - INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include) + INCLUDE_DIRECTORIES(${ELOQSQL_SOURCE_DIR}/include) ADD_EXECUTABLE(comp_sql comp_sql.c ../sql/sql_bootstrap.cc) TARGET_LINK_LIBRARIES(comp_sql) ENDIF() @@ -40,7 +40,7 @@ IF(NOT WIN32 OR (CMAKE_CROSSCOMPILING AND NOT DEFINED CMAKE_CROSSCOMPILING_EMUL MARK_AS_ADVANCED(CAT_EXECUTABLE) ENDIF() -IF(NOT ("${CMAKE_BINARY_DIR}" STREQUAL "${CMAKE_SOURCE_DIR}")) +IF(NOT ("${ELOQSQL_BINARY_DIR}" STREQUAL "${ELOQSQL_SOURCE_DIR}")) FOREACH(f fill_help_tables.sql mysql_system_tables_fix.sql mysql_system_tables.sql mysql_system_tables_data.sql mysql_performance_tables.sql) CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/${f} ${CMAKE_CURRENT_BINARY_DIR}/${f} COPYONLY) ENDFOREACH() @@ -84,7 +84,7 @@ ADD_CUSTOM_TARGET(GenFixPrivs IF(UNIX AND NOT WITHOUT_SERVER) FILE(WRITE ${CMAKE_CURRENT_BINARY_DIR}/make_binary_distribution - "cd ${CMAKE_BINARY_DIR} && '${CMAKE_CPACK_COMMAND}' -G TGZ --config CPackConfig.cmake\n" ) + "cd ${ELOQSQL_BINARY_DIR} && '${CMAKE_CPACK_COMMAND}' -G TGZ --config CPackConfig.cmake\n" ) EXECUTE_PROCESS( COMMAND chmod +x ${CMAKE_CURRENT_BINARY_DIR}/make_binary_distribution ) diff --git a/scripts/sys_schema/CMakeLists.txt b/scripts/sys_schema/CMakeLists.txt index ccb268cc4fd..bf73a697fb1 100644 --- a/scripts/sys_schema/CMakeLists.txt +++ b/scripts/sys_schema/CMakeLists.txt @@ -174,4 +174,4 @@ FOREACH(f ${files}) FILE(READ ${f} content) SET(CMAKE_CONFIGURABLE_FILE_CONTENT "${CMAKE_CONFIGURABLE_FILE_CONTENT}${content}\n") ENDFOREACH() -CONFIGURE_FILE(${PROJECT_SOURCE_DIR}/cmake/configurable_file_content.in ${CMAKE_BINARY_DIR}/scripts/mysql_sys_schema.sql) +CONFIGURE_FILE(${PROJECT_SOURCE_DIR}/cmake/configurable_file_content.in ${ELOQSQL_BINARY_DIR}/scripts/mysql_sys_schema.sql) diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt index 842a3106542..70fcc9bed2d 100644 --- a/sql/CMakeLists.txt +++ b/sql/CMakeLists.txt @@ -118,13 +118,13 @@ IF(ELOQ_MODULE_ENABLED) ENDIF() INCLUDE_DIRECTORIES( -${CMAKE_SOURCE_DIR}/include -${CMAKE_SOURCE_DIR}/sql +${ELOQSQL_SOURCE_DIR}/include +${ELOQSQL_SOURCE_DIR}/sql ${PCRE_INCLUDES} ${ZLIB_INCLUDE_DIR} ${SSL_INCLUDE_DIRS} -${CMAKE_BINARY_DIR}/sql -${CMAKE_SOURCE_DIR}/tpool +${ELOQSQL_BINARY_DIR}/sql +${ELOQSQL_SOURCE_DIR}/tpool ${Boost_INCLUDE_DIRS} ) @@ -212,7 +212,7 @@ SET (SQL_SOURCE sql_signal.cc mdl.cc sql_admin.cc transaction.cc sys_vars.cc sql_truncate.cc datadict.cc sql_reload.cc - mysql_metrics.cc + mysql_metrics.cc circular_queue.h # added in MariaDB: grant.cc @@ -260,7 +260,7 @@ IF ((CMAKE_SYSTEM_NAME MATCHES "Linux" OR ENDIF() SET(SQL_SOURCE ${SQL_SOURCE} threadpool_generic.cc) SET(SQL_SOURCE ${SQL_SOURCE} threadpool_common.cc) - MYSQL_ADD_PLUGIN(thread_pool_info thread_pool_info.cc DEFAULT STATIC_ONLY NOT_EMBEDDED) + MYSQL_ADD_PLUGIN(thread_pool_info thread_pool_info.cc DEFAULT STATIC_ONLY NOT_EMBEDDED LINK_LIBRARIES data_substrate) ENDIF() IF(WIN32) @@ -312,7 +312,7 @@ TARGET_LINK_LIBRARIES(sql ${SQL_DEP_LIBS}) # IF(PLUGIN_ELOQ) ADD_DEFINITIONS(-DWITH_ELOQ_STORAGE_ENGINE) - SET(ELOQ_SOURCE_DIR ${CMAKE_SOURCE_DIR}/storage/eloq) + SET(ELOQ_SOURCE_DIR ${ELOQSQL_SOURCE_DIR}/storage/eloq) set(SQL_INCLUDE_DIR ${SQL_INCLUDE_DIR} ${ELOQ_SOURCE_DIR}) # ENDIF() @@ -387,13 +387,13 @@ TARGET_LINK_LIBRARIES(sql_builtins ${MYSQLD_STATIC_PLUGIN_LIBS}) # Export libraries for converged build if(BUILD_ELOQSQL_AS_LIBRARY) # Make sql and sql_builtins available to parent - set(ELOQSQL_SQL_LIBRARY sql PARENT_SCOPE) - set(ELOQSQL_SQL_BUILTINS_LIBRARY sql_builtins PARENT_SCOPE) + set(ELOQSQL_SQL_LIBRARY sql CACHE STRING "EloqSQL sql library") + set(ELOQSQL_SQL_BUILTINS_LIBRARY sql_builtins CACHE STRING "EloqSQL sql_builtins library") set(ELOQSQL_INCLUDE_DIRS - ${CMAKE_SOURCE_DIR}/include - ${CMAKE_SOURCE_DIR}/sql - ${CMAKE_BINARY_DIR}/include - PARENT_SCOPE + ${ELOQSQL_SOURCE_DIR}/include + ${ELOQSQL_SOURCE_DIR}/sql + ${ELOQSQL_BINARY_DIR}/include + CACHE STRING "EloqSQL include directories" ) endif() @@ -520,14 +520,14 @@ IF(WIN32 OR HAVE_DLOPEN AND NOT DISABLE_SHARED) ENDIF() CONFIGURE_FILE( - ${CMAKE_SOURCE_DIR}/cmake/make_dist.cmake.in - ${CMAKE_BINARY_DIR}/make_dist.cmake @ONLY) + ${ELOQSQL_SOURCE_DIR}/cmake/make_dist.cmake.in + ${ELOQSQL_BINARY_DIR}/make_dist.cmake @ONLY) ADD_CUSTOM_TARGET(dist - COMMAND ${CMAKE_COMMAND} -P ${CMAKE_BINARY_DIR}/make_dist.cmake - DEPENDS ${CMAKE_BINARY_DIR}/sql/yy_mariadb.cc ${CMAKE_BINARY_DIR}/sql/yy_mariadb.hh - DEPENDS ${CMAKE_BINARY_DIR}/sql/yy_oracle.cc ${CMAKE_BINARY_DIR}/sql/yy_oracle.hh - WORKING_DIRECTORY ${CMAKE_BINARY_DIR} + COMMAND ${CMAKE_COMMAND} -P ${ELOQSQL_BINARY_DIR}/make_dist.cmake + DEPENDS ${ELOQSQL_BINARY_DIR}/sql/yy_mariadb.cc ${ELOQSQL_BINARY_DIR}/sql/yy_mariadb.hh + DEPENDS ${ELOQSQL_BINARY_DIR}/sql/yy_oracle.cc ${ELOQSQL_BINARY_DIR}/sql/yy_oracle.hh + WORKING_DIRECTORY ${ELOQSQL_BINARY_DIR} ) ADD_CUSTOM_TARGET(distclean @@ -548,11 +548,11 @@ IF(TARGET mariadbd AND (NOT CMAKE_CROSSCOMPILING OR DEFINED CMAKE_CROSSCOMPILING COMMAND ${CMAKE_COMMAND} -E make_directory data COMMAND ${CMAKE_COMMAND} -E chdir data ${CMAKE_COMMAND} ${CONFIG_PARAM} - -DTOP_SRCDIR="${CMAKE_BINARY_DIR}" + -DTOP_SRCDIR="${ELOQSQL_BINARY_DIR}" -DBINDIR="${CMAKE_CURRENT_BINARY_DIR}" -DMYSQLD_EXECUTABLE="$" -DCMAKE_CFG_INTDIR="${CMAKE_CFG_INTDIR}" - -P ${CMAKE_SOURCE_DIR}/cmake/create_initial_db.cmake + -P ${ELOQSQL_SOURCE_DIR}/cmake/create_initial_db.cmake COMMAND ${CMAKE_COMMAND} -E touch ${CMAKE_CURRENT_BINARY_DIR}/initdb.dep WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/ DEPENDS mariadbd @@ -575,15 +575,15 @@ IF(WIN32) # Create bootstrapper SQL script ADD_CUSTOM_COMMAND(OUTPUT ${my_bootstrap_sql} - COMMAND ${CMAKE_COMMAND} -E chdir ${CMAKE_BINARY_DIR}/scripts + COMMAND ${CMAKE_COMMAND} -E chdir ${ELOQSQL_BINARY_DIR}/scripts cmd /c copy mysql_system_tables.sql+mysql_system_tables_data.sql+fill_help_tables.sql+mysql_performance_tables.sql+mysql_test_db.sql+mysql_sys_schema.sql ${native_outfile} DEPENDS - ${CMAKE_SOURCE_DIR}/scripts/mysql_system_tables.sql - ${CMAKE_SOURCE_DIR}/scripts/mysql_system_tables_data.sql - ${CMAKE_SOURCE_DIR}/scripts/fill_help_tables.sql - ${CMAKE_SOURCE_DIR}/scripts/mysql_performance_tables.sql - ${CMAKE_SOURCE_DIR}/scripts/mysql_test_db.sql - ${CMAKE_BINARY_DIR}/scripts/mysql_sys_schema.sql + ${ELOQSQL_SOURCE_DIR}/scripts/mysql_system_tables.sql + ${ELOQSQL_SOURCE_DIR}/scripts/mysql_system_tables_data.sql + ${ELOQSQL_SOURCE_DIR}/scripts/fill_help_tables.sql + ${ELOQSQL_SOURCE_DIR}/scripts/mysql_performance_tables.sql + ${ELOQSQL_SOURCE_DIR}/scripts/mysql_test_db.sql + ${ELOQSQL_BINARY_DIR}/scripts/mysql_sys_schema.sql ) ADD_CUSTOM_COMMAND( diff --git a/sql/circular_queue.h b/sql/circular_queue.h new file mode 100644 index 00000000000..e01bff59caa --- /dev/null +++ b/sql/circular_queue.h @@ -0,0 +1,258 @@ +/** + * Copyright (C) 2025 EloqData Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under either of the following two licenses: + * 1. GNU Affero General Public License, version 3, as published by the Free + * Software Foundation. + * 2. GNU General Public License as published by the Free Software + * Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License or GNU General Public License for more + * details. + * + * You should have received a copy of the GNU Affero General Public License + * and GNU General Public License V2 along with this program. If not, see + * . + * + */ +#pragma once + +#ifndef CIRCULAR_QUEUE_H +#define CIRCULAR_QUEUE_H + +#include +#include + +template +class CircularQueue +{ +public: + CircularQueue(size_t capacity = 8) : head_(0), cnt_(0), capacity_(capacity) + { + vec_ = std::make_unique(capacity); + } + + CircularQueue(CircularQueue &&rhs) + { + head_ = rhs.head_; + cnt_ = rhs.cnt_; + capacity_ = rhs.capacity_; + vec_ = std::move(rhs.vec_); + } + + CircularQueue &operator=(CircularQueue &&rhs) + { + if (this != &rhs) + { + head_ = rhs.head_; + cnt_ = rhs.cnt_; + capacity_ = rhs.capacity_; + vec_ = std::move(rhs.vec_); + } + return *this; + } + + CircularQueue(const CircularQueue &rhs) = delete; + CircularQueue &operator=(const CircularQueue &rhs) = delete; + + ~CircularQueue() = default; + + void Reset() + { + head_ = 0; + cnt_ = 0; + if (capacity_ > 8) + { + capacity_ = 8; + vec_ = std::make_unique(capacity_); + } + } + + void Enqueue(const T &item) + { + if (cnt_ == 0) + { + vec_[0] = item; + head_ = 0; + cnt_ = 1; + } + else if (cnt_ == capacity_) + { + size_t new_capacity = capacity_ << 1; + std::unique_ptr new_vec = std::make_unique(new_capacity); + + // Before: 0-------Tail-Head---------N-1 + // After: 0----------------------------Tail------------M-1 + // Copy Head --> N-1 + std::copy( + vec_.get() + head_, vec_.get() + capacity_, new_vec.get()); + + size_t half_cnt = capacity_ - head_; + // Copy 0 --> Tail + std::copy(vec_.get(), vec_.get() + head_, new_vec.get() + half_cnt); + + new_vec[capacity_] = item; + cnt_ = capacity_ + 1; + capacity_ = new_capacity; + head_ = 0; + vec_ = std::move(new_vec); + } + else + { + size_t tail = (head_ + cnt_) % capacity_; + vec_[tail] = item; + ++cnt_; + } + } + + void Enqueue(T &&item) + { + if (cnt_ == 0) + { + vec_[0] = std::move(item); + head_ = 0; + cnt_ = 1; + } + else if (cnt_ == capacity_) + { + size_t new_capacity = static_cast(capacity_ * 1.5); + std::unique_ptr new_vec = std::make_unique(new_capacity); + + // Before: 0-------Tail-Head---------N-1 + // After: 0----------------------------Tail------------M-1 + // Copy Head --> N-1 + size_t end = 0; + for (size_t idx = head_; idx < capacity_; ++idx, ++end) + { + new_vec[end] = std::move(vec_[idx]); + } + + // Copy 0 --> Tail + for (size_t idx = 0; idx < head_; ++idx, ++end) + { + new_vec[end] = std::move(vec_[idx]); + } + + new_vec[capacity_] = std::move(item); + cnt_ = capacity_ + 1; + capacity_ = new_capacity; + head_ = 0; + vec_ = std::move(new_vec); + } + else + { + size_t tail = (head_ + cnt_) % capacity_; + vec_[tail] = std::move(item); + ++cnt_; + } + } + + void EnqueueAsFirst(const T &item) + { + if (cnt_ == 0) + { + vec_[0] = item; + head_ = 0; + cnt_ = 1; + } + else if (cnt_ == capacity_) + { + size_t new_capacity = static_cast(capacity_ * 1.5); + std::unique_ptr new_vec = std::make_unique(new_capacity); + + // Before: 0-------Tail-Head---------N-1 + // After: 0----------------------------Tail------------M-1 + // Copy Head --> N-1 + std::copy( + vec_.get() + head_, vec_.get() + capacity_, new_vec.get()); + + size_t half_cnt = capacity_ - head_; + // Copy 0 --> Tail + std::copy(vec_.get(), vec_.get() + head_, new_vec.get() + half_cnt); + + new_vec[new_capacity - 1] = item; + head_ = new_capacity - 1; + cnt_ = capacity_ + 1; + capacity_ = new_capacity; + vec_ = std::move(new_vec); + } + else + { + if (head_ != 0) + { + vec_[head_ - 1] = item; + --head_; + } + else + { + vec_[capacity_ - 1] = item; + head_ = capacity_ - 1; + } + ++cnt_; + } + } + + void Dequeue() + { + if (cnt_ > 0) + { + ++head_; + if (head_ == capacity_) + { + head_ = 0; + } + --cnt_; + } + } + + T &Peek() + { + return vec_[head_]; + } + + size_t Size() const + { + return cnt_; + } + + size_t Capacity() const + { + return capacity_; + } + + size_t MemUsage() const + { + return (sizeof(CircularQueue) + capacity_ * sizeof(uint64_t)); + } + + T &Get(size_t index) const + { + return vec_[(head_ + index) % capacity_]; + } + + void Erase(size_t index) + { + while (index < cnt_ - 1) + { + vec_[(head_ + index) % capacity_] = + vec_[(head_ + index + 1) % capacity_]; + index++; + } + cnt_--; + if (cnt_ == 0) + { + head_ = 0; + } + } + +private: + std::unique_ptr vec_; + size_t head_; + size_t cnt_; + size_t capacity_; +}; +#endif // CIRCULAR_QUEUE_H \ No newline at end of file diff --git a/sql/mdl.h b/sql/mdl.h index 92ada782018..a15616ff392 100644 --- a/sql/mdl.h +++ b/sql/mdl.h @@ -23,7 +23,7 @@ #include #include -#include "../data_substrate/tx_service/include/circular_queue.h" +#include "circular_queue.h" class THD; diff --git a/sql/mysql_metrics.h b/sql/mysql_metrics.h index 496f03c799e..c0f98a25965 100644 --- a/sql/mysql_metrics.h +++ b/sql/mysql_metrics.h @@ -1,7 +1,7 @@ #pragma once #include -#include "../data_substrate/eloq_metrics/include/metrics.h" -#include "../data_substrate/eloq_metrics/include/meter.h" +#include "metrics.h" +#include "meter.h" namespace metrics { diff --git a/sql/mysqld.cc b/sql/mysqld.cc index e37163f9c9f..997ccc01eb0 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -108,8 +108,8 @@ #include "semisync_slave.h" #ifdef MYSQLD_LIBRARY_MODE -#include "mysqld_init.h" - +#include +#include // Define synchronization variables namespace mysqld_converged_sync { std::mutex init_mutex; @@ -5566,6 +5566,10 @@ static void test_lc_time_sz() } #endif//DBUG_OFF +void shutdown_mysqld() +{ + break_connect_loop(); +} int mysqld_main(int argc, char **argv) { diff --git a/sql/mysqld_init.h b/sql/mysqld_init.h deleted file mode 100644 index b23f540945d..00000000000 --- a/sql/mysqld_init.h +++ /dev/null @@ -1,25 +0,0 @@ -#ifndef MYSQLD_INIT_INCLUDED -#define MYSQLD_INIT_INCLUDED - -#include -#include - -// Synchronization for converged binary initialization sequence -// This ensures proper order: MySQL basic init → data substrate init → MySQL rest -namespace mysqld_converged_sync { - // Mutex and condition variables for initialization synchronization - extern std::mutex init_mutex; - extern std::condition_variable mysqld_basic_init_done_cv; - extern std::condition_variable data_substrate_init_done_cv; - extern bool mysqld_basic_init_done; - extern bool data_substrate_init_done; -} - -// Function to initialize and run mysqld in library mode -// Returns 0 on success, non-zero on error -extern "C" int mysqld_main(int argc, char **argv); - -// Function to shutdown mysqld gracefully -extern "C" void mysqld_shutdown(); - -#endif // MYSQLD_INIT_INCLUDED diff --git a/sql/sql_class.h b/sql/sql_class.h index 058634ae253..672e2c9689d 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -51,7 +51,6 @@ #include "backup.h" #include "xa.h" #include "ddl_log.h" /* DDL_LOG_STATE */ -#include "../data_substrate/eloq_metrics/include/metrics.h" extern "C" void set_thd_stage_info(void *thd, @@ -3083,7 +3082,7 @@ class THD: public THD_count, /* this must be first */ } } start_time; - metrics::TimePoint tx_start_; + std::chrono::steady_clock::time_point tx_start_; bool tx_registered_{false}; /* Tables changed in transaction (that must be invalidated in query cache). diff --git a/sql/threadpool_generic.h b/sql/threadpool_generic.h index 193c4107472..5502540dbb7 100644 --- a/sql/threadpool_generic.h +++ b/sql/threadpool_generic.h @@ -29,7 +29,7 @@ #endif #ifdef COROUTINE_ENABLED -#include "../data_substrate/tx_service/include/concurrent_queue_wsize.h" +#include "concurrent_queue_wsize.h" #include #include #include diff --git a/storage/eloq/CMakeLists.txt b/storage/eloq/CMakeLists.txt index 25341cca3f9..d67a7fb58bb 100644 --- a/storage/eloq/CMakeLists.txt +++ b/storage/eloq/CMakeLists.txt @@ -228,23 +228,23 @@ include_directories(${MIMALLOC_INCLUDE_DIR}) # include_directories( # ${CMAKE_CURRENT_SOURCE_DIR} -# ${CMAKE_SOURCE_DIR}/data_substrate/ -# ${CMAKE_SOURCE_DIR}/data_substrate/core/include -# ${CMAKE_SOURCE_DIR}/data_substrate/tx_service/include -# ${CMAKE_SOURCE_DIR}/data_substrate/tx_service/include/cc -# ${CMAKE_SOURCE_DIR}/data_substrate/tx_service/include/remote -# ${CMAKE_SOURCE_DIR}/data_substrate/tx_service/include/fault -# ${CMAKE_SOURCE_DIR}/data_substrate/tx_service/include/store -# ${CMAKE_SOURCE_DIR}/data_substrate/tx_service/tx-log-protos -# ${CMAKE_SOURCE_DIR}/data_substrate/eloq_metrics/include -# ${CMAKE_SOURCE_DIR}/data_substrate/store_handler +# ${ELOQSQL_SOURCE_DIR}/data_substrate/ +# ${ELOQSQL_SOURCE_DIR}/data_substrate/core/include +# ${ELOQSQL_SOURCE_DIR}/data_substrate/tx_service/include +# ${ELOQSQL_SOURCE_DIR}/data_substrate/tx_service/include/cc +# ${ELOQSQL_SOURCE_DIR}/data_substrate/tx_service/include/remote +# ${ELOQSQL_SOURCE_DIR}/data_substrate/tx_service/include/fault +# ${ELOQSQL_SOURCE_DIR}/data_substrate/tx_service/include/store +# ${ELOQSQL_SOURCE_DIR}/data_substrate/tx_service/tx-log-protos +# ${ELOQSQL_SOURCE_DIR}/data_substrate/eloq_metrics/include +# ${ELOQSQL_SOURCE_DIR}/data_substrate/store_handler # ) # if(WITH_LOG_SERVICE) # if (OPEN_LOG_SERVICE) -# include_directories(${CMAKE_SOURCE_DIR}/data_substrate/log_service/include) +# include_directories(${ELOQSQL_SOURCE_DIR}/data_substrate/log_service/include) # else() -# include_directories(${CMAKE_SOURCE_DIR}/data_substrate/eloq_log_service/include) +# include_directories(${ELOQSQL_SOURCE_DIR}/data_substrate/eloq_log_service/include) # endif() # endif() diff --git a/storage/eloq/eloq_key_def.cc b/storage/eloq/eloq_key_def.cc index 76a9d57d631..9b509c1935b 100644 --- a/storage/eloq/eloq_key_def.cc +++ b/storage/eloq/eloq_key_def.cc @@ -33,7 +33,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111-1301 USA */ -#include "eloq_schema.h" +#include "eloqsql_schema.h" #ifdef USE_PRAGMA_IMPLEMENTATION #pragma implementation // gcc: Class implementation #endif diff --git a/storage/eloq/eloq_key_def.h b/storage/eloq/eloq_key_def.h index a6d8e53e931..8a76c373b89 100644 --- a/storage/eloq/eloq_key_def.h +++ b/storage/eloq/eloq_key_def.h @@ -56,7 +56,7 @@ #include "field.h" #include "eloq_buff.h" -#include "eloq_schema.h" +#include "eloqsql_schema.h" #include "slice.h" /* The following is copied from storage/innobase/univ.i: */ diff --git a/storage/innobase/CMakeLists.txt b/storage/innobase/CMakeLists.txt index 35fcf1bf6d8..8903a17ff2f 100644 --- a/storage/innobase/CMakeLists.txt +++ b/storage/innobase/CMakeLists.txt @@ -400,7 +400,7 @@ ENDIF() IF(NOT (PLUGIN_INNOBASE STREQUAL DYNAMIC)) TARGET_LINK_LIBRARIES(innobase tpool mysys) - ADD_SUBDIRECTORY(${CURRENT_SOURCE_DIR}/extra/mariabackup ${CMAKE_BINARY_DIR}/extra/mariabackup) + ADD_SUBDIRECTORY(${ELOQSQL_SOURCE_DIR}/extra/mariabackup ${ELOQSQL_BINARY_DIR}/extra/mariabackup) ENDIF() IF(WITH_UNIT_TESTS) diff --git a/storage/innobase/innodb.cmake b/storage/innobase/innodb.cmake index ae83fe8f97d..4f8694fef0c 100644 --- a/storage/innobase/innodb.cmake +++ b/storage/innobase/innodb.cmake @@ -126,8 +126,8 @@ ENDIF() CHECK_FUNCTION_EXISTS(vasprintf HAVE_VASPRINTF) # Include directories under innobase -INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/storage/innobase/include - ${CMAKE_SOURCE_DIR}/storage/innobase/handler) +INCLUDE_DIRECTORIES(${ELOQSQL_SOURCE_DIR}/storage/innobase/include + ${ELOQSQL_SOURCE_DIR}/storage/innobase/handler) # Sun Studio bug with -xO2 IF(CMAKE_CXX_COMPILER_ID MATCHES "SunPro" @@ -150,6 +150,6 @@ IF(MSVC) ENDIF() # Include directories under innobase -INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/storage/innobase/include - ${CMAKE_SOURCE_DIR}/storage/innobase/handler - ${CMAKE_SOURCE_DIR}/libbinlogevents/include ) +INCLUDE_DIRECTORIES(${ELOQSQL_SOURCE_DIR}/storage/innobase/include + ${ELOQSQL_SOURCE_DIR}/storage/innobase/handler + ${ELOQSQL_SOURCE_DIR}/libbinlogevents/include ) diff --git a/storage/innobase/unittest/CMakeLists.txt b/storage/innobase/unittest/CMakeLists.txt index 7dd7c111baa..0ad8dc9b219 100644 --- a/storage/innobase/unittest/CMakeLists.txt +++ b/storage/innobase/unittest/CMakeLists.txt @@ -13,10 +13,10 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA -INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include - ${CMAKE_SOURCE_DIR}/unittest/mytap - ${CMAKE_SOURCE_DIR}/storage/innobase/include - ${CMAKE_SOURCE_DIR}/tpool) +INCLUDE_DIRECTORIES(${ELOQSQL_SOURCE_DIR}/include + ${ELOQSQL_SOURCE_DIR}/unittest/mytap + ${ELOQSQL_SOURCE_DIR}/storage/innobase/include + ${ELOQSQL_SOURCE_DIR}/tpool) ADD_EXECUTABLE(innodb_fts-t innodb_fts-t.cc) TARGET_LINK_LIBRARIES(innodb_fts-t mysys mytap) ADD_DEPENDENCIES(innodb_fts-t GenError) diff --git a/storage/maria/unittest/CMakeLists.txt b/storage/maria/unittest/CMakeLists.txt index a2da1507c97..46281a8ade6 100644 --- a/storage/maria/unittest/CMakeLists.txt +++ b/storage/maria/unittest/CMakeLists.txt @@ -13,8 +13,8 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA -INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include ${CMAKE_SOURCE_DIR}/zlib - ${CMAKE_SOURCE_DIR}/unittest/mytap) +INCLUDE_DIRECTORIES(${ELOQSQL_SOURCE_DIR}/include ${ELOQSQL_SOURCE_DIR}/zlib + ${ELOQSQL_SOURCE_DIR}/unittest/mytap) LINK_LIBRARIES(aria myisam mytap mysys dbug strings ${ZLIB_LIBRARY}) MY_ADD_TESTS(ma_control_file trnman) diff --git a/storage/mroonga/CMakeLists.txt b/storage/mroonga/CMakeLists.txt index c048b214658..cea0dfba10c 100644 --- a/storage/mroonga/CMakeLists.txt +++ b/storage/mroonga/CMakeLists.txt @@ -20,7 +20,7 @@ cmake_minimum_required(VERSION 2.8.12) project(mroonga) -if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}") +if("${ELOQSQL_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}") set(MRN_BUNDLED FALSE) else() set(MRN_BUNDLED TRUE) @@ -75,9 +75,9 @@ endif() set(MRN_PLUGIN_SUFFIX ${CMAKE_SHARED_LIBRARY_SUFFIX}) if(MRN_BUNDLED) - set(MRN_SOURCE_DIR ${CMAKE_SOURCE_DIR}/storage/mroonga) + set(MRN_SOURCE_DIR ${ELOQSQL_SOURCE_DIR}/storage/mroonga) else() - set(MRN_SOURCE_DIR ${CMAKE_SOURCE_DIR}) + set(MRN_SOURCE_DIR ${ELOQSQL_SOURCE_DIR}) endif() file(READ ${MRN_SOURCE_DIR}/version_full MRN_VERSION) @@ -168,9 +168,9 @@ string(REGEX REPLACE "([^;]+)" "${MRN_RELATIVE_DIR_PREFIX}udf/\\1" MRN_UDF_SOURCES "${MRN_UDF_SOURCES}") if(MRN_BUNDLED) - set(MYSQL_SOURCE_DIR ${CMAKE_SOURCE_DIR}) + set(MYSQL_SOURCE_DIR ${ELOQSQL_SOURCE_DIR}) set(MYSQL_BUILD_DIR ${MYSQL_SOURCE_DIR}) - set(MYSQL_CONFIG ${CMAKE_SOURCE_DIR}/scripts/mysql_config) + set(MYSQL_CONFIG ${ELOQSQL_SOURCE_DIR}/scripts/mysql_config) else() set(MYSQL_SOURCE_DIR "/PATH/TO/MYSQL/SOURCE/DIRECTORY/" CACHE PATH "MySQL source directory") @@ -422,7 +422,7 @@ configure_file( "${PROJECT_SOURCE_DIR}/config.sh.in" "${PROJECT_BINARY_DIR}/config.sh") -set(MRN_TEST_SUITE_DIR "${CMAKE_SOURCE_DIR}/mysql-test/suite/mroonga") +set(MRN_TEST_SUITE_DIR "${ELOQSQL_SOURCE_DIR}/mysql-test/suite/mroonga") if(NOT EXISTS "${MRN_TEST_SUITE_DIR}") set(MRN_TEST_SUITE_DIR "${PROJECT_SOURCE_DIR}/mysql-test/mroonga") endif() diff --git a/storage/mroonga/vendor/groonga/CMakeLists.txt b/storage/mroonga/vendor/groonga/CMakeLists.txt index 27afb935467..8efab906952 100644 --- a/storage/mroonga/vendor/groonga/CMakeLists.txt +++ b/storage/mroonga/vendor/groonga/CMakeLists.txt @@ -21,7 +21,7 @@ set(GRN_PROJECT_NAME "groonga") set(GRN_PROJECT_LABEL "Groonga") project("${GRN_PROJECT_NAME}") -if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}") +if("${ELOQSQL_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}") set(GRN_BUNDLED FALSE) else() set(GRN_BUNDLED TRUE) diff --git a/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/CMakeLists.txt b/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/CMakeLists.txt index 4c2aa343089..616cbb363f6 100644 --- a/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/CMakeLists.txt +++ b/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/CMakeLists.txt @@ -32,7 +32,7 @@ set(GROONGA_NORMALIZER_MYSQL_EMBED ${GROONGA_NORMALIZER_MYSQL_EMBED_DEFAULT} file(READ "${CMAKE_CURRENT_SOURCE_DIR}/version_full" VERSION) -if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}") +if("${ELOQSQL_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}") set(GROONGA_NORMALIZER_MYSQL_BUNDLED FALSE) else() set(GROONGA_NORMALIZER_MYSQL_BUNDLED TRUE) @@ -57,7 +57,7 @@ else() endif() include_directories( - ${CMAKE_BINARY_DIR} + ${ELOQSQL_BINARY_DIR} ${GROONGA_INCLUDE_DIRS}) link_directories( diff --git a/storage/perfschema/CMakeLists.txt b/storage/perfschema/CMakeLists.txt index 56ea79755a3..605f338e8e3 100644 --- a/storage/perfschema/CMakeLists.txt +++ b/storage/perfschema/CMakeLists.txt @@ -20,9 +20,9 @@ # along with this program; if not, write to the Free Software Foundation, # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA -INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include - ${CMAKE_SOURCE_DIR}/sql - ${CMAKE_BINARY_DIR}/sql +INCLUDE_DIRECTORIES(${ELOQSQL_SOURCE_DIR}/include + ${ELOQSQL_SOURCE_DIR}/sql + ${ELOQSQL_BINARY_DIR}/sql ${CMAKE_CURRENT_BINARY_DIR} ${PCRE_INCLUDES} ${SSL_INCLUDE_DIRS}) diff --git a/storage/perfschema/unittest/CMakeLists.txt b/storage/perfschema/unittest/CMakeLists.txt index ce20006a00e..5b61157291d 100644 --- a/storage/perfschema/unittest/CMakeLists.txt +++ b/storage/perfschema/unittest/CMakeLists.txt @@ -20,13 +20,13 @@ # along with this program; if not, write to the Free Software Foundation, # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA -INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include - ${CMAKE_SOURCE_DIR}/include/mysql +INCLUDE_DIRECTORIES(${ELOQSQL_SOURCE_DIR}/include + ${ELOQSQL_SOURCE_DIR}/include/mysql ${PCRE_INCLUDES} - ${CMAKE_SOURCE_DIR}/sql + ${ELOQSQL_SOURCE_DIR}/sql ${SSL_INCLUDE_DIRS} - ${CMAKE_SOURCE_DIR}/unittest/mytap - ${CMAKE_SOURCE_DIR}/storage/perfschema) + ${ELOQSQL_SOURCE_DIR}/unittest/mytap + ${ELOQSQL_SOURCE_DIR}/storage/perfschema) ADD_DEFINITIONS(-DMYSQL_SERVER ${SSL_DEFINES}) diff --git a/storage/rocksdb/unittest/CMakeLists.txt b/storage/rocksdb/unittest/CMakeLists.txt index de8d0d82aea..c703cd74b79 100644 --- a/storage/rocksdb/unittest/CMakeLists.txt +++ b/storage/rocksdb/unittest/CMakeLists.txt @@ -1,7 +1,7 @@ IF (TARGET rocksdb) - INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include ${CMAKE_SOURCE_DIR}/zlib - ${CMAKE_SOURCE_DIR}/unittest/mytap - ${CMAKE_SOURCE_DIR}/rocksdb/third-party/gtest-1.7.0/fused-src + INCLUDE_DIRECTORIES(${ELOQSQL_SOURCE_DIR}/include ${ELOQSQL_SOURCE_DIR}/zlib + ${ELOQSQL_SOURCE_DIR}/unittest/mytap + ${ELOQSQL_SOURCE_DIR}/rocksdb/third-party/gtest-1.7.0/fused-src ) LINK_LIBRARIES(mytap mysys dbug strings) diff --git a/storage/spider/CMakeLists.txt b/storage/spider/CMakeLists.txt index 898421fa532..ebe86a1aaec 100644 --- a/storage/spider/CMakeLists.txt +++ b/storage/spider/CMakeLists.txt @@ -40,15 +40,15 @@ IF(EXISTS ${PROJECT_SOURCE_DIR}/storage/mysql_storage_engine.cmake) INCLUDE("${PROJECT_SOURCE_DIR}/storage/mysql_storage_engine.cmake") INCLUDE_DIRECTORIES( - ${CMAKE_SOURCE_DIR}/include - ${CMAKE_SOURCE_DIR}/sql - ${CMAKE_SOURCE_DIR}/regex) + ${ELOQSQL_SOURCE_DIR}/include + ${ELOQSQL_SOURCE_DIR}/sql + ${ELOQSQL_SOURCE_DIR}/regex) MYSQL_STORAGE_ENGINE(SPIDER) ELSEIF(PLUGIN_PARTITION MATCHES "^NO$") MESSAGE(STATUS "Spider is skipped because partitioning is disabled") ELSE() - INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/storage/spider/hs_client) + INCLUDE_DIRECTORIES(${ELOQSQL_SOURCE_DIR}/storage/spider/hs_client) IF(DEB) SET(extra_options COMPONENT spider-engine) diff --git a/strings/CMakeLists.txt b/strings/CMakeLists.txt index 0e62f9e34ad..ffde59f2963 100644 --- a/strings/CMakeLists.txt +++ b/strings/CMakeLists.txt @@ -13,7 +13,7 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA -INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include) +INCLUDE_DIRECTORIES(${ELOQSQL_SOURCE_DIR}/include) SET(STRINGS_SOURCES bchange.c bmove_upp.c ctype-big5.c ctype-bin.c ctype-cp932.c ctype-czech.c ctype-euc_kr.c ctype-eucjpms.c ctype-extra.c ctype-gb2312.c ctype-gbk.c diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 8d4d77590a1..4150a49fe56 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -16,11 +16,11 @@ ADD_DEFINITIONS("-DMYSQL_CLIENT") -INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include - ${CMAKE_SOURCE_DIR}/client) +INCLUDE_DIRECTORIES(${ELOQSQL_SOURCE_DIR}/include + ${ELOQSQL_SOURCE_DIR}/client) INCLUDE_DIRECTORIES(BEFORE - ${CMAKE_BINARY_DIR}/libmariadb/include - ${CMAKE_SOURCE_DIR}/libmariadb/include) + ${ELOQSQL_BINARY_DIR}/libmariadb/include + ${ELOQSQL_SOURCE_DIR}/libmariadb/include) MYSQL_ADD_EXECUTABLE(mariadb-client-test mysql_client_test.c COMPONENT Test) SET(CLIENT_LIB mariadbclient mysys) diff --git a/unittest/embedded/CMakeLists.txt b/unittest/embedded/CMakeLists.txt index cf48550c377..f2fdb68cf88 100644 --- a/unittest/embedded/CMakeLists.txt +++ b/unittest/embedded/CMakeLists.txt @@ -1,8 +1,8 @@ -INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include - ${CMAKE_SOURCE_DIR}/libmysqld/include +INCLUDE_DIRECTORIES(${ELOQSQL_SOURCE_DIR}/include + ${ELOQSQL_SOURCE_DIR}/libmysqld/include ${PCRE_INCLUDES} - ${CMAKE_SOURCE_DIR}/sql + ${ELOQSQL_SOURCE_DIR}/sql ${MY_READLINE_INCLUDE_DIR} ) diff --git a/unittest/json_lib/CMakeLists.txt b/unittest/json_lib/CMakeLists.txt index 1b2a89b28cd..6d12ce81f2b 100644 --- a/unittest/json_lib/CMakeLists.txt +++ b/unittest/json_lib/CMakeLists.txt @@ -13,10 +13,10 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA -INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include - ${CMAKE_SOURCE_DIR}/sql - ${CMAKE_SOURCE_DIR}/regex - ${CMAKE_SOURCE_DIR}/unittest/mytap) +INCLUDE_DIRECTORIES(${ELOQSQL_SOURCE_DIR}/include + ${ELOQSQL_SOURCE_DIR}/sql + ${ELOQSQL_SOURCE_DIR}/regex + ${ELOQSQL_SOURCE_DIR}/unittest/mytap) # MY_ADD_TESTS(json_lib LINK_LIBRARIES strings dbug) diff --git a/unittest/my_decimal/CMakeLists.txt b/unittest/my_decimal/CMakeLists.txt index 0b5b228276d..879904358fb 100644 --- a/unittest/my_decimal/CMakeLists.txt +++ b/unittest/my_decimal/CMakeLists.txt @@ -13,10 +13,10 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA -INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include - ${CMAKE_SOURCE_DIR}/sql - ${CMAKE_SOURCE_DIR}/regex - ${CMAKE_SOURCE_DIR}/unittest/mytap) +INCLUDE_DIRECTORIES(${ELOQSQL_SOURCE_DIR}/include + ${ELOQSQL_SOURCE_DIR}/sql + ${ELOQSQL_SOURCE_DIR}/regex + ${ELOQSQL_SOURCE_DIR}/unittest/mytap) # MY_ADD_TESTS(my_decimal EXT "cc" LINK_LIBRARIES strings dbug) diff --git a/unittest/mytap/CMakeLists.txt b/unittest/mytap/CMakeLists.txt index 189ea5a1c00..b48df55e579 100644 --- a/unittest/mytap/CMakeLists.txt +++ b/unittest/mytap/CMakeLists.txt @@ -13,5 +13,5 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA -INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include) +INCLUDE_DIRECTORIES(${ELOQSQL_SOURCE_DIR}/include) ADD_LIBRARY(mytap tap.c) diff --git a/unittest/sql/CMakeLists.txt b/unittest/sql/CMakeLists.txt index b8682de74c3..cc66b8a98d1 100644 --- a/unittest/sql/CMakeLists.txt +++ b/unittest/sql/CMakeLists.txt @@ -15,10 +15,10 @@ MY_ADD_TESTS(my_apc LINK_LIBRARIES mysys EXT cc) -INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/sql - ${CMAKE_SOURCE_DIR}/include - ${CMAKE_SOURCE_DIR}/unittest/mytap - ${CMAKE_SOURCE_DIR}/extra/yassl/include) +INCLUDE_DIRECTORIES(${ELOQSQL_SOURCE_DIR}/sql + ${ELOQSQL_SOURCE_DIR}/include + ${ELOQSQL_SOURCE_DIR}/unittest/mytap + ${ELOQSQL_SOURCE_DIR}/extra/yassl/include) ADD_EXECUTABLE(explain_filename-t explain_filename-t.cc dummy_builtins.cc) diff --git a/vio/CMakeLists.txt b/vio/CMakeLists.txt index 85810840273..eb0c1c7e4fe 100644 --- a/vio/CMakeLists.txt +++ b/vio/CMakeLists.txt @@ -13,7 +13,7 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA -INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include +INCLUDE_DIRECTORIES(${ELOQSQL_SOURCE_DIR}/include ${SSL_INCLUDE_DIRS}) ADD_DEFINITIONS(${SSL_DEFINES}) diff --git a/win/packaging/CMakeLists.txt b/win/packaging/CMakeLists.txt index a4d951bfa5c..53f5617bda1 100644 --- a/win/packaging/CMakeLists.txt +++ b/win/packaging/CMakeLists.txt @@ -141,7 +141,7 @@ IF(WITH_THIRD_PARTY) IF(THIRD_PARTY_DOWNLOAD_LOCATION) FILE(TO_CMAKE_PATH "${THIRD_PARTY_DOWNLOAD_LOCATION}" THIRD_PARTY_DOWNLOAD_LOCATION) ELSE() - SET(THIRD_PARTY_DOWNLOAD_LOCATION "${CMAKE_BINARY_DIR}") + SET(THIRD_PARTY_DOWNLOAD_LOCATION "${ELOQSQL_BINARY_DIR}") ENDIF() ENDIF() @@ -196,7 +196,7 @@ ADD_CUSTOM_TARGET( -DTHIRD_PARTY_DOWNLOAD_LOCATION="${THIRD_PARTY_DOWNLOAD_LOCATION}" -DTHIRD_PARTY_FEATURE_CONDITION="${THIRD_PARTY_FEATURE_CONDITION}" -DTINY_VERSION="${TINY_VERSION}" - -DTOP_BINDIR="${CMAKE_BINARY_DIR}" + -DTOP_BINDIR="${ELOQSQL_BINARY_DIR}" -DVERSION="${VERSION}" -DWITH_THIRD_PARTY="${WITH_THIRD_PARTY}" -DWIXCA_LOCATION="$" @@ -221,19 +221,19 @@ ADD_CUSTOM_TARGET( win_package_zip ${SIGN_COMMAND} COMMAND ${CMAKE_CPACK_COMMAND} ${CPACK_CONFIG_PARAM} --config ${CMAKE_CURRENT_SOURCE_DIR}/CPackZipConfig.cmake - WORKING_DIRECTORY ${CMAKE_BINARY_DIR} + WORKING_DIRECTORY ${ELOQSQL_BINARY_DIR} ) ADD_CUSTOM_TARGET( win_package_debuginfo COMMAND ${CMAKE_CPACK_COMMAND} ${CPACK_CONFIG_PARAM} --config ${CMAKE_CURRENT_SOURCE_DIR}/CPackZipDebugInfoConfig.cmake - WORKING_DIRECTORY ${CMAKE_BINARY_DIR} + WORKING_DIRECTORY ${ELOQSQL_BINARY_DIR} ) ADD_CUSTOM_TARGET( win_package_test COMMAND ${CMAKE_CPACK_COMMAND} ${CPACK_CONFIG_PARAM} --config ${CMAKE_CURRENT_SOURCE_DIR}/CPackZipTestConfig.cmake - WORKING_DIRECTORY ${CMAKE_BINARY_DIR} + WORKING_DIRECTORY ${ELOQSQL_BINARY_DIR} ) ADD_CUSTOM_TARGET(win_package DEPENDS win_package_zip win_package_debuginfo) diff --git a/win/packaging/ca/CMakeLists.txt b/win/packaging/ca/CMakeLists.txt index 368a844f830..4302645a64e 100644 --- a/win/packaging/ca/CMakeLists.txt +++ b/win/packaging/ca/CMakeLists.txt @@ -15,10 +15,10 @@ SET(WIXCA_SOURCES CustomAction.cpp CustomAction.def) -INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/sql ${CMAKE_CURRENT_SOURCE_DIR} ${WIX_INCLUDE_DIR}) +INCLUDE_DIRECTORIES(${ELOQSQL_SOURCE_DIR}/sql ${CMAKE_CURRENT_SOURCE_DIR} ${WIX_INCLUDE_DIR}) # Custom action should not depend on C runtime, since we do not know if CRT is installed. FORCE_STATIC_CRT() ADD_VERSION_INFO(wixca SHARED WIXCA_SOURCES) -ADD_LIBRARY(wixca SHARED EXCLUDE_FROM_ALL ${WIXCA_SOURCES} ${CMAKE_SOURCE_DIR}/sql/winservice.c) +ADD_LIBRARY(wixca SHARED EXCLUDE_FROM_ALL ${WIXCA_SOURCES} ${ELOQSQL_SOURCE_DIR}/sql/winservice.c) TARGET_LINK_LIBRARIES(wixca ${WIX_WCAUTIL_LIBRARY} ${WIX_DUTIL_LIBRARY} msi version) diff --git a/win/upgrade_wizard/CMakeLists.txt b/win/upgrade_wizard/CMakeLists.txt index 20a06a41215..9560372a6dc 100644 --- a/win/upgrade_wizard/CMakeLists.txt +++ b/win/upgrade_wizard/CMakeLists.txt @@ -20,7 +20,7 @@ IF(MSVC_CRT_TYPE MATCHES "/MD") # FORCE static CRT and MFC for upgrade wizard, # so we do not have to redistribute MFC. FORCE_STATIC_CRT() - SET(UPGRADE_WIZARD_SOURCES ${CMAKE_SOURCE_DIR}/sql/winservice.c) + SET(UPGRADE_WIZARD_SOURCES ${ELOQSQL_SOURCE_DIR}/sql/winservice.c) ELSE() SET(UPGRADE_WIZARD_LINK_LIBRARIES winservice) ENDIF() @@ -31,7 +31,7 @@ SET(CMAKE_MFC_FLAG 1) # Enable exception handling (avoids warnings) SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /EHsc -DNO_WARN_MBCS_MFC_DEPRECATION") -INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/sql) +INCLUDE_DIRECTORIES(${ELOQSQL_SOURCE_DIR}/sql) MYSQL_ADD_EXECUTABLE(mariadb-upgrade-wizard upgrade.cpp upgradeDlg.cpp upgrade.rc ${UPGRADE_WIZARD_SOURCES} COMPONENT Server) From 8f7b657439237221205f47db0530e9ca47590018 Mon Sep 17 00:00:00 2001 From: liunyl Date: Thu, 23 Oct 2025 10:06:03 +0000 Subject: [PATCH 06/10] update latest submodule --- CMakeLists.txt | 6 +- data_substrate/CMakeLists.txt | 10 +- data_substrate/core/include/data_substrate.h | 17 +- .../core/include/glog_error_logging.h | 139 ++++++ data_substrate/core/include/mysql_metrics.h | 2 +- data_substrate/core/include/redis_metrics.h | 69 +++ data_substrate/core/src/data_substrate.cpp | 11 +- data_substrate/core/src/metrics_init.cpp | 321 +++++++----- data_substrate/core/src/tx_service_init.cpp | 463 ++++++++++-------- data_substrate/store_handler | 2 +- data_substrate/tx_service | 2 +- sql/main.cc | 13 +- sql/mysqld.cc | 50 +- storage/eloq/CMakeLists.txt | 246 ++-------- storage/eloq/ha_eloq.cc | 44 +- 15 files changed, 745 insertions(+), 650 deletions(-) create mode 100644 data_substrate/core/include/glog_error_logging.h create mode 100644 data_substrate/core/include/redis_metrics.h diff --git a/CMakeLists.txt b/CMakeLists.txt index fd19baa01a9..1952cc8cf6a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -43,10 +43,10 @@ MESSAGE(STATUS "Running cmake version ${CMAKE_VERSION}") # Option to build as library for converged binary option(BUILD_ELOQSQL_AS_LIBRARY "Build eloqsql as library instead of executable" OFF) -IF(BUILD_ELOQSQL_AS_LIBRARY) +# IF(BUILD_ELOQSQL_AS_LIBRARY) SET(ELOQSQL_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) SET(ELOQSQL_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}) -ENDIF() +# ENDIF() SET(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${ELOQSQL_SOURCE_DIR}/cmake ${ELOQSQL_SOURCE_DIR}/cmake/Internal/CPack) @@ -461,7 +461,7 @@ MARK_AS_ADVANCED(PYTHON_SHEBANG) # Add storage engines and plugins. CONFIGURE_PLUGINS() - +add_compile_definitions(ELOQ_MODULE_ELOQSQL) # Build data substrate library first (before sql/ and storage/) if(NOT BUILD_ELOQSQL_AS_LIBRARY) # Build data substrate library first (standalone mode) diff --git a/data_substrate/CMakeLists.txt b/data_substrate/CMakeLists.txt index e6c7dcfc5b7..6fb9616f990 100644 --- a/data_substrate/CMakeLists.txt +++ b/data_substrate/CMakeLists.txt @@ -17,13 +17,15 @@ option(EXT_TX_PROC_ENABLED "Allows external threads to move forward the tx servi option(ELOQ_MODULE_ENABLED "Register the tx service as an ELOQ module." OFF) -# TODO: Remove this after testing -add_compile_definitions(ELOQ_MODULE_ELOQSQL) - message(NOTICE "WITH_DATA_STORE: ${WITH_DATA_STORE}") message(NOTICE "WITH_LOG_SERVICE: ${WITH_LOG_SERVICE}") message(NOTICE "OPEN_LOG_SERVICE: ${OPEN_LOG_SERVICE}") +option(USE_ONE_ELOQDSS_PARTITION_ENABLED "Whether use one partition for kv store" OFF) +message(NOTICE "USE_ONE_ELOQDSS_PARTITION_ENABLED : ${USE_ONE_ELOQDSS_PARTITION_ENABLED}") +if(USE_ONE_ELOQDSS_PARTITION_ENABLED) + add_compile_definitions(USE_ONE_ELOQDSS_PARTITION) +endif() # Add compile definitions based on options if(WITH_DATA_STORE STREQUAL "DYNAMODB") set(KV_STORAGE_VAL 1 CACHE STRING "dynamodb" FORCE) @@ -156,7 +158,6 @@ if(STATISTICS) add_compile_definitions(STATISTICS) endif() -add_compile_definitions(ELOQ_MODULE_ELOQSQL) # Include build files for subcomponents INCLUDE(build_tx_service.cmake) @@ -278,6 +279,7 @@ TARGET_LINK_LIBRARIES(${DATA_SUBSTRATE_LIB} ${DATA_SUBSTRATE_LINK_LIBS}) # Export include directories for other targets TARGET_INCLUDE_DIRECTORIES(${DATA_SUBSTRATE_LIB} PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/core/include ${CMAKE_CURRENT_SOURCE_DIR}/tx_service/include ${CMAKE_CURRENT_SOURCE_DIR}/eloq_metrics/include diff --git a/data_substrate/core/include/data_substrate.h b/data_substrate/core/include/data_substrate.h index 792250a8c10..f9a801da4d3 100644 --- a/data_substrate/core/include/data_substrate.h +++ b/data_substrate/core/include/data_substrate.h @@ -62,6 +62,7 @@ #include #endif #include "metrics.h" +#include "meter.h" #include "type.h" // Forward declaration for INIReader @@ -205,6 +206,10 @@ class DataSubstrate NetworkConfig network_config_; LogServiceConfig log_service_config_; metrics::CommonLabels tx_service_common_labels_{}; + std::vector>> + external_metrics = {}; // TODO(liunyl): system handler is used to refresh auth related cache. In // converged db, there should only be one system handler. txservice::SystemHandler *system_handler_{nullptr}; @@ -226,22 +231,10 @@ class DataSubstrate // Engine registry EngineConfig engines_[NUM_EXTERNAL_ENGINES]; -// #ifdef ELOQ_MODULE_ELOQKV -// EloqKV::RedisCatalogFactory eloqkv_catalog_factory_; -// #endif - -// #ifdef ELOQ_MODULE_ELOQSQL -// MyEloq::MariaCatalogFactory eloqsql_catalog_factory_; -// #endif - std::unordered_map prebuilt_tables_; }; -// Global DataSubstrate instance -extern std::unique_ptr g_data_substrate; - - // Helper function to check if a gflag was set from command line static inline bool CheckCommandLineFlagIsDefault(const char *name) { diff --git a/data_substrate/core/include/glog_error_logging.h b/data_substrate/core/include/glog_error_logging.h new file mode 100644 index 00000000000..46a8b8c42f7 --- /dev/null +++ b/data_substrate/core/include/glog_error_logging.h @@ -0,0 +1,139 @@ +/** + * Copyright (C) 2025 EloqData Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under either of the following two licenses: + * 1. GNU Affero General Public License, version 3, as published by the Free + * Software Foundation. + * 2. GNU General Public License as published by the Free Software + * Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License or GNU General Public License for more + * details. + * + * You should have received a copy of the GNU Affero General Public License + * and GNU General Public License V2 along with this program. If not, see + * . + * + */ +#pragma once +#include +#include + +#include +#include +#include +#include + +DECLARE_string(log_file_name_prefix); + +inline void CustomPrefix(std::ostream &s, + const google::LogMessageInfo &l, + void *) +{ + s << "[" // + << std::setw(4) << 1900 + l.time.year() // YY + << '-' // - + << std::setw(2) << 1 + l.time.month() // MM + << '-' // - + << std::setw(2) << l.time.day() // DD + << 'T' // T + << std::setw(2) << l.time.hour() // hh + << ':' // : + << std::setw(2) << l.time.min() // mm + << ':' // : + << std::setw(2) << l.time.sec() // ss + << '.' // . + << std::setfill('0') << std::setw(6) // + << l.time.usec() // usec + << " " << l.severity[0] << " " + << "" << l.thread_id << "] " +#ifndef DISABLE_CODE_LINE_IN_LOG + << "[" << l.filename << ':' << l.line_number << "]"; +#else + ; +#endif +}; + +inline void InitGoogleLogging(char **argv) +{ + // If `GLOG_logtostderr` is specified then log to `stderr` only . + // + // NOTE: This is for cases where disk space needs protection, like when + // deployed in the cloud. + if (FLAGS_logtostderr && FLAGS_log_dir.empty()) + { + FLAGS_alsologtostderr = false; + FLAGS_logtostdout = false; + } + else + { + // Log to `stderr` and `GLOG_log_dir/logfiles`. + // + // NOTE: If `GLOG_log_dir` is not specified then it will be default to + // `path/to/eloqkv/logs` + if (FLAGS_log_dir.empty()) + { + // Get the absolute path of the bin directory + char bin_path[PATH_MAX]; + ssize_t len = readlink("/proc/self/exe", bin_path, PATH_MAX); + std::filesystem::path fullPath(std::string(bin_path, len)); + std::filesystem::path dir_path = + fullPath.parent_path().parent_path(); + FLAGS_log_dir = dir_path.string() + "/logs"; + } + + if (!std::filesystem::exists(FLAGS_log_dir)) + { + std::filesystem::create_directories(FLAGS_log_dir); + } + + // Log to stderr and logfiles + // FLAGS_alsologtostderr = true; + + // NOTE: Enable this will log to `stdout` instead of logfiles. + FLAGS_logtostdout = false; + + // NOTE: Enable this will log to `stderr` instead of logfiles. + FLAGS_logtostderr = false; + + // Log INFO/WARNING/ERROR/FATAL + FLAGS_minloglevel = 0; + + // stderrthreshold (log messages at or above this level are copied to + // stderr in addition to logfiles.) default: 2. + FLAGS_stderrthreshold = google::GLOG_FATAL; + + // Don't buffer anything. NOTE: If `logtostderr` or `logtostdout` is + // `true` then glog will force this value to -1. + FLAGS_logbuflevel = -1; + + FLAGS_log_file_header = false; + + auto log_file_name_prefix = std::getenv("GLOG_log_file_name_prefix"); + FLAGS_log_file_name_prefix = log_file_name_prefix == NULL + ? FLAGS_log_file_name_prefix + : log_file_name_prefix; + auto log_file_prefix = + FLAGS_log_dir + "/" + FLAGS_log_file_name_prefix + "."; + + // Configure log destinations. + google::SetLogDestination(google::INFO, + (log_file_prefix + "INFO.").c_str()); + google::SetLogDestination(google::WARNING, + (log_file_prefix + "WARNING.").c_str()); + google::SetLogDestination(google::ERROR, + (log_file_prefix + "ERROR.").c_str()); + + // Configure symlink for logfiles. + google::SetLogSymlink(google::INFO, FLAGS_log_file_name_prefix.c_str()); + google::SetLogSymlink(google::WARNING, + FLAGS_log_file_name_prefix.c_str()); + google::SetLogSymlink(google::ERROR, + FLAGS_log_file_name_prefix.c_str()); + } + google::InitGoogleLogging(argv[0], &CustomPrefix); +} diff --git a/data_substrate/core/include/mysql_metrics.h b/data_substrate/core/include/mysql_metrics.h index c0f98a25965..b7b0f427f9b 100644 --- a/data_substrate/core/include/mysql_metrics.h +++ b/data_substrate/core/include/mysql_metrics.h @@ -14,7 +14,7 @@ inline const Name NAME_MYSQL_TX_DURATION{"mysql_tx_duration"}; inline const Name NAME_MYSQL_PROCESSED_TX_TOTAL{"mysql_processed_tx_total"}; inline const Name NAME_MYSQL_PROCESSED_QUERY_TOTAL{ "mysql_processed_query_total"}; -inline const Name NAME_CONNECTION_COUNT{"mysql_connection_count"}; +inline const Name NAME_MYSQL_CONNECTION_COUNT{"mysql_connection_count"}; inline const Name NAME_MAX_CONNECTIONS{"mysql_max_connections"}; inline const Name NAME_MYSQL_DML_TOTAL{"mysql_dml_total"}; inline const Name NAME_MYSQL_DML_DURATION{"mysql_dml_duration"}; diff --git a/data_substrate/core/include/redis_metrics.h b/data_substrate/core/include/redis_metrics.h new file mode 100644 index 00000000000..3d928fb3611 --- /dev/null +++ b/data_substrate/core/include/redis_metrics.h @@ -0,0 +1,69 @@ +/** + * Copyright (C) 2025 EloqData Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under either of the following two licenses: + * 1. GNU Affero General Public License, version 3, as published by the Free + * Software Foundation. + * 2. GNU General Public License as published by the Free Software + * Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License or GNU General Public License for more + * details. + * + * You should have received a copy of the GNU Affero General Public License + * and GNU General Public License V2 along with this program. If not, see + * . + * + */ +#pragma once + +#include +#include +#include + +#include "metrics.h" +#include "meter.h" + +namespace metrics +{ +inline const Name NAME_REDIS_CONNECTION_COUNT{"redis_connection_count"}; +inline const Name NAME_MAX_CONNECTION{"redis_max_connections"}; + +inline const Name NAME_REDIS_COMMAND_TOTAL{"redis_command_total"}; +inline const Name NAME_REDIS_COMMAND_DURATION{"redis_command_duration"}; + +inline const Name NAME_REDIS_COMMAND_AGGREGATED_TOTAL{ + "redis_command_aggregated_total"}; +inline const Name NAME_REDIS_COMMAND_AGGREGATED_DURATION{ + "redis_command_aggregated_duration"}; + +inline const Name NAME_REDIS_SLOW_LOG_LEN{"redis_slow_log_len"}; + +inline size_t collect_redis_command_duration_round{0}; +inline std::unique_ptr redis_meter{nullptr}; + +inline void register_redis_metrics(MetricsRegistry *metrics_registry, + CommonLabels &common_labels, + size_t core_num) +{ + redis_meter = std::make_unique(metrics_registry, common_labels); + redis_meter->Register(metrics::NAME_REDIS_CONNECTION_COUNT, + metrics::Type::Gauge); + redis_meter->Register(metrics::NAME_MAX_CONNECTION, metrics::Type::Gauge); + std::vector labels; + labels.emplace_back("core_id", std::vector()); + for (size_t idx = 0; idx < core_num; ++idx) + { + labels[0].second.push_back(std::to_string(idx)); + } + + redis_meter->Register(metrics::NAME_REDIS_SLOW_LOG_LEN, + metrics::Type::Gauge, + std::move(labels)); + +} +} // namespace metrics diff --git a/data_substrate/core/src/data_substrate.cpp b/data_substrate/core/src/data_substrate.cpp index 47c67fdfd42..cef219f313f 100644 --- a/data_substrate/core/src/data_substrate.cpp +++ b/data_substrate/core/src/data_substrate.cpp @@ -171,6 +171,10 @@ DataSubstrate::~DataSubstrate() void DataSubstrate::Shutdown() { +#ifdef ELOQ_MODULE_ELOQSQL + system_handler_->Shutdown(); + system_handler_ = nullptr; +#endif if (tx_service_ != nullptr) { LOG(INFO) << "Shutting down the tx service."; @@ -200,9 +204,6 @@ void DataSubstrate::Shutdown() } #endif txservice::Sequences::Destory(); -// #ifdef ELOQ_MODULE_ELOQSQL -// MyEloq::MariaSystemHandler::Instance().Shutdown(); -// #endif #if defined(DATA_STORE_TYPE_DYNAMODB) || \ defined(DATA_STORE_TYPE_ELOQDSS_ROCKSDB_CLOUD_S3) || \ @@ -219,6 +220,7 @@ bool DataSubstrate::RegisterEngines() engines_[1]= {nullptr, txservice::TableEngine::EloqKv, false}; engines_[2]= {nullptr, txservice::TableEngine::EloqDoc, false}; + prebuilt_tables_.try_emplace(txservice::Sequences::table_name_, std::string(txservice::Sequences::table_name_sv_)); #ifdef ELOQ_MODULE_ELOQSQL engines_[0].enable_engine= true; if (eloqsql_catalog_factory) { @@ -486,10 +488,11 @@ bool DataSubstrate::LoadCoreAndNetworkConfig(const INIReader &config_reader) // Set bthread concurrency GFLAGS_NAMESPACE::SetCommandLineOption( "bthread_concurrency", std::to_string(core_config_.core_num).c_str()); - // TODO(liunyl): wrap with eloq module macro? +#ifdef ELOQ_MODULE_ENABLED GFLAGS_NAMESPACE::SetCommandLineOption("worker_polling_time_us", "1000"); GFLAGS_NAMESPACE::SetCommandLineOption("brpc_worker_as_ext_processor", "true"); +#endif GFLAGS_NAMESPACE::SetCommandLineOption("use_pthread_event_dispatcher", "true"); GFLAGS_NAMESPACE::SetCommandLineOption("max_body_size", "536870912"); diff --git a/data_substrate/core/src/metrics_init.cpp b/data_substrate/core/src/metrics_init.cpp index a638c4f8ca4..713ef064f32 100644 --- a/data_substrate/core/src/metrics_init.cpp +++ b/data_substrate/core/src/metrics_init.cpp @@ -1,163 +1,216 @@ +#include "INIReader.h" #include "data_substrate.h" -#include "metrics.h" -#include "glog/logging.h" #include "gflags/gflags.h" +#include "glog/logging.h" +#include "metrics.h" #include "metrics_registry_impl.h" -#include "tx_service_metrics.h" #include "store/data_store_handler.h" -#include "INIReader.h" +#include "tx_service_metrics.h" #if (WITH_LOG_SERVICE) #include "log_service_metrics.h" #endif #ifdef ELOQ_MODULE_ELOQSQL -#include "mysql_metrics.h" //TDOO(liunyl): move this file to correct location +#include "mysql_metrics.h" //TDOO(liunyl): move this file to correct location +#endif +#ifdef ELOQ_MODULE_ELOQKV +#include "redis_metrics.h" #endif DEFINE_bool(enable_metrics, false, "Enable metrics"); DEFINE_string(metrics_port, "18081", "Metrics port"); +namespace EloqKV +{ +enum struct RedisCommandType; +extern const std::vector> + command_types; +} // namespace EloqKV bool DataSubstrate::InitializeMetrics(const INIReader &config_reader) { - /* Parse metrics config */ - metrics::enable_metrics= - config_reader.GetBoolean("metrics", "enable_metrics", false); - DLOG(INFO) << "enable_metrics: " << (metrics::enable_metrics ? "ON" : "OFF"); - if (metrics::enable_metrics) - { - FLAGS_metrics_port= std::to_string(config_reader.GetInteger( - "metrics", "metrics_port", std::stoi(FLAGS_metrics_port))); - - LOG(INFO) << "metrics_port: " << FLAGS_metrics_port; - - // global metrics - metrics::enable_memory_usage= - config_reader.GetBoolean("metrics", "enable_memory_usage", true); - LOG(INFO) << "enable_memory_usage: " - << (metrics::enable_memory_usage ? "ON" : "OFF"); - if (metrics::enable_memory_usage) - { - metrics::collect_memory_usage_round= config_reader.GetInteger( - "metrics", "collect_memory_usage_round", 10000); - LOG(INFO) << "collect memory usage every " - << metrics::collect_memory_usage_round << " round(s)"; - } - - metrics::enable_cache_hit_rate= - config_reader.GetBoolean("metrics", "enable_cache_hit_rate", true); - LOG(INFO) << "enable_cache_hit_rate: " - << (metrics::enable_cache_hit_rate ? "ON" : "OFF"); - - // tx metrics - metrics::enable_tx_metrics= - config_reader.GetBoolean("metrics", "enable_tx_metrics", true); - LOG(INFO) << "enable_tx_metrics: " - << (metrics::enable_tx_metrics ? "ON" : "OFF"); - if (metrics::enable_tx_metrics) - { - metrics::collect_tx_duration_round= - config_reader.GetInteger("metrics", "collect_tx_duration_round", - metrics::collect_tx_duration_round); - LOG(INFO) << "collect tx duration every " - << metrics::collect_tx_duration_round << " round(s)"; - } - - // busy round metrics - metrics::enable_busy_round_metrics= - config_reader.GetBoolean("metrics", "enable_busy_round_metrics", true); - LOG(INFO) << "enable_busy_round_metrics: " - << (metrics::enable_busy_round_metrics ? "ON" : "OFF"); - if (metrics::enable_busy_round_metrics) - { - metrics::busy_round_threshold= - config_reader.GetInteger("metrics", "busy_round_threshold", 10); - LOG(INFO) << "collect busy round metrics when reaching the " - "busy round " - "threshold " - << metrics::busy_round_threshold; - } - - // remote request metrics - metrics::enable_remote_request_metrics= config_reader.GetBoolean( - "metrics", "enable_busy_round_metrics", metrics::enable_tx_metrics); - LOG(INFO) << "enable_remote_request_metrics: " - << (metrics::enable_remote_request_metrics ? "ON" : "OFF"); - if (core_config_.enable_data_store) + /* Parse metrics config */ + metrics::enable_metrics = + config_reader.GetBoolean("metrics", "enable_metrics", false); + DLOG(INFO) << "enable_metrics: " + << (metrics::enable_metrics ? "ON" : "OFF"); + if (metrics::enable_metrics) { - metrics::enable_kv_metrics= - config_reader.GetBoolean("metrics", "enable_kv_metrics", true); - LOG(INFO) << "enable_kv_metrics: " - << (metrics::enable_kv_metrics ? "ON" : "OFF"); - } + FLAGS_metrics_port = std::to_string(config_reader.GetInteger( + "metrics", "metrics_port", std::stoi(FLAGS_metrics_port))); + + LOG(INFO) << "metrics_port: " << FLAGS_metrics_port; + + // global metrics + metrics::enable_memory_usage = + config_reader.GetBoolean("metrics", "enable_memory_usage", true); + LOG(INFO) << "enable_memory_usage: " + << (metrics::enable_memory_usage ? "ON" : "OFF"); + if (metrics::enable_memory_usage) + { + metrics::collect_memory_usage_round = config_reader.GetInteger( + "metrics", "collect_memory_usage_round", 10000); + LOG(INFO) << "collect memory usage every " + << metrics::collect_memory_usage_round << " round(s)"; + } + + metrics::enable_cache_hit_rate = + config_reader.GetBoolean("metrics", "enable_cache_hit_rate", true); + LOG(INFO) << "enable_cache_hit_rate: " + << (metrics::enable_cache_hit_rate ? "ON" : "OFF"); + + // tx metrics + metrics::enable_tx_metrics = + config_reader.GetBoolean("metrics", "enable_tx_metrics", true); + LOG(INFO) << "enable_tx_metrics: " + << (metrics::enable_tx_metrics ? "ON" : "OFF"); + if (metrics::enable_tx_metrics) + { + metrics::collect_tx_duration_round = + config_reader.GetInteger("metrics", + "collect_tx_duration_round", + metrics::collect_tx_duration_round); + LOG(INFO) << "collect tx duration every " + << metrics::collect_tx_duration_round << " round(s)"; + } + + // busy round metrics + metrics::enable_busy_round_metrics = config_reader.GetBoolean( + "metrics", "enable_busy_round_metrics", true); + LOG(INFO) << "enable_busy_round_metrics: " + << (metrics::enable_busy_round_metrics ? "ON" : "OFF"); + if (metrics::enable_busy_round_metrics) + { + metrics::busy_round_threshold = + config_reader.GetInteger("metrics", "busy_round_threshold", 10); + LOG(INFO) << "collect busy round metrics when reaching the " + "busy round " + "threshold " + << metrics::busy_round_threshold; + } + + // remote request metrics + metrics::enable_remote_request_metrics = config_reader.GetBoolean( + "metrics", "enable_busy_round_metrics", metrics::enable_tx_metrics); + LOG(INFO) << "enable_remote_request_metrics: " + << (metrics::enable_remote_request_metrics ? "ON" : "OFF"); + if (core_config_.enable_data_store) + { + metrics::enable_kv_metrics = + config_reader.GetBoolean("metrics", "enable_kv_metrics", true); + LOG(INFO) << "enable_kv_metrics: " + << (metrics::enable_kv_metrics ? "ON" : "OFF"); + } #if (WITH_LOG_SERVICE) - if (core_config_.enable_wal) - { - // log_service metrics - metrics::enable_log_service_metrics= config_reader.GetBoolean( - "metrics", "enable_log_service_metrics", true); - LOG(INFO) << "enable_log_service_metrics: " - << (metrics::enable_log_service_metrics ? "ON" : "OFF"); - } + if (core_config_.enable_wal) + { + // log_service metrics + metrics::enable_log_service_metrics = config_reader.GetBoolean( + "metrics", "enable_log_service_metrics", true); + LOG(INFO) << "enable_log_service_metrics: " + << (metrics::enable_log_service_metrics ? "ON" : "OFF"); + } #endif - // failed forward msgs metrics - metrics::enable_standby_metrics= - config_reader.GetBoolean("metrics", "enable_standby_metrics", true); - LOG(INFO) << "enable standby metrics: " - << (metrics::enable_standby_metrics ? "ON" : "OFF"); - if (metrics::enable_standby_metrics) - { - metrics::collect_standby_metrics_round= config_reader.GetInteger( - "metrics", "collect_standby_metrics_round", 10000); - LOG(INFO) << "collect standby metrics every " - << metrics::collect_standby_metrics_round << " round(s)"; - } + // failed forward msgs metrics + metrics::enable_standby_metrics = + config_reader.GetBoolean("metrics", "enable_standby_metrics", true); + LOG(INFO) << "enable standby metrics: " + << (metrics::enable_standby_metrics ? "ON" : "OFF"); + if (metrics::enable_standby_metrics) + { + metrics::collect_standby_metrics_round = config_reader.GetInteger( + "metrics", "collect_standby_metrics_round", 10000); + LOG(INFO) << "collect standby metrics every " + << metrics::collect_standby_metrics_round << " round(s)"; + } #ifdef ELOQ_MODULE_ELOQSQL - metrics::enable_mysql_tx_metrics= - config_reader.GetBoolean("metrics", "enable_mysql_tx_metrics", true); - metrics::enable_mysql_dml_metrics= - config_reader.GetBoolean("metrics", "enable_mysql_dml_metrics", true); + metrics::enable_mysql_tx_metrics = config_reader.GetBoolean( + "metrics", "enable_mysql_tx_metrics", true); + metrics::enable_mysql_dml_metrics = config_reader.GetBoolean( + "metrics", "enable_mysql_dml_metrics", true); #endif - setenv("ELOQ_METRICS_PORT", FLAGS_metrics_port.c_str(), false); - eloq_metrics_app::MetricsRegistryImpl::MetricsRegistryResult - metrics_registry_result= - eloq_metrics_app::MetricsRegistryImpl::GetRegistry(); - - if (metrics_registry_result.not_ok_ != nullptr) - { - LOG(ERROR) << "!!!!!!!! Failed to initialize MetricsRegristry !!!!!!!!"; - return false; - } - - metrics_registry_= std::move(metrics_registry_result.metrics_registry_); - - // TODO(liunyl): Finish all metrics registration before collect metrics. - // We might need to finish all registration before starting tx service. - if (core_config_.enable_data_store) - { - metrics::CommonLabels kv_common_common_labels{}; - kv_common_common_labels["node_ip"]= network_config_.local_ip; - kv_common_common_labels["node_port"]= - std::to_string(network_config_.local_port); - store_hd_->RegisterKvMetrics(metrics_registry_.get(), - kv_common_common_labels); - } + setenv("ELOQ_METRICS_PORT", FLAGS_metrics_port.c_str(), false); + eloq_metrics_app::MetricsRegistryImpl::MetricsRegistryResult + metrics_registry_result = + eloq_metrics_app::MetricsRegistryImpl::GetRegistry(); + + if (metrics_registry_result.not_ok_ != nullptr) + { + LOG(ERROR) + << "!!!!!!!! Failed to initialize MetricsRegristry !!!!!!!!"; + return false; + } + + metrics_registry_ = + std::move(metrics_registry_result.metrics_registry_); + + if (core_config_.enable_data_store) + { + metrics::CommonLabels kv_common_common_labels{}; + kv_common_common_labels["node_ip"] = network_config_.local_ip; + kv_common_common_labels["node_port"] = + std::to_string(network_config_.local_port); + store_hd_->RegisterKvMetrics(metrics_registry_.get(), + kv_common_common_labels); + } #ifdef ELOQ_MODULE_ELOQSQL - metrics::CommonLabels mysql_common_labels{}; - mysql_common_labels["node_ip"]= network_config_.local_ip; - mysql_common_labels["node_port"]= std::to_string(network_config_.local_port); - metrics::register_mysql_metrics(metrics_registry_.get(), - mysql_common_labels); + metrics::CommonLabels mysql_common_labels{}; + mysql_common_labels["node_ip"] = network_config_.local_ip; + mysql_common_labels["node_port"] = + std::to_string(network_config_.local_port); + metrics::register_mysql_metrics(metrics_registry_.get(), + mysql_common_labels); +#endif + +#ifdef ELOQ_MODULE_ELOQKV + metrics::CommonLabels redis_common_labels{}; + redis_common_labels["node_ip"] = network_config_.local_ip; + redis_common_labels["node_port"] = + std::to_string(network_config_.local_port); + redis_common_labels["node_id"] = + std::to_string(network_config_.node_id); + metrics::register_redis_metrics(metrics_registry_.get(), + redis_common_labels, + core_config_.core_num); + metrics::redis_meter->Collect( + metrics::NAME_MAX_CONNECTION, + DataSubstrate::GetGlobal()->GetCoreConfig().maxclients); + for (const auto &[cmd, _] : EloqKV::command_types) + { + std::vector label_groups = {{"type", {cmd}}}; + + external_metrics.push_back( + std::make_tuple(metrics::NAME_REDIS_COMMAND_DURATION, + metrics::Type::Histogram, + label_groups)); + external_metrics.push_back( + std::make_tuple(metrics::NAME_REDIS_COMMAND_TOTAL, + metrics::Type::Counter, + label_groups)); + } + for (const auto &access_type : {"read", "write"}) + { + external_metrics.push_back( + std::make_tuple(metrics::NAME_REDIS_COMMAND_AGGREGATED_TOTAL, + metrics::Type::Counter, + std::vector{ + {"access_type", {access_type}}})); + external_metrics.push_back( + std::make_tuple(metrics::NAME_REDIS_COMMAND_AGGREGATED_DURATION, + metrics::Type::Histogram, + std::vector{ + {"access_type", {access_type}}})); + } #endif - tx_service_common_labels_["node_ip"]= network_config_.local_ip; - tx_service_common_labels_["node_port"]= - std::to_string(network_config_.local_port); - tx_service_common_labels_["node_id"]= - std::to_string(network_config_.node_id); - } + tx_service_common_labels_["node_ip"] = network_config_.local_ip; + tx_service_common_labels_["node_port"] = + std::to_string(network_config_.local_port); + tx_service_common_labels_["node_id"] = + std::to_string(network_config_.node_id); + } - return true; + return true; } \ No newline at end of file diff --git a/data_substrate/core/src/tx_service_init.cpp b/data_substrate/core/src/tx_service_init.cpp index 1dcae56c381..6ed3f8963f0 100644 --- a/data_substrate/core/src/tx_service_init.cpp +++ b/data_substrate/core/src/tx_service_init.cpp @@ -5,249 +5,286 @@ #include #include "data_substrate.h" -#include "tx_service.h" #include "eloq_log_wrapper.h" #include "sequences/sequences.h" +#include "tx_service.h" DEFINE_int32(checkpointer_interval, 10, "Checkpointer interval in seconds"); DEFINE_int32(node_memory_limit_mb, 8192, "Node memory limit in MB"); DEFINE_int32(checkpointer_delay_seconds, 5, "Checkpointer delay in seconds"); -DEFINE_int32(collect_active_tx_ts_interval_seconds, 2, +DEFINE_int32(collect_active_tx_ts_interval_seconds, + 2, "Active transaction timestamp collection interval"); DEFINE_bool(kickout_data_for_test, false, "Kickout data for test"); DEFINE_bool(enable_key_cache, false, "Enable key cache"); -DEFINE_uint32(max_standby_lag, 400000, +DEFINE_uint32(max_standby_lag, + 400000, "txservice max msg lag between primary and standby"); DEFINE_string(tx_service_data_path, "", "path for tx_service data"); DEFINE_bool(fork_host_manager, true, "fork host manager process"); DEFINE_string(hm_ip, "", "Host manager IP"); DEFINE_int32(hm_port, 0, "Host manager port"); -DEFINE_string(hm_bin_path, "", +DEFINE_string(hm_bin_path, + "", "host manager binary path if forking host manager process from " "main process"); -DEFINE_uint32(deadlock_check_interval_seconds, 10, +DEFINE_uint32(deadlock_check_interval_seconds, + 10, "Deadlock check interval in seconds"); -DEFINE_bool(realtime_sampling, true, +DEFINE_bool(realtime_sampling, + true, "Whether enable realtime sampling. If disable it, user may need " "to execute " "analyze command at some time. Different from Innodb, Eloq never " "analyze table automatically."); DEFINE_uint32(range_split_worker_num, 0, "Range split worker number"); -DEFINE_bool(auto_redirect, false, "If redirect remote object command to remote node internally"); +DEFINE_bool(auto_redirect, + false, + "If redirect remote object command to remote node internally"); + +#ifdef ELOQ_MODULE_ELOQKV +namespace EloqKV +{ +extern std::function + eloqkv_publish_func; +} +#endif bool DataSubstrate::InitializeTxService(const INIReader &config_reader) { - uint64_t checkpointer_interval= - !CheckCommandLineFlagIsDefault("checkpointer_interval") - ? FLAGS_checkpointer_interval - : config_reader.GetInteger("local", "checkpointer_interval", - FLAGS_checkpointer_interval); - - uint64_t checkpointer_delay_seconds= - !CheckCommandLineFlagIsDefault("checkpointer_delay_seconds") - ? FLAGS_checkpointer_delay_seconds - : config_reader.GetInteger("local", "checkpointer_delay_seconds", - FLAGS_checkpointer_delay_seconds); - - uint64_t collect_active_tx_ts_interval_seconds= - !CheckCommandLineFlagIsDefault("collect_active_tx_ts_interval_seconds") - ? FLAGS_collect_active_tx_ts_interval_seconds - : config_reader.GetInteger( - "local", "collect_active_tx_ts_interval_seconds", - FLAGS_collect_active_tx_ts_interval_seconds); - - uint64_t max_standby_lag= - !CheckCommandLineFlagIsDefault("max_standby_lag") - ? FLAGS_max_standby_lag - : config_reader.GetInteger("local", "max_standby_lag", - FLAGS_max_standby_lag); - - bool kickout_data_for_test= - !CheckCommandLineFlagIsDefault("kickout_data_for_test") - ? FLAGS_kickout_data_for_test - : config_reader.GetBoolean("local", "kickout_data_for_test", - FLAGS_kickout_data_for_test); - - bool enable_key_cache= - !CheckCommandLineFlagIsDefault("enable_key_cache") - ? FLAGS_enable_key_cache - : config_reader.GetBoolean("local", "enable_key_cache", - FLAGS_enable_key_cache); - - std::string tx_service_data_path= - !CheckCommandLineFlagIsDefault("tx_service_data_path") - ? FLAGS_tx_service_data_path - : config_reader.GetString("local", "tx_service_data_path", - FLAGS_tx_service_data_path); - std::string tx_path("local://"); - if (tx_service_data_path.empty()) - { - tx_path.append(core_config_.data_path); - } - else - { - tx_path.append(tx_service_data_path); - } - - const char *field_mem= "node_memory_limit_mb"; - uint64_t node_memory_limit_mb= FLAGS_node_memory_limit_mb; - if (CheckCommandLineFlagIsDefault(field_mem)) - { - if (config_reader.HasValue("local", field_mem)) + uint64_t checkpointer_interval = + !CheckCommandLineFlagIsDefault("checkpointer_interval") + ? FLAGS_checkpointer_interval + : config_reader.GetInteger("local", + "checkpointer_interval", + FLAGS_checkpointer_interval); + + uint64_t checkpointer_delay_seconds = + !CheckCommandLineFlagIsDefault("checkpointer_delay_seconds") + ? FLAGS_checkpointer_delay_seconds + : config_reader.GetInteger("local", + "checkpointer_delay_seconds", + FLAGS_checkpointer_delay_seconds); + + uint64_t collect_active_tx_ts_interval_seconds = + !CheckCommandLineFlagIsDefault("collect_active_tx_ts_interval_seconds") + ? FLAGS_collect_active_tx_ts_interval_seconds + : config_reader.GetInteger( + "local", + "collect_active_tx_ts_interval_seconds", + FLAGS_collect_active_tx_ts_interval_seconds); + + uint64_t max_standby_lag = + !CheckCommandLineFlagIsDefault("max_standby_lag") + ? FLAGS_max_standby_lag + : config_reader.GetInteger( + "local", "max_standby_lag", FLAGS_max_standby_lag); + + bool kickout_data_for_test = + !CheckCommandLineFlagIsDefault("kickout_data_for_test") + ? FLAGS_kickout_data_for_test + : config_reader.GetBoolean("local", + "kickout_data_for_test", + FLAGS_kickout_data_for_test); + + bool enable_key_cache = + !CheckCommandLineFlagIsDefault("enable_key_cache") + ? FLAGS_enable_key_cache + : config_reader.GetBoolean( + "local", "enable_key_cache", FLAGS_enable_key_cache); + + std::string tx_service_data_path = + !CheckCommandLineFlagIsDefault("tx_service_data_path") + ? FLAGS_tx_service_data_path + : config_reader.GetString( + "local", "tx_service_data_path", FLAGS_tx_service_data_path); + std::string tx_path("local://"); + if (tx_service_data_path.empty()) { - node_memory_limit_mb= config_reader.GetInteger("local", field_mem, 0); - assert(node_memory_limit_mb); + tx_path.append(core_config_.data_path); } else { - struct sysinfo meminfo; - if (sysinfo(&meminfo)) - { - LOG(ERROR) << "config is missing: " << field_mem; - } - uint32_t mem_mib= - ((uint64_t) meminfo.totalram * meminfo.mem_unit) / (1024 * 1024); - node_memory_limit_mb= std::max(uint32_t(512), (mem_mib * 4) / 5); - LOG(INFO) << "config is automatically set: " << field_mem << "=" - << node_memory_limit_mb << "(MiB), total memory=" << mem_mib; + tx_path.append(tx_service_data_path); } - } - - uint64_t deadlock_check_interval_seconds= - !CheckCommandLineFlagIsDefault("deadlock_check_interval_seconds") - ? FLAGS_deadlock_check_interval_seconds - : config_reader.GetInteger("local", - "deadlock_check_interval_seconds", - FLAGS_deadlock_check_interval_seconds); - txservice::DeadLockCheck::SetTimeInterval(deadlock_check_interval_seconds); - - bool realtime_sampling= - !CheckCommandLineFlagIsDefault("realtime_sampling") - ? FLAGS_realtime_sampling - : config_reader.GetBoolean("local", "realtime_sampling", - FLAGS_realtime_sampling); - - uint64_t range_split_worker_num= - !CheckCommandLineFlagIsDefault("range_split_worker_num") - ? FLAGS_range_split_worker_num - : config_reader.GetInteger("local", "range_split_worker_num", - FLAGS_range_split_worker_num); - - bool auto_redirect = - !CheckCommandLineFlagIsDefault("auto_redirect") - ? FLAGS_auto_redirect - : config_reader.GetBoolean("local", "auto_redirect", - FLAGS_auto_redirect); - - bool fork_host_manager= - !CheckCommandLineFlagIsDefault("fork_host_manager") - ? FLAGS_fork_host_manager - : config_reader.GetBoolean("local", "fork_host_manager", - FLAGS_fork_host_manager); - - std::string hm_ip= !CheckCommandLineFlagIsDefault("hm_ip") - ? FLAGS_hm_ip - : config_reader.Get("local", "hm_ip", FLAGS_hm_ip); - - uint16_t hm_port= - !CheckCommandLineFlagIsDefault("hm_port") - ? FLAGS_hm_port - : config_reader.GetInteger("local", "hm_port", FLAGS_hm_port); - - std::string hm_bin_path= - !CheckCommandLineFlagIsDefault("hm_bin_path") - ? FLAGS_hm_bin_path - : config_reader.Get("local", "hm_bin_path", FLAGS_hm_bin_path); + + const char *field_mem = "node_memory_limit_mb"; + uint64_t node_memory_limit_mb = FLAGS_node_memory_limit_mb; + if (CheckCommandLineFlagIsDefault(field_mem)) + { + if (config_reader.HasValue("local", field_mem)) + { + node_memory_limit_mb = + config_reader.GetInteger("local", field_mem, 0); + assert(node_memory_limit_mb); + } + else + { + struct sysinfo meminfo; + if (sysinfo(&meminfo)) + { + LOG(ERROR) << "config is missing: " << field_mem; + } + uint32_t mem_mib = + ((uint64_t) meminfo.totalram * meminfo.mem_unit) / + (1024 * 1024); + node_memory_limit_mb = std::max(uint32_t(512), (mem_mib * 4) / 5); + LOG(INFO) << "config is automatically set: " << field_mem << "=" + << node_memory_limit_mb + << "(MiB), total memory=" << mem_mib; + } + } + FLAGS_node_memory_limit_mb = node_memory_limit_mb; + + uint64_t deadlock_check_interval_seconds = + !CheckCommandLineFlagIsDefault("deadlock_check_interval_seconds") + ? FLAGS_deadlock_check_interval_seconds + : config_reader.GetInteger("local", + "deadlock_check_interval_seconds", + FLAGS_deadlock_check_interval_seconds); + txservice::DeadLockCheck::SetTimeInterval(deadlock_check_interval_seconds); + + bool realtime_sampling = + !CheckCommandLineFlagIsDefault("realtime_sampling") + ? FLAGS_realtime_sampling + : config_reader.GetBoolean( + "local", "realtime_sampling", FLAGS_realtime_sampling); + + uint64_t range_split_worker_num = + !CheckCommandLineFlagIsDefault("range_split_worker_num") + ? FLAGS_range_split_worker_num + : config_reader.GetInteger("local", + "range_split_worker_num", + FLAGS_range_split_worker_num); + + bool auto_redirect = + !CheckCommandLineFlagIsDefault("auto_redirect") + ? FLAGS_auto_redirect + : config_reader.GetBoolean( + "local", "auto_redirect", FLAGS_auto_redirect); + + bool fork_host_manager = + !CheckCommandLineFlagIsDefault("fork_host_manager") + ? FLAGS_fork_host_manager + : config_reader.GetBoolean( + "local", "fork_host_manager", FLAGS_fork_host_manager); + + std::string hm_ip = !CheckCommandLineFlagIsDefault("hm_ip") + ? FLAGS_hm_ip + : config_reader.Get("local", "hm_ip", FLAGS_hm_ip); + + uint16_t hm_port = + !CheckCommandLineFlagIsDefault("hm_port") + ? FLAGS_hm_port + : config_reader.GetInteger("local", "hm_port", FLAGS_hm_port); + + std::string hm_bin_path = + !CheckCommandLineFlagIsDefault("hm_bin_path") + ? FLAGS_hm_bin_path + : config_reader.Get("local", "hm_bin_path", FLAGS_hm_bin_path); #ifdef FORK_HM_PROCESS - if (hm_ip.empty()) - { - hm_ip= network_config_.local_ip; - } - if (hm_port == 0) - { - hm_port= network_config_.local_port + 4; - } - if (hm_bin_path.empty()) - { - char path_buf[PATH_MAX]; - ssize_t len= ::readlink("/proc/self/exe", path_buf, sizeof(path_buf)); - path_buf[len]= '\0'; - std::string s_path(path_buf); - std::string::size_type pos= s_path.find_last_of("/"); - std::string parent_path= s_path.substr(0, pos); - hm_bin_path= parent_path + "/host_manager"; - } + if (hm_ip.empty()) + { + hm_ip = network_config_.local_ip; + } + if (hm_port == 0) + { + hm_port = network_config_.local_port + 4; + } + if (hm_bin_path.empty()) + { + char path_buf[PATH_MAX]; + ssize_t len = ::readlink("/proc/self/exe", path_buf, sizeof(path_buf)); + path_buf[len] = '\0'; + std::string s_path(path_buf); + std::string::size_type pos = s_path.find_last_of("/"); + std::string parent_path = s_path.substr(0, pos); + hm_bin_path = parent_path + "/host_manager"; + } +#endif + + std::map tx_service_conf{ + {"core_num", core_config_.core_num}, + {"checkpointer_interval", checkpointer_interval}, + {"node_memory_limit_mb", node_memory_limit_mb}, + {"checkpointer_delay_seconds", checkpointer_delay_seconds}, + {"collect_active_tx_ts_interval_seconds", + collect_active_tx_ts_interval_seconds}, + {"realtime_sampling", realtime_sampling ? 1 : 0}, + {"rep_group_cnt", network_config_.node_group_replica_num}, + {"range_split_worker_num", range_split_worker_num}, + {"enable_shard_heap_defragment", + core_config_.enable_heap_defragment ? 1 : 0}, + {"enable_key_cache", enable_key_cache}, + {"max_standby_lag", max_standby_lag}, + {"kickout_data_for_test", kickout_data_for_test ? 1 : 0}}; + + txservice::CatalogFactory *catalog_factory[NUM_EXTERNAL_ENGINES] = { + nullptr, nullptr, nullptr}; + + for (size_t i = 0; i < NUM_EXTERNAL_ENGINES; i++) + { + catalog_factory[i] = engines_[i].catalog_factory; + } + + auto log_agent = std::make_unique( + network_config_.node_group_replica_num); + + tx_service_ = std::make_unique( + catalog_factory, + system_handler_, + tx_service_conf, + network_config_.node_id, + network_config_.native_ng_id, + &network_config_.ng_configs, + network_config_.cluster_config_version, + core_config_.enable_data_store ? store_hd_.get() : nullptr, + log_agent.get(), + core_config_.enable_mvcc, // enable_mvcc + !core_config_.enable_wal, // skip_wal + !core_config_.enable_data_store, // skip_kv + core_config_.enable_cache_replacement, // enable_cache_replacement + auto_redirect, // auto_redirect + metrics_registry_.get(), // metrics_registry + tx_service_common_labels_, // common_labels + &prebuilt_tables_ +#ifdef ELOQ_MODULE_ELOQKV + , + EloqKV::eloqkv_publish_func, + external_metrics #endif + ); + + if (core_config_.enable_data_store) + { + store_hd_->SetTxService(tx_service_.get()); + } + + if (tx_service_->Start(network_config_.node_id, + network_config_.native_ng_id, + &network_config_.ng_configs, + network_config_.cluster_config_version, + &log_service_config_.txlog_ips, + &log_service_config_.txlog_ports, + &hm_ip, + &hm_port, + &hm_bin_path, + tx_service_conf, + std::move(log_agent), + tx_path, + network_config_.cluster_config_file_path, + fork_host_manager) != 0) + { + LOG(ERROR) << "Failed to start tx service!!!!!"; + return false; + } + + txservice::Sequences::InitSequence(tx_service_.get(), store_hd_.get()); + + // tx_service is a distributed service, should wait for all the tx_service + // nodes to finish the log recovery process and setup the cc_stream_sender. + tx_service_->WaitClusterReady(); + // wait for the tx_service node to become the native group leader. + tx_service_->WaitNodeBecomeNativeGroupLeader(); - std::map tx_service_conf{ - {"core_num", core_config_.core_num}, - {"checkpointer_interval", checkpointer_interval}, - {"node_memory_limit_mb", node_memory_limit_mb}, - {"checkpointer_delay_seconds", checkpointer_delay_seconds}, - {"collect_active_tx_ts_interval_seconds", - collect_active_tx_ts_interval_seconds}, - {"realtime_sampling", realtime_sampling ? 1 : 0}, - {"rep_group_cnt", network_config_.node_group_replica_num}, - {"range_split_worker_num", range_split_worker_num}, - {"enable_shard_heap_defragment", - core_config_.enable_heap_defragment ? 1 : 0}, - {"enable_key_cache", enable_key_cache}, - {"max_standby_lag", max_standby_lag}, - {"kickout_data_for_test", kickout_data_for_test ? 1 : 0}}; - - txservice::CatalogFactory *catalog_factory[NUM_EXTERNAL_ENGINES]= { - nullptr, nullptr, nullptr}; - - for (size_t i= 0; i < NUM_EXTERNAL_ENGINES; i++) - { - catalog_factory[i]= engines_[i].catalog_factory; - } - - auto log_agent= std::make_unique( - network_config_.node_group_replica_num); - - tx_service_= std::make_unique( - catalog_factory, system_handler_, tx_service_conf, - network_config_.node_id, network_config_.native_ng_id, - &network_config_.ng_configs, network_config_.cluster_config_version, - core_config_.enable_data_store ? store_hd_.get() : nullptr, - log_agent.get(), - core_config_.enable_mvcc, // enable_mvcc - !core_config_.enable_wal, // skip_wal - !core_config_.enable_data_store, // skip_kv - core_config_.enable_cache_replacement, // enable_cache_replacement - auto_redirect, // auto_redirect - metrics_registry_.get(), // metrics_registry - tx_service_common_labels_, // common_labels - &prebuilt_tables_ - // , - // publish_func, // TODO(liunyl) - // external_metrics - ); - - if (core_config_.enable_data_store) - { - store_hd_->SetTxService(tx_service_.get()); - } - - if (tx_service_->Start( - network_config_.node_id, network_config_.native_ng_id, - &network_config_.ng_configs, network_config_.cluster_config_version, - &log_service_config_.txlog_ips, &log_service_config_.txlog_ports, - &hm_ip, &hm_port, &hm_bin_path, tx_service_conf, - std::move(log_agent), tx_path, - network_config_.cluster_config_file_path, fork_host_manager) != 0) - { - LOG(ERROR) << "Failed to start tx service!!!!!"; - return false; - } - - txservice::Sequences::InitSequence(tx_service_.get(), store_hd_.get()); - - // tx_service is a distributed service, should wait for all the tx_service - // nodes to finish the log recovery process and setup the cc_stream_sender. - tx_service_->WaitClusterReady(); - // wait for the tx_service node to become the native group leader. - tx_service_->WaitNodeBecomeNativeGroupLeader(); - - return true; + return true; } \ No newline at end of file diff --git a/data_substrate/store_handler b/data_substrate/store_handler index 60176c420db..86f996199cd 160000 --- a/data_substrate/store_handler +++ b/data_substrate/store_handler @@ -1 +1 @@ -Subproject commit 60176c420db7178f878939ff6d1103203da17cc2 +Subproject commit 86f996199cdbf7812e9f394ea25f9b1f7a1b2809 diff --git a/data_substrate/tx_service b/data_substrate/tx_service index b69746c2eec..6c4756431c1 160000 --- a/data_substrate/tx_service +++ b/data_substrate/tx_service @@ -1 +1 @@ -Subproject commit b69746c2eec7c8ac9b4c48c3626fc589068aa617 +Subproject commit 6c4756431c15e774337109a2ca385a7c83f9b92e diff --git a/sql/main.cc b/sql/main.cc index 962f805a6de..ffd28c94014 100644 --- a/sql/main.cc +++ b/sql/main.cc @@ -20,14 +20,8 @@ On Windows, might do some service handling. */ #include -#include -// Only MySQL-related CLI flag we accept -DEFINE_string(eloqsql_config, "", - "Path to MySQL configuration file. All MySQL options must be in this config file."); -DEFINE_string(data_substrate_config, "", - "Path to data substrate configuration file."); #ifdef WITH_GLOG #include "glog_error_logging.h" @@ -40,7 +34,6 @@ extern int mysqld_win_main(int argc, char **argv); extern int mysqld_main(int argc, char **argv); #endif - int main(int argc, char **argv) { // Initialize gflags - will error on unknown flags @@ -50,12 +43,8 @@ int main(int argc, char **argv) "Data substrate flags can be passed on command line.\n" "Use --help to see all available flags."); - gflags::ParseCommandLineFlags(&argc, &argv, true); - - // DataSubstrate::InitializeGlobal(FLAGS_data_substrate_config); - - + #ifdef WITH_GLOG InitGoogleLogging(argv); #endif /* WITH_GLOG */ diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 997ccc01eb0..5e2b2f8f8b7 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -107,18 +107,6 @@ #include "semisync_master.h" #include "semisync_slave.h" -#ifdef MYSQLD_LIBRARY_MODE -#include -#include -// Define synchronization variables -namespace mysqld_converged_sync { - std::mutex init_mutex; - std::condition_variable mysqld_basic_init_done_cv; - std::condition_variable data_substrate_init_done_cv; - bool mysqld_basic_init_done = false; - bool data_substrate_init_done = false; -} -#endif #include "transaction.h" @@ -127,7 +115,6 @@ namespace mysqld_converged_sync { #endif #include "glog/logging.h" -#include "data_substrate.h" #include #include @@ -656,12 +643,10 @@ struct system_variables global_system_variables; const char *current_dbug_option=""; // Declare gflags -DECLARE_string(eloqsql_config); +DEFINE_string(eloqsql_config, "", + "Path to MySQL configuration file. All MySQL options must be in this config file."); DECLARE_bool(bootstrap); -DECLARE_string(data_substrate_config); -// Global string to hold defaults file argument -static std::string g_defaults_file_arg; struct system_variables max_system_variables; struct system_status_var global_status_var; @@ -5059,32 +5044,6 @@ static int init_server_components() /* It's now safe to use thread specific memory */ mysqld_server_initialized= 1; -#ifdef MYSQLD_LIBRARY_MODE - // In library mode (converged binary), use synchronization: - // 1. Signal that MySQL basic initialization is done - // 2. Wait for data substrate to be initialized by converged main - { - std::unique_lock lock(mysqld_converged_sync::init_mutex); - mysqld_converged_sync::mysqld_basic_init_done = true; - mysqld_converged_sync::mysqld_basic_init_done_cv.notify_one(); - - LOG(INFO) << "MySQL basic initialization complete, waiting for data substrate..."; - - // Wait for data substrate initialization to complete - mysqld_converged_sync::data_substrate_init_done_cv.wait(lock, [] { - return mysqld_converged_sync::data_substrate_init_done; - }); - - LOG(INFO) << "Data substrate initialized, MySQL continuing..."; - } -#else - // In standalone mode, initialize data substrate here - // Wait for mysqld to initialize before initializing data substrate - // as data substrate may need to access some mysqld variables during replay. - LOG(INFO) << "Standalone mode: Initializing data substrate..."; - DataSubstrate::InitializeGlobal(FLAGS_data_substrate_config); -#endif - #ifndef EMBEDDED_LIBRARY wsrep_thr_init(); #endif @@ -5626,10 +5585,11 @@ int mysqld_main(int argc, char **argv) } // Append --defaults-file if --eloqsql_config specified + std::string defaults_file_arg; if (!FLAGS_eloqsql_config.empty()) { - g_defaults_file_arg = std::string("--defaults-file=") + FLAGS_eloqsql_config; - load_default_argv[load_default_argc] = const_cast(g_defaults_file_arg.c_str()); + defaults_file_arg = std::string("--defaults-file=") + FLAGS_eloqsql_config; + load_default_argv[load_default_argc] = const_cast(defaults_file_arg.c_str()); load_default_argc++; } diff --git a/storage/eloq/CMakeLists.txt b/storage/eloq/CMakeLists.txt index d67a7fb58bb..a0a8f4fe343 100644 --- a/storage/eloq/CMakeLists.txt +++ b/storage/eloq/CMakeLists.txt @@ -15,144 +15,12 @@ cmake_minimum_required(VERSION 3.8) project(eloq) -# set(WITH_DATA_STORE "ELOQDSS_ROCKSDB" CACHE STRING "Which key-value storage to use") -# set_property(CACHE WITH_DATA_STORE PROPERTY STRINGS "DYNAMODB" "BIGTABLE" "ELOQDSS_ROCKSDB_CLOUD_S3" "ELOQDSS_ELOQSTORE" "ELOQDSS_ROCKSDB") -# message(NOTICE "WITH_DATA_STORE" : ${WITH_DATA_STORE}) +set(WITH_DATA_STORE "ELOQDSS_ROCKSDB" CACHE STRING "Which key-value storage to use") +set_property(CACHE WITH_DATA_STORE PROPERTY STRINGS "DYNAMODB" "BIGTABLE" "ELOQDSS_ROCKSDB_CLOUD_S3" "ELOQDSS_ELOQSTORE" "ELOQDSS_ROCKSDB") +message(NOTICE "WITH_DATA_STORE" : ${WITH_DATA_STORE}) set(METRICS_LIB "eloq-metrics" CACHE STRING "metrics library name.") - -# add_compile_definitions(ELOQ_MODULE_ELOQSQL) -# if (WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB") -# find_path(ROCKSDB_INCLUDE_PATH NAMES rocksdb/db.h) -# if (NOT ROCKSDB_INCLUDE_PATH) -# message(FATAL_ERROR "Fail to find RocksDB include path") -# endif() -# message(STATUS "ROCKSDB_INCLUDE_PATH: ${ROCKSDB_INCLUDE_PATH}") -# include_directories(${ROCKSDB_INCLUDE_PATH}) - -# find_library(ROCKSDB_LIB NAMES rocksdb) -# if (NOT ROCKSDB_LIB) -# message(FATAL_ERROR "Fail to find RocksDB lib path") -# endif() -# message(STATUS "ROCKSDB_LIB: ${ROCKSDB_LIB}") -# set(ROCKSDB_LIBRARIES ${ROCKSDB_LIBRARIES} ${ROCKSDB_LIB}) -# endif() - -# if ((WITH_DATA_STORE STREQUAL "ROCKSDB_CLOUD_S3") OR (WITH_DATA_STORE STREQUAL "ROCKSDB_CLOUD_GCS") -# OR (WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB_CLOUD_S3") OR (WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB_CLOUD_GCS")) -# if ((WITH_DATA_STORE STREQUAL "ROCKSDB_CLOUD_S3") OR (WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB_CLOUD_S3")) -# find_path(AWS_CORE_INCLUDE_PATH aws/core/Aws.h) -# if((NOT AWS_CORE_INCLUDE_PATH)) -# message(FATAL_ERROR "Fail to find aws/core include path") -# endif() -# message(STATUS "aws/core include path: ${AWS_CORE_INCLUDE_PATH}") - -# find_library(AWS_CORE_LIB aws-cpp-sdk-core) -# if((NOT AWS_CORE_LIB )) -# message(FATAL_ERROR "Fail to find aws-cpp-sdk-core lib") -# endif() -# message(STATUS "aws-cpp-sdk-core library: ${AWS_CORE_LIB}") - -# find_path(AWS_KINESIS_INCLUDE_PATH aws/kinesis/KinesisClient.h) -# if((NOT AWS_KINESIS_INCLUDE_PATH)) -# message(FATAL_ERROR "Fail to find aws/kinesis include path") -# endif() -# message(STATUS "aws/kinesis include path: ${AWS_KINESIS_INCLUDE_PATH}") - -# find_library(AWS_KINESIS_LIB aws-cpp-sdk-kinesis) -# if((NOT AWS_KINESIS_LIB)) -# message(FATAL_ERROR "Fail to find aws-cpp-sdk-kinesis lib") -# endif() -# message(STATUS "aws-cpp-sdk-kinesis library: ${AWS_KINESIS_LIB}") - -# find_path(AWS_KINESIS_INCLUDE_PATH aws/kinesis/KinesisClient.h) -# if((NOT AWS_KINESIS_INCLUDE_PATH)) -# message(FATAL_ERROR "Fail to find aws/kinesis include path") -# endif() -# message(STATUS "aws/kinesis include path: ${AWS_KINESIS_INCLUDE_PATH}") - -# find_library(AWS_KINESIS_LIB aws-cpp-sdk-kinesis) -# if((NOT AWS_KINESIS_LIB)) -# message(FATAL_ERROR "Fail to find aws-cpp-sdk-kinesis lib") -# endif() -# message(STATUS "aws-cpp-sdk-kinesis library: ${AWS_KINESIS_LIB}") - -# find_path(AWS_S3_INCLUDE_PATH aws/s3/S3Client.h) -# if((NOT AWS_S3_INCLUDE_PATH)) -# message(FATAL_ERROR "Fail to find aws/s3 include path") -# endif() -# message(STATUS "aws/s3 include path: ${AWS_S3_INCLUDE_PATH}") - -# find_library(AWS_S3_LIB aws-cpp-sdk-s3) -# if((NOT AWS_S3_LIB )) -# message(FATAL_ERROR "Fail to find aws-cpp-sdk-s3 lib") -# endif() -# message(STATUS "aws-cpp-sdk-s3 library: ${AWS_S3_LIB}") - -# set(ROCKSDB_INCLUDE_PATH ${ROCKSDB_INCLUDE_PATH} ${AWS_CORE_INCLUDE_PATH}) -# set(ROCKSDB_INCLUDE_PATH ${ROCKSDB_INCLUDE_PATH} ${AWS_KINESIS_INCLUDE_PATH}) -# set(ROCKSDB_INCLUDE_PATH ${ROCKSDB_INCLUDE_PATH} ${AWS_S3_INCLUDE_PATH}) - -# set(ROCKSDB_LIBRARIES ${ROCKSDB_LIBRARIES} ${AWS_CORE_LIB}) -# set(ROCKSDB_LIBRARIES ${ROCKSDB_LIBRARIES} ${AWS_KINESIS_LIB}) -# set(ROCKSDB_LIBRARIES ${ROCKSDB_LIBRARIES} ${AWS_S3_LIB}) - -# find_library(ROCKSDB_CLOUD_LIB NAMES rocksdb-cloud-aws) -# elseif ((WITH_DATA_STORE STREQUAL "ROCKSDB_CLOUD_GCS") OR (WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB_CLOUD_GCS")) -# find_path(GCP_CS_INCLUDE_PATH google/cloud/storage/client.h) -# if((NOT GCP_CS_INCLUDE_PATH)) -# message(FATAL_ERROR "Fail to find google/cloud/storage include path") -# endif() -# message(STATUS "google/cloud/storage include path: ${GCP_CS_INCLUDE_PATH}") - -# find_library(GCP_COMMON_LIB google_cloud_cpp_common) -# if((NOT GCP_COMMON_LIB)) -# message(FATAL_ERROR "Fail to find google_cloud_cpp_common lib") -# endif() -# message(STATUS "google_cloud_cpp_common library: ${GCP_COMMON_LIB}") - -# find_library(GCP_CS_LIB google_cloud_cpp_storage) -# if((NOT GCP_CS_LIB)) -# message(FATAL_ERROR "Fail to find google_cloud_cpp_storage lib") -# endif() -# message(STATUS "google_cloud_cpp_storage library: ${GCP_CS_LIB}") - -# set(ROCKSDB_LIBRARIES ${ROCKSDB_LIBRARIES} ${GCP_COMMON_LIB}) -# set(ROCKSDB_LIBRARIES ${ROCKSDB_LIBRARIES} ${GCP_CS_LIB}) - -# find_library(ROCKSDB_CLOUD_LIB NAMES rocksdb-cloud-gcp) -# endif () - -# # Common RocksDB Cloud setup (outside the S3/GCS specific blocks) -# find_path(ROCKSDB_CLOUD_INCLUDE_PATH NAMES rocksdb/db.h PATH_SUFFIXES "rocksdb_cloud_header") -# if (NOT ROCKSDB_CLOUD_INCLUDE_PATH) -# message(FATAL_ERROR "Fail to find RocksDB Cloud include path") -# endif () -# message(STATUS "ROCKSDB_CLOUD_INCLUDE_PATH: ${ROCKSDB_CLOUD_INCLUDE_PATH}") -# set(ROCKSDB_INCLUDE_PATH ${ROCKSDB_INCLUDE_PATH} ${ROCKSDB_CLOUD_INCLUDE_PATH}) -# include_directories(${ROCKSDB_INCLUDE_PATH}) - -# if (NOT ROCKSDB_CLOUD_LIB) -# message(FATAL_ERROR "Fail to find RocksDB Cloud lib path") -# endif () -# message(STATUS "ROCKSDB_CLOUD_LIB: ${ROCKSDB_CLOUD_LIB}") -# set(ROCKSDB_LIBRARIES ${ROCKSDB_LIBRARIES} ${ROCKSDB_CLOUD_LIB}) -# endif() - -# if ((WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB_CLOUD_S3") OR (WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB_CLOUD_GCS") OR (WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB") OR (WITH_DATA_STORE STREQUAL "ELOQDSS_ELOQSTORE")) -# # compile proto file -# set(DS_PROTO_DIR ${CMAKE_CURRENT_SOURCE_DIR}/store_handler/eloq_data_store_service) -# message(NOTICE "data store service proto dir: ${DS_PROTO_DIR}") -# set(PROTO_SRC ${DS_PROTO_DIR}) -# set(PROTO_NAME ds_request) -# execute_process( -# COMMAND protoc ./${PROTO_NAME}.proto --cpp_out=./ -# WORKING_DIRECTORY ${PROTO_SRC} -# ) -# endif() - - option(WITH_LOG_SERVICE "Compile with built-in log service." ON) if(WITH_LOG_SERVICE) @@ -171,8 +39,6 @@ elseif(WITH_LOG_STATE STREQUAL "ROCKSDB_CLOUD_GCS") else() message(FATAL_ERROR "Unknown WITH_LOG_STATE: ${WITH_LOG_STATE}") endif() - - endif() option(OPEN_LOG_SERVICE "Compile with open log service." ON) @@ -180,33 +46,8 @@ message("OPEN_LOG_SERVICE: " ${OPEN_LOG_SERVICE}) if (OPEN_LOG_SERVICE) add_compile_definitions(OPEN_LOG_SERVICE=1) endif() - -# # Add log state type definition to match data_substrate configuration -# if(WITH_LOG_STATE STREQUAL "MEMORY") -# add_compile_definitions(LOG_STATE_TYPE_MEM) -# elseif(WITH_LOG_STATE STREQUAL "ROCKSDB") -# add_compile_definitions(LOG_STATE_TYPE_RKDB) -# elseif(WITH_LOG_STATE STREQUAL "ROCKSDB_CLOUD_S3") -# add_compile_definitions(LOG_STATE_TYPE_RKDB_S3) -# elseif(WITH_LOG_STATE STREQUAL "ROCKSDB_CLOUD_GCS") -# add_compile_definitions(LOG_STATE_TYPE_RKDB_GCS) -# endif() - set(STATISTICS ON CACHE BOOL "Turn on STATISTICS") -# Data substrate components now build separately in data_substrate/ directory -# INCLUDE(build_tx_service.cmake) -# -# if(WITH_LOG_SERVICE) -# if (OPEN_LOG_SERVICE) -# INCLUDE(build_log_service.cmake) -# else() -# INCLUDE(build_eloq_log_service.cmake) -# endif() -# endif() -# -# INCLUDE(build_eloq_metrics.cmake) - if(ELOQ_MODULE_ENABLED) add_compile_definitions(ELOQ_MODULE_ENABLED) endif() @@ -226,28 +67,6 @@ endif() find_package(MIMALLOC REQUIRED) include_directories(${MIMALLOC_INCLUDE_DIR}) -# include_directories( -# ${CMAKE_CURRENT_SOURCE_DIR} -# ${ELOQSQL_SOURCE_DIR}/data_substrate/ -# ${ELOQSQL_SOURCE_DIR}/data_substrate/core/include -# ${ELOQSQL_SOURCE_DIR}/data_substrate/tx_service/include -# ${ELOQSQL_SOURCE_DIR}/data_substrate/tx_service/include/cc -# ${ELOQSQL_SOURCE_DIR}/data_substrate/tx_service/include/remote -# ${ELOQSQL_SOURCE_DIR}/data_substrate/tx_service/include/fault -# ${ELOQSQL_SOURCE_DIR}/data_substrate/tx_service/include/store -# ${ELOQSQL_SOURCE_DIR}/data_substrate/tx_service/tx-log-protos -# ${ELOQSQL_SOURCE_DIR}/data_substrate/eloq_metrics/include -# ${ELOQSQL_SOURCE_DIR}/data_substrate/store_handler -# ) - -# if(WITH_LOG_SERVICE) -# if (OPEN_LOG_SERVICE) -# include_directories(${ELOQSQL_SOURCE_DIR}/data_substrate/log_service/include) -# else() -# include_directories(${ELOQSQL_SOURCE_DIR}/data_substrate/eloq_log_service/include) -# endif() -# endif() - option(SMALL_RANGE "Whether enable small range" OFF) message(NOTICE "SMALL_RANGE : ${SMALL_RANGE}") @@ -275,51 +94,42 @@ SET(MYELOQ_SOURCES slice.cc # eloq_tests.hpp - # store_handler/kv_store.h # Now in data_substrate library ) -# if(WITH_DATA_STORE STREQUAL "DYNAMODB") -# # SET(MYELOQ_SOURCES ${MYELOQ_SOURCES} -# # store_handler/dynamo_handler.h -# # store_handler/dynamo_handler.cpp -# # store_handler/dynamo_scanner.h -# # store_handler/dynamo_scanner.cpp) -# SET(MYELOQ_LIBRARY txservice ${AWSSDK_LINK_LIBRARIES}) -# elseif(WITH_DATA_STORE STREQUAL "BIGTABLE") -# # SET(MYELOQ_SOURCES -# # ${MYELOQ_SOURCES} -# # store_handler/bigtable_handler.h -# # store_handler/bigtable_handler.cpp -# # store_handler/bigtable_scanner.h -# # store_handler/bigtable_scanner.cpp) -# SET(MYELOQ_LIBRARY txservice google-cloud-cpp::bigtable) -# elseif(WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB_CLOUD_S3" OR WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB") -# # Store handler sources are now in data_substrate library -# SET(MYELOQ_LIBRARY ${MYELOQ_LIBRARY} ${ROCKSDB_LIBRARIES}) -# elseif(WITH_DATA_STORE STREQUAL "ELOQDSS_ELOQSTORE") -# # Store handler sources are now in data_substrate library -# SET(MYELOQ_LIBRARY ${MYELOQ_LIBRARY} eloqstore) -# else() -# message(FATAL_ERROR "Unset WITH_DATA_STORE") -# endif() - -# Log service is now in data_substrate library -# if(WITH_LOG_SERVICE) -# SET(MYELOQ_LIBRARY ${MYELOQ_LIBRARY} logservice) -# endif() - -# add_compile_definitions(WITH_DATA_STORE=${KV_STORAGE_VAL}) +# Add compile definitions based on options +if(WITH_DATA_STORE STREQUAL "DYNAMODB") + set(KV_STORAGE_VAL 1 CACHE STRING "dynamodb" FORCE) + add_compile_definitions(DATA_STORE_TYPE_DYNAMODB) + find_package(AWSSDK REQUIRED COMPONENTS dynamodb) +elseif(WITH_DATA_STORE STREQUAL "BIGTABLE") + set(KV_STORAGE_VAL 2 CACHE STRING "big table" FORCE) + add_compile_definitions(DATA_STORE_TYPE_BIGTABLE) + find_package(google_cloud_cpp_bigtable REQUIRED) +elseif(WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB_CLOUD_S3") + set(KV_STORAGE_VAL 3 CACHE STRING "eloq_ds" FORCE) + add_compile_definitions(DATA_STORE_TYPE_ELOQDSS_ROCKSDB_CLOUD_S3) +elseif(WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB") + set(KV_STORAGE_VAL 3 CACHE STRING "eloq_ds" FORCE) + add_compile_definitions(DATA_STORE_TYPE_ELOQDSS_ROCKSDB) +elseif(WITH_DATA_STORE STREQUAL "ELOQDSS_ELOQSTORE") + set(KV_STORAGE_VAL 3 CACHE STRING "eloq_ds" FORCE) + add_compile_definitions(DATA_STORE_TYPE_ELOQDSS_ELOQSTORE) + option(WITH_TXSERVICE "Whether compile eloqstore with txservice" ON) + set(ELOQSTORE_PARENT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/store_handler/eloq_data_store_service CACHE PATH "EloqStore parent directory") + INCLUDE(${CMAKE_CURRENT_SOURCE_DIR}/store_handler/eloq_data_store_service/build_eloq_store.cmake) +else() + message(FATAL_ERROR "Unset WITH_DATA_STORE") +endif() +add_compile_definitions(WITH_DATA_STORE=${KV_STORAGE_VAL}) if(USE_ONE_DATA_STORE_SHARD_ENABLED) # add preprocessor defition USE_ONE_DATA_STORE_SHARD add_compile_definitions(USE_ONE_DATA_STORE_SHARD) endif() -SET(MYELOQ_LIBRARY ${MYELOQ_LIBRARY} ${METRICS_LIB}) MYSQL_ADD_PLUGIN(eloq ${MYELOQ_SOURCES} STORAGE_ENGINE - # MODULE_ONLY MANDATORY STATIC_ONLY MODULE_OUTPUT_NAME ha_eloq diff --git a/storage/eloq/ha_eloq.cc b/storage/eloq/ha_eloq.cc index 32d2be7d4ce..3d6daacdd39 100644 --- a/storage/eloq/ha_eloq.cc +++ b/storage/eloq/ha_eloq.cc @@ -183,10 +183,22 @@ #include "tx_service/include/type.h" #include "tx_service/include/tx_execution.h" #include "tx_service/include/tx_service.h" -// #include "tx_service/include/tx_service_metrics.h" #include "tx_service/include/tx_request.h" #include "tx_service/include/util.h" +#ifdef MYSQLD_LIBRARY_MODE +#include +#include +// Define synchronization variables +namespace mysqld_converged_sync { + std::mutex init_mutex; + std::condition_variable mysqld_basic_init_done_cv; + std::condition_variable data_substrate_init_done_cv; + bool mysqld_basic_init_done = false; + bool data_substrate_init_done = false; +} +#endif + #define DEFAULT_SCAN_TUPLE_SIZE 128 // eloq_debug_set prefix "+d,eloq;" #define ELOQ_DEBUG_SET_PREFIX_LEN 8 @@ -198,6 +210,9 @@ using namespace MyEloq; using namespace txservice; +DEFINE_string(config, "", + "Path to data substrate configuration file."); + MariaCatalogFactory maria_catalog_factory; txservice::CatalogFactory *eloqsql_catalog_factory= &maria_catalog_factory; txservice::SystemHandler *eloqsql_system_handler= @@ -1333,7 +1348,7 @@ static void eloq_pre_shutdown(void) // MariaSystemHandler::ReloadCache() may create a temporary THD, whose // constructor depends on global_system_variables.table_plugin. However, it // will be set to NULL during plugin_shutdown(). - MariaSystemHandler::Instance().Shutdown(); + // MariaSystemHandler::Instance().Shutdown(); DBUG_VOID_RETURN; } @@ -1436,6 +1451,31 @@ static int eloq_init_func(void *p) MY_MUTEX_INIT_FAST); mysql_mutex_init(mono_mem_cmp_space_mutex_key, &mono_mem_cmp_space_mutex, MY_MUTEX_INIT_FAST); +#ifdef MYSQLD_LIBRARY_MODE + // In library mode (converged binary), use synchronization: + // 1. Signal that MySQL basic initialization is done + // 2. Wait for data substrate to be initialized by converged main + { + std::unique_lock lock(mysqld_converged_sync::init_mutex); + mysqld_converged_sync::mysqld_basic_init_done = true; + mysqld_converged_sync::mysqld_basic_init_done_cv.notify_one(); + + LOG(INFO) << "MySQL basic initialization complete, waiting for data substrate..."; + + // Wait for data substrate initialization to complete + mysqld_converged_sync::data_substrate_init_done_cv.wait(lock, [] { + return mysqld_converged_sync::data_substrate_init_done; + }); + + LOG(INFO) << "Data substrate initialized, MySQL continuing..."; + } +#else + // In standalone mode, initialize data substrate here + // Wait for mysqld to initialize before initializing data substrate + // as data substrate may need to access some mysqld variables during replay. + LOG(INFO) << "Standalone mode: Initializing data substrate..."; + DataSubstrate::InitializeGlobal(FLAGS_config); +#endif eloq_hton= (handlerton *) p; eloq_hton->create= eloq_create_handler; From 302c4ce1e3f7abb73eea893edaaee688df4e1afb Mon Sep 17 00:00:00 2001 From: liunyl Date: Thu, 23 Oct 2025 10:37:17 +0000 Subject: [PATCH 07/10] fix shutdown crash --- data_substrate/tx_service | 2 +- storage/eloq/eloq_system_handler.h | 2 +- storage/eloq/ha_eloq.cc | 4 +++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/data_substrate/tx_service b/data_substrate/tx_service index 6c4756431c1..285d2ad6879 160000 --- a/data_substrate/tx_service +++ b/data_substrate/tx_service @@ -1 +1 @@ -Subproject commit 6c4756431c15e774337109a2ca385a7c83f9b92e +Subproject commit 285d2ad68793b940705e75f3382e604ff747863a diff --git a/storage/eloq/eloq_system_handler.h b/storage/eloq/eloq_system_handler.h index 939b81c1bd4..0f2fbfb1c35 100644 --- a/storage/eloq/eloq_system_handler.h +++ b/storage/eloq/eloq_system_handler.h @@ -40,7 +40,7 @@ class MariaSystemHandler : public txservice::SystemHandler void ReloadCache(std::function done) override; - void Shutdown(); + void Shutdown() override; private: MariaSystemHandler(); diff --git a/storage/eloq/ha_eloq.cc b/storage/eloq/ha_eloq.cc index 3d6daacdd39..b2423362ae8 100644 --- a/storage/eloq/ha_eloq.cc +++ b/storage/eloq/ha_eloq.cc @@ -1569,8 +1569,10 @@ static int eloq_done_func(void *p) { DBUG_ENTER_FUNC(); +#ifndef MYSQLD_LIBRARY_MODE sql_print_information("Shutting down the data substrate."); - // DataSubstrate::GetGlobal()->Shutdown(); + DataSubstrate::GetGlobal()->Shutdown(); +#endif mysql_mutex_destroy(&mono_collation_data_mutex); mysql_mutex_destroy(&mono_mem_cmp_space_mutex); From 0be9c6adc1496826b9a1bb8f53482d34676caa8a Mon Sep 17 00:00:00 2001 From: liunyl Date: Fri, 24 Oct 2025 08:05:24 +0000 Subject: [PATCH 08/10] rename tx service to data_substrate --- .gitmodules | 2 +- data_substrate | 1 + data_substrate/CMakeLists.txt | 287 ---------- data_substrate/build_eloq_log_service.cmake | 282 --------- data_substrate/build_eloq_metrics.cmake | 44 -- data_substrate/build_log_service.cmake | 158 ----- data_substrate/build_tx_service.cmake | 346 ----------- data_substrate/core/include/data_substrate.h | 250 -------- .../core/include/glog_error_logging.h | 139 ----- data_substrate/core/include/mysql_metrics.h | 81 --- data_substrate/core/include/redis_metrics.h | 69 --- data_substrate/core/src/data_substrate.cpp | 530 ----------------- data_substrate/core/src/log_init.cpp | 541 ------------------ data_substrate/core/src/metrics_init.cpp | 216 ------- data_substrate/core/src/storage_init.cpp | 318 ---------- data_substrate/core/src/tx_service_init.cpp | 290 ---------- data_substrate/eloq_metrics | 1 - data_substrate/log_service | 1 - data_substrate/store_handler | 1 - data_substrate/tx_service | 1 - 20 files changed, 2 insertions(+), 3556 deletions(-) create mode 160000 data_substrate delete mode 100644 data_substrate/CMakeLists.txt delete mode 100644 data_substrate/build_eloq_log_service.cmake delete mode 100644 data_substrate/build_eloq_metrics.cmake delete mode 100644 data_substrate/build_log_service.cmake delete mode 100644 data_substrate/build_tx_service.cmake delete mode 100644 data_substrate/core/include/data_substrate.h delete mode 100644 data_substrate/core/include/glog_error_logging.h delete mode 100644 data_substrate/core/include/mysql_metrics.h delete mode 100644 data_substrate/core/include/redis_metrics.h delete mode 100644 data_substrate/core/src/data_substrate.cpp delete mode 100644 data_substrate/core/src/log_init.cpp delete mode 100644 data_substrate/core/src/metrics_init.cpp delete mode 100644 data_substrate/core/src/storage_init.cpp delete mode 100644 data_substrate/core/src/tx_service_init.cpp delete mode 160000 data_substrate/eloq_metrics delete mode 160000 data_substrate/log_service delete mode 160000 data_substrate/store_handler delete mode 160000 data_substrate/tx_service diff --git a/.gitmodules b/.gitmodules index 975797de598..08967b854c7 100644 --- a/.gitmodules +++ b/.gitmodules @@ -21,7 +21,7 @@ path = data_substrate/eloq_metrics url = https://github.com/eloqdata/eloq-metrics.git [submodule "data_substrate/tx_service"] - path = data_substrate/tx_service + path = data_substrate url = https://github.com/eloqdata/tx_service.git [submodule "data_substrate/store_handler"] path = data_substrate/store_handler diff --git a/data_substrate b/data_substrate new file mode 160000 index 00000000000..09b423ebb72 --- /dev/null +++ b/data_substrate @@ -0,0 +1 @@ +Subproject commit 09b423ebb72f83d9013116bf7b7b591a7105692c diff --git a/data_substrate/CMakeLists.txt b/data_substrate/CMakeLists.txt deleted file mode 100644 index 6fb9616f990..00000000000 --- a/data_substrate/CMakeLists.txt +++ /dev/null @@ -1,287 +0,0 @@ -cmake_minimum_required(VERSION 3.8) -project(data_substrate) - -# Set library name -set(DATA_SUBSTRATE_LIB "data_substrate" CACHE STRING "Data substrate library name") -set(METRICS_LIB "eloq-metrics" CACHE STRING "Metrics library name") - -# Options from original eloq CMakeLists -set(WITH_DATA_STORE "ELOQDSS_ROCKSDB" CACHE STRING "Which key-value storage to use") -set_property(CACHE WITH_DATA_STORE PROPERTY STRINGS "DYNAMODB" "BIGTABLE" "ELOQDSS_ROCKSDB_CLOUD_S3" "ELOQDSS_ELOQSTORE" "ELOQDSS_ROCKSDB") - -option(WITH_LOG_SERVICE "Compile with built-in log service." ON) -option(OPEN_LOG_SERVICE "Compile with open log service." ON) -option(SMALL_RANGE "Whether enable small range" OFF) -option(STATISTICS "Whether enable table statistics" ON) -option(EXT_TX_PROC_ENABLED "Allows external threads to move forward the tx service." ON) -option(ELOQ_MODULE_ENABLED "Register the tx service as an ELOQ module." OFF) - - -message(NOTICE "WITH_DATA_STORE: ${WITH_DATA_STORE}") -message(NOTICE "WITH_LOG_SERVICE: ${WITH_LOG_SERVICE}") -message(NOTICE "OPEN_LOG_SERVICE: ${OPEN_LOG_SERVICE}") -option(USE_ONE_ELOQDSS_PARTITION_ENABLED "Whether use one partition for kv store" OFF) -message(NOTICE "USE_ONE_ELOQDSS_PARTITION_ENABLED : ${USE_ONE_ELOQDSS_PARTITION_ENABLED}") - -if(USE_ONE_ELOQDSS_PARTITION_ENABLED) - add_compile_definitions(USE_ONE_ELOQDSS_PARTITION) -endif() -# Add compile definitions based on options -if(WITH_DATA_STORE STREQUAL "DYNAMODB") - set(KV_STORAGE_VAL 1 CACHE STRING "dynamodb" FORCE) - add_compile_definitions(DATA_STORE_TYPE_DYNAMODB) - find_package(AWSSDK REQUIRED COMPONENTS dynamodb) -elseif(WITH_DATA_STORE STREQUAL "BIGTABLE") - set(KV_STORAGE_VAL 2 CACHE STRING "big table" FORCE) - add_compile_definitions(DATA_STORE_TYPE_BIGTABLE) - find_package(google_cloud_cpp_bigtable REQUIRED) -elseif(WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB_CLOUD_S3") - set(KV_STORAGE_VAL 3 CACHE STRING "eloq_ds" FORCE) - add_compile_definitions(DATA_STORE_TYPE_ELOQDSS_ROCKSDB_CLOUD_S3) -elseif(WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB") - set(KV_STORAGE_VAL 3 CACHE STRING "eloq_ds" FORCE) - add_compile_definitions(DATA_STORE_TYPE_ELOQDSS_ROCKSDB) -elseif(WITH_DATA_STORE STREQUAL "ELOQDSS_ELOQSTORE") - set(KV_STORAGE_VAL 3 CACHE STRING "eloq_ds" FORCE) - add_compile_definitions(DATA_STORE_TYPE_ELOQDSS_ELOQSTORE) - option(WITH_TXSERVICE "Whether compile eloqstore with txservice" ON) - set(ELOQSTORE_PARENT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/store_handler/eloq_data_store_service CACHE PATH "EloqStore parent directory") - INCLUDE(${CMAKE_CURRENT_SOURCE_DIR}/store_handler/eloq_data_store_service/build_eloq_store.cmake) -else() - message(FATAL_ERROR "Unset WITH_DATA_STORE") -endif() - -# Find RocksDB if needed -if(WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB") - find_path(ROCKSDB_INCLUDE_PATH NAMES rocksdb/db.h) - if(NOT ROCKSDB_INCLUDE_PATH) - message(FATAL_ERROR "Fail to find RocksDB include path") - endif() - message(STATUS "ROCKSDB_INCLUDE_PATH: ${ROCKSDB_INCLUDE_PATH}") - include_directories(${ROCKSDB_INCLUDE_PATH}) - - find_library(ROCKSDB_LIB NAMES rocksdb) - if(NOT ROCKSDB_LIB) - message(FATAL_ERROR "Fail to find RocksDB lib path") - endif() - message(STATUS "ROCKSDB_LIB: ${ROCKSDB_LIB}") - set(ROCKSDB_LIBRARIES ${ROCKSDB_LIBRARIES} ${ROCKSDB_LIB}) -endif() - -# Handle RocksDB Cloud S3/GCS configurations -if((WITH_DATA_STORE STREQUAL "ROCKSDB_CLOUD_S3") OR (WITH_DATA_STORE STREQUAL "ROCKSDB_CLOUD_GCS") - OR (WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB_CLOUD_S3") OR (WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB_CLOUD_GCS")) - if((WITH_DATA_STORE STREQUAL "ROCKSDB_CLOUD_S3") OR (WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB_CLOUD_S3")) - find_path(AWS_CORE_INCLUDE_PATH aws/core/Aws.h) - if((NOT AWS_CORE_INCLUDE_PATH)) - message(FATAL_ERROR "Fail to find aws/core include path") - endif() - message(STATUS "aws/core include path: ${AWS_CORE_INCLUDE_PATH}") - - find_library(AWS_CORE_LIB aws-cpp-sdk-core) - if((NOT AWS_CORE_LIB )) - message(FATAL_ERROR "Fail to find aws-cpp-sdk-core lib") - endif() - message(STATUS "aws-cpp-sdk-core library: ${AWS_CORE_LIB}") - - find_path(AWS_S3_INCLUDE_PATH aws/s3/S3Client.h) - if((NOT AWS_S3_INCLUDE_PATH)) - message(FATAL_ERROR "Fail to find aws/s3 include path") - endif() - message(STATUS "aws/s3 include path: ${AWS_S3_INCLUDE_PATH}") - - find_library(AWS_S3_LIB aws-cpp-sdk-s3) - if((NOT AWS_S3_LIB )) - message(FATAL_ERROR "Fail to find aws-cpp-sdk-s3 lib") - endif() - message(STATUS "aws-cpp-sdk-s3 library: ${AWS_S3_LIB}") - - set(ROCKSDB_INCLUDE_PATH ${ROCKSDB_INCLUDE_PATH} ${AWS_CORE_INCLUDE_PATH}) - set(ROCKSDB_INCLUDE_PATH ${ROCKSDB_INCLUDE_PATH} ${AWS_S3_INCLUDE_PATH}) - - set(ROCKSDB_LIBRARIES ${ROCKSDB_LIBRARIES} ${AWS_CORE_LIB}) - set(ROCKSDB_LIBRARIES ${ROCKSDB_LIBRARIES} ${AWS_S3_LIB}) - - find_library(ROCKSDB_CLOUD_LIB NAMES rocksdb-cloud-aws) - endif() - - # Common RocksDB Cloud setup - find_path(ROCKSDB_CLOUD_INCLUDE_PATH NAMES rocksdb/db.h PATH_SUFFIXES "rocksdb_cloud_header") - if(NOT ROCKSDB_CLOUD_INCLUDE_PATH) - message(FATAL_ERROR "Fail to find RocksDB Cloud include path") - endif() - message(STATUS "ROCKSDB_CLOUD_INCLUDE_PATH: ${ROCKSDB_CLOUD_INCLUDE_PATH}") - set(ROCKSDB_INCLUDE_PATH ${ROCKSDB_INCLUDE_PATH} ${ROCKSDB_CLOUD_INCLUDE_PATH}) - include_directories(${ROCKSDB_INCLUDE_PATH}) - - if(NOT ROCKSDB_CLOUD_LIB) - message(FATAL_ERROR "Fail to find RocksDB Cloud lib path") - endif() - message(STATUS "ROCKSDB_CLOUD_LIB: ${ROCKSDB_CLOUD_LIB}") - set(ROCKSDB_LIBRARIES ${ROCKSDB_LIBRARIES} ${ROCKSDB_CLOUD_LIB}) -endif() - -# Compile proto file for data store service -if((WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB_CLOUD_S3") OR (WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB_CLOUD_GCS") OR (WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB") OR (WITH_DATA_STORE STREQUAL "ELOQDSS_ELOQSTORE")) - set(DS_PROTO_DIR ${CMAKE_CURRENT_SOURCE_DIR}/store_handler/eloq_data_store_service) - message(NOTICE "data store service proto dir: ${DS_PROTO_DIR}") - set(PROTO_SRC ${DS_PROTO_DIR}) - set(PROTO_NAME ds_request) - execute_process( - COMMAND protoc ./${PROTO_NAME}.proto --cpp_out=./ - WORKING_DIRECTORY ${PROTO_SRC} - ) -endif() - -# Add compile definitions based on options -if(WITH_LOG_SERVICE) - add_compile_definitions(WITH_LOG_SERVICE) -endif() - -if(OPEN_LOG_SERVICE) - add_compile_definitions(OPEN_LOG_SERVICE=1) -endif() - -if(EXT_TX_PROC_ENABLED) - add_compile_definitions(EXT_TX_PROC_ENABLED) -endif() - -if(ELOQ_MODULE_ENABLED) - add_compile_definitions(ELOQ_MODULE_ENABLED) -endif() - -if(SMALL_RANGE) - add_compile_definitions(SMALL_RANGE) -endif() - -if(STATISTICS) - add_compile_definitions(STATISTICS) -endif() - - -# Include build files for subcomponents -INCLUDE(build_tx_service.cmake) -INCLUDE(build_eloq_metrics.cmake) - -if(WITH_LOG_SERVICE) - if(OPEN_LOG_SERVICE) - INCLUDE(build_log_service.cmake) - else() - INCLUDE(build_eloq_log_service.cmake) - endif() -endif() - -# Find mimalloc -find_package(MIMALLOC REQUIRED) -include_directories(${MIMALLOC_INCLUDE_DIR}) - -# Include directories -include_directories( - ${CMAKE_CURRENT_SOURCE_DIR} - ${CMAKE_CURRENT_SOURCE_DIR}/core/include - ${CMAKE_CURRENT_SOURCE_DIR}/tx_service/include - ${CMAKE_CURRENT_SOURCE_DIR}/tx_service/include/cc - ${CMAKE_CURRENT_SOURCE_DIR}/tx_service/include/remote - ${CMAKE_CURRENT_SOURCE_DIR}/tx_service/include/fault - ${CMAKE_CURRENT_SOURCE_DIR}/tx_service/include/store - ${CMAKE_CURRENT_SOURCE_DIR}/tx_service/tx-log-protos - ${CMAKE_CURRENT_SOURCE_DIR}/eloq_metrics/include - ${CMAKE_CURRENT_SOURCE_DIR}/store_handler -) - -if(WITH_LOG_SERVICE) - if(OPEN_LOG_SERVICE) - include_directories(${CMAKE_CURRENT_SOURCE_DIR}/log_service/include) - else() - include_directories(${CMAKE_CURRENT_SOURCE_DIR}/eloq_log_service/include) - endif() -endif() - -# Collect all source files for the library -SET(DATA_SUBSTRATE_SOURCES - # Core data substrate files - core/src/data_substrate.cpp - core/src/log_init.cpp - core/src/metrics_init.cpp - core/src/storage_init.cpp - core/src/tx_service_init.cpp -) - -# Add store handler sources based on data store type -if(WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB" OR WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB_CLOUD_S3") - SET(DATA_SUBSTRATE_SOURCES ${DATA_SUBSTRATE_SOURCES} - store_handler/data_store_service_client.cpp - store_handler/data_store_service_client_closure.cpp - store_handler/data_store_service_scanner.cpp - store_handler/store_util.cpp - store_handler/eloq_data_store_service/ini.c - store_handler/eloq_data_store_service/INIReader.cpp - store_handler/eloq_data_store_service/thread_worker_pool.cpp - store_handler/eloq_data_store_service/data_store_service.cpp - store_handler/eloq_data_store_service/data_store_fault_inject.cpp - store_handler/eloq_data_store_service/data_store_service_config.cpp - store_handler/eloq_data_store_service/ds_request.pb.cc - store_handler/eloq_data_store_service/rocksdb_config.cpp - store_handler/eloq_data_store_service/rocksdb_data_store_common.cpp - ) - - if(WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB") - SET(DATA_SUBSTRATE_SOURCES ${DATA_SUBSTRATE_SOURCES} - store_handler/eloq_data_store_service/rocksdb_data_store.cpp - ) - elseif(WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB_CLOUD_S3") - SET(DATA_SUBSTRATE_SOURCES ${DATA_SUBSTRATE_SOURCES} - store_handler/eloq_data_store_service/rocksdb_cloud_data_store.cpp - store_handler/eloq_data_store_service/purger_event_listener.cpp - store_handler/eloq_data_store_service/purger_sliding_window.cpp - ) - endif() -elseif(WITH_DATA_STORE STREQUAL "ELOQDSS_ELOQSTORE") - SET(DATA_SUBSTRATE_SOURCES ${DATA_SUBSTRATE_SOURCES} - store_handler/data_store_service_client.cpp - store_handler/data_store_service_client_closure.cpp - store_handler/data_store_service_scanner.cpp - store_handler/store_util.cpp - store_handler/eloq_data_store_service/thread_worker_pool.cpp - store_handler/eloq_data_store_service/data_store_service.cpp - store_handler/eloq_data_store_service/data_store_fault_inject.cpp - store_handler/eloq_data_store_service/data_store_service_config.cpp - store_handler/eloq_data_store_service/eloq_store_data_store.cpp - store_handler/eloq_data_store_service/ds_request.pb.cc - store_handler/eloq_data_store_service/eloq_store_config.cpp - ) -endif() - -# Build the static library -ADD_LIBRARY(${DATA_SUBSTRATE_LIB} STATIC ${DATA_SUBSTRATE_SOURCES}) - -# Link with dependencies -SET(DATA_SUBSTRATE_LINK_LIBS - txservice - ${METRICS_LIB} -) - -if(WITH_LOG_SERVICE) - SET(DATA_SUBSTRATE_LINK_LIBS ${DATA_SUBSTRATE_LINK_LIBS} logservice) -endif() - -if(WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB" OR WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB_CLOUD_S3") - SET(DATA_SUBSTRATE_LINK_LIBS ${DATA_SUBSTRATE_LINK_LIBS} ${ROCKSDB_LIBRARIES}) -elseif(WITH_DATA_STORE STREQUAL "DYNAMODB") - SET(DATA_SUBSTRATE_LINK_LIBS ${DATA_SUBSTRATE_LINK_LIBS} ${AWSSDK_LINK_LIBRARIES}) -elseif(WITH_DATA_STORE STREQUAL "BIGTABLE") - SET(DATA_SUBSTRATE_LINK_LIBS ${DATA_SUBSTRATE_LINK_LIBS} google-cloud-cpp::bigtable) -elseif(WITH_DATA_STORE STREQUAL "ELOQDSS_ELOQSTORE") - SET(DATA_SUBSTRATE_LINK_LIBS ${DATA_SUBSTRATE_LINK_LIBS} eloqstore) -endif() - -TARGET_LINK_LIBRARIES(${DATA_SUBSTRATE_LIB} ${DATA_SUBSTRATE_LINK_LIBS}) - -# Export include directories for other targets -TARGET_INCLUDE_DIRECTORIES(${DATA_SUBSTRATE_LIB} PUBLIC - ${CMAKE_CURRENT_SOURCE_DIR} - ${CMAKE_CURRENT_SOURCE_DIR}/core/include - ${CMAKE_CURRENT_SOURCE_DIR}/tx_service/include - ${CMAKE_CURRENT_SOURCE_DIR}/eloq_metrics/include - ${CMAKE_CURRENT_SOURCE_DIR}/store_handler -) diff --git a/data_substrate/build_eloq_log_service.cmake b/data_substrate/build_eloq_log_service.cmake deleted file mode 100644 index 5412da2d33e..00000000000 --- a/data_substrate/build_eloq_log_service.cmake +++ /dev/null @@ -1,282 +0,0 @@ -SET (LOG_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/eloq_log_service) -SET(TX_LOG_PROTOS_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/tx_service/tx-log-protos) - -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error") - -option(BRPC_WITH_GLOG "With glog" ON) - -set(WITH_LOG_STATE "ROCKSDB" CACHE STRING "The log state implementation") -set_property(CACHE WITH_LOG_STATE PROPERTY STRINGS "MEMORY" "ROCKSDB" "ROCKSDB_CLOUD_S3" "ROCKSDB_CLOUD_GCS") -message(NOTICE "WITH_LOG_STATE: ${WITH_LOG_STATE}") - -# Add compile flags for LOG STATE TYPE -if(WITH_LOG_STATE STREQUAL "MEMORY") - add_compile_definitions(LOG_STATE_TYPE_MEM) -elseif(WITH_LOG_STATE STREQUAL "ROCKSDB") - add_compile_definitions(LOG_STATE_TYPE_RKDB) - add_compile_definitions(LOG_STATE_TYPE_RKDB_ALL) -elseif(WITH_LOG_STATE STREQUAL "ROCKSDB_CLOUD_S3") - add_compile_definitions(LOG_STATE_TYPE_RKDB_S3) -elseif(WITH_LOG_STATE STREQUAL "ROCKSDB_CLOUD_GCS") - add_compile_definitions(LOG_STATE_TYPE_RKDB_GCS) -else() - message(FATAL_ERROR "Unknown WITH_LOG_STATE: ${WITH_LOG_STATE}") -endif() - - -find_path(BRPC_INCLUDE_PATH NAMES brpc/stream.h) -find_library(BRPC_LIB NAMES brpc) -if ((NOT BRPC_INCLUDE_PATH) OR (NOT BRPC_LIB)) - message(FATAL_ERROR "Fail to find brpc") -endif() -include_directories(${BRPC_INCLUDE_PATH}) - -find_path(BRAFT_INCLUDE_PATH NAMES braft/raft.h) -find_library(BRAFT_LIB NAMES braft) -if ((NOT BRAFT_INCLUDE_PATH) OR (NOT BRAFT_LIB)) - message (FATAL_ERROR "Fail to find braft") -endif() -include_directories(${BRAFT_INCLUDE_PATH}) - -find_path(GFLAGS_INCLUDE_PATH gflags/gflags.h) -find_library(GFLAGS_LIBRARY NAMES gflags libgflags) -if((NOT GFLAGS_INCLUDE_PATH) OR (NOT GFLAGS_LIBRARY)) - message(FATAL_ERROR "Fail to find gflags") -endif() -include_directories(${GFLAGS_INCLUDE_PATH}) - -if(BRPC_WITH_GLOG) - message(NOTICE "log service brpc with glog") - find_path(GLOG_INCLUDE_PATH NAMES glog/logging.h) - find_library(GLOG_LIB NAMES glog VERSION ">=0.6.0" REQUIRED) - if((NOT GLOG_INCLUDE_PATH) OR (NOT GLOG_LIB)) - message(FATAL_ERROR "Fail to find glog") - endif() - include_directories(${GLOG_INCLUDE_PATH}) - set(LOG_LIB ${LOG_LIB} ${GLOG_LIB}) -endif() - -execute_process( - COMMAND bash -c "grep \"namespace [_A-Za-z0-9]\\+ {\" ${GFLAGS_INCLUDE_PATH}/gflags/gflags_declare.h | head -1 | awk '{print $2}' | tr -d '\n'" - OUTPUT_VARIABLE GFLAGS_NS -) -if(${GFLAGS_NS} STREQUAL "GFLAGS_NAMESPACE") - execute_process( - COMMAND bash -c "grep \"#define GFLAGS_NAMESPACE [_A-Za-z0-9]\\+\" ${GFLAGS_INCLUDE_PATH}/gflags/gflags_declare.h | head -1 | awk '{print $3}' | tr -d '\n'" - OUTPUT_VARIABLE GFLAGS_NS - ) -else() - add_compile_definitions(OVERRIDE_GFLAGS_NAMESPACE) -endif() - -#find_path(GPERFTOOLS_INCLUDE_DIR NAMES gperftools/heap-profiler.h) -#find_library(GPERFTOOLS_LIBRARIES NAMES tcmalloc_and_profiler) -#if (GPERFTOOLS_INCLUDE_DIR AND GPERFTOOLS_LIBRARIES) -# set(CMAKE_CXX_FLAGS "-DBRPC_ENABLE_CPU_PROFILER") -# include_directories(${GPERFTOOLS_INCLUDE_DIR}) -#else () -# set (GPERFTOOLS_LIBRARIES "") -#endif () - -#set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_CPP_FLAGS} -DGFLAGS_NS=${GFLAGS_NS} -DNDEBUG -O2 -D__const__= -pipe -W -Wall -Wno-unused-parameter -fPIC -fno-omit-frame-pointer") -if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - # require at least gcc 4.8 - if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 4.8) - message(FATAL_ERROR "GCC is too old, please install a newer version supporting C++11") - endif() -elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") - # require at least clang 3.3 - if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 3.3) - message(FATAL_ERROR "Clang is too old, please install a newer version supporting C++11") - endif() -else() - message(WARNING "You are using an unsupported compiler! Compilation has only been tested with Clang and GCC.") -endif() - - -#if(CMAKE_VERSION VERSION_LESS "3.1.3") -# if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") -# set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17") -# endif() -# if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") -# set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17") -# endif() -#else() -# set(CMAKE_CXX_STANDARD 17) -# set(CMAKE_CXX_STANDARD_REQUIRED ON) -#endif() - -find_path(LEVELDB_INCLUDE_PATH NAMES leveldb/db.h) -find_library(LEVELDB_LIB NAMES leveldb) -if ((NOT LEVELDB_INCLUDE_PATH) OR (NOT LEVELDB_LIB)) - message(FATAL_ERROR "Fail to find leveldb") -endif() -include_directories(${LEVELDB_INCLUDE_PATH}) - -set(LOG_SHIPPING_THREADS_NUM 1) - -# Add RocksDB include and libs for LOG_STATE -if (WITH_LOG_STATE MATCHES "ROCKSDB|ROCKSDB_CLOUD_S3|ROCKSDB_CLOUD_GCS") - if (WITH_LOG_STATE MATCHES "ROCKSDB_CLOUD_S3|ROCKSDB_CLOUD_GCS") - if (WITH_LOG_STATE STREQUAL "ROCKSDB_CLOUD_S3") - find_path(AWS_CORE_INCLUDE_PATH aws/core/Aws.h) - if((NOT AWS_CORE_INCLUDE_PATH)) - message(FATAL_ERROR "Fail to find aws/core include path") - endif() - message(STATUS "aws/core include path: ${AWS_CORE_INCLUDE_PATH}") - - find_library(AWS_CORE_LIB aws-cpp-sdk-core) - if((NOT AWS_CORE_LIB )) - message(FATAL_ERROR "Fail to find aws-cpp-sdk-core lib") - endif() - message(STATUS "aws-cpp-sdk-core library: ${AWS_CORE_LIB}") - - find_path(AWS_KINESIS_INCLUDE_PATH aws/kinesis/KinesisClient.h) - if((NOT AWS_KINESIS_INCLUDE_PATH)) - message(FATAL_ERROR "Fail to find aws/kinesis include path") - endif() - message(STATUS "aws/kinesis include path: ${AWS_KINESIS_INCLUDE_PATH}") - - find_library(AWS_KINESIS_LIB aws-cpp-sdk-kinesis) - if((NOT AWS_KINESIS_LIB)) - message(FATAL_ERROR "Fail to find aws-cpp-sdk-kinesis lib") - endif() - message(STATUS "aws-cpp-sdk-kinesis library: ${AWS_KINESIS_LIB}") - - - find_path(AWS_KINESIS_INCLUDE_PATH aws/kinesis/KinesisClient.h) - if((NOT AWS_KINESIS_INCLUDE_PATH)) - message(FATAL_ERROR "Fail to find aws/kinesis include path") - endif() - message(STATUS "aws/kinesis include path: ${AWS_KINESIS_INCLUDE_PATH}") - - find_library(AWS_KINESIS_LIB aws-cpp-sdk-kinesis) - if((NOT AWS_KINESIS_LIB)) - message(FATAL_ERROR "Fail to find aws-cpp-sdk-kinesis lib") - endif() - message(STATUS "aws-cpp-sdk-kinesis library: ${AWS_KINESIS_LIB}") - - find_path(AWS_S3_INCLUDE_PATH aws/s3/S3Client.h) - if((NOT AWS_S3_INCLUDE_PATH)) - message(FATAL_ERROR "Fail to find aws/s3 include path") - endif() - message(STATUS "aws/s3 include path: ${AWS_S3_INCLUDE_PATH}") - - find_library(AWS_S3_LIB aws-cpp-sdk-s3) - if((NOT AWS_S3_LIB )) - message(FATAL_ERROR "Fail to find aws-cpp-sdk-s3 lib") - endif() - message(STATUS "aws-cpp-sdk-s3 library: ${AWS_S3_LIB}") - - set(ROCKSDB_INCLUDE_PATH ${ROCKSDB_INCLUDE_PATH} ${AWS_CORE_INCLUDE_PATH}) - set(ROCKSDB_INCLUDE_PATH ${ROCKSDB_INCLUDE_PATH} ${AWS_KINESIS_INCLUDE_PATH}) - set(ROCKSDB_INCLUDE_PATH ${ROCKSDB_INCLUDE_PATH} ${AWS_S3_INCLUDE_PATH}) - - set(ROCKSDB_LIB ${ROCKSDB_LIB} ${AWS_CORE_LIB}) - set(ROCKSDB_LIB ${ROCKSDB_LIB} ${AWS_KINESIS_LIB}) - set(ROCKSDB_LIB ${ROCKSDB_LIB} ${AWS_S3_LIB}) - - find_library(ROCKSDB_CLOUD_LIB NAMES rocksdb-cloud-aws) - - add_compile_definitions(USE_AWS) - elseif (WITH_LOG_STATE STREQUAL "ROCKSDB_CLOUD_GCS") - find_path(GCP_CS_INCLUDE_PATH google/cloud/storage/client.h) - if((NOT GCP_CS_INCLUDE_PATH)) - message(FATAL_ERROR "Fail to find google/cloud/storage include path") - endif() - message(STATUS "google/cloud/storage include path: ${GCP_CS_INCLUDE_PATH}") - - find_library(GCP_COMMON_LIB google_cloud_cpp_common) - if((NOT GCP_COMMON_LIB)) - message(FATAL_ERROR "Fail to find google_cloud_cpp_common lib") - endif() - message(STATUS "google_cloud_cpp_common library: ${GCP_COMMON_LIB}") - - find_library(GCP_CS_LIB google_cloud_cpp_storage) - if((NOT GCP_CS_LIB)) - message(FATAL_ERROR "Fail to find google_cloud_cpp_storage lib") - endif() - message(STATUS "google_cloud_cpp_storage library: ${GCP_CS_LIB}") - - set(ROCKSDB_LIB ${ROCKSDB_LIB} ${GCP_COMMON_LIB}) - set(ROCKSDB_LIB ${ROCKSDB_LIB} ${GCP_CS_LIB}) - - find_library(ROCKSDB_CLOUD_LIB NAMES rocksdb-cloud-gcp) - - add_compile_definitions(USE_GCP) - endif () - - find_path(ROCKSDB_CLOUD_INCLUDE_PATH NAMES rocksdb/db.h PATH_SUFFIXES "rocksdb_cloud_header") - if (NOT ROCKSDB_CLOUD_INCLUDE_PATH) - message(FATAL_ERROR "Fail to find RocksDB Cloud include path") - endif () - message(STATUS "ROCKSDB_CLOUD_INCLUDE_PATH: ${ROCKSDB_CLOUD_INCLUDE_PATH}") - set(ROCKSDB_INCLUDE_PATH ${ROCKSDB_INCLUDE_PATH} ${ROCKSDB_CLOUD_INCLUDE_PATH}) - - if (NOT ROCKSDB_CLOUD_LIB) - message(FATAL_ERROR "Fail to find RocksDB Cloud lib path") - endif () - message(STATUS "ROCKSDB_CLOUD_LIB: ${ROCKSDB_CLOUD_LIB}") - set(ROCKSDB_LIB ${ROCKSDB_LIB} ${ROCKSDB_CLOUD_LIB}) - else () - find_path(ROCKSDB_INCLUDE_PATH NAMES rocksdb/db.h) - if (NOT ROCKSDB_INCLUDE_PATH) - message(FATAL_ERROR "Fail to find RocksDB include path") - endif () - message(STATUS "ROCKSDB_INCLUDE_PATH: ${ROCKSDB_INCLUDE_PATH}") - - find_library(ROCKSDB_LIB NAMES rocksdb) - if (NOT ROCKSDB_LIB) - message(FATAL_ERROR "Fail to find RocksDB lib path") - endif () - message(STATUS "ROCKSDB_LIB: ${ROCKSDB_LIB}") - endif () - - include_directories(${ROCKSDB_INCLUDE_PATH}) - - set(LOG_LIB - ${LOG_LIB} - ${ROCKSDB_LIB} - ) - - # one shipping thread is enough for rocksdb version log state - set(LOG_SHIPPING_THREADS_NUM 1) -endif () - -add_compile_definitions(LOG_SHIPPING_THREADS_NUM=${LOG_SHIPPING_THREADS_NUM}) - -set(LOG_INCLUDE_DIR - ${LOG_SOURCE_DIR}/include - ${TX_LOG_PROTOS_SOURCE_DIR} - ) - -set(LOG_LIB - ${LOG_LIB} - ${CMAKE_THREAD_LIBS_INIT} - ${GFLAGS_LIBRARY} - ${PROTOBUF_LIBRARY} - ${GPERFTOOLS_LIBRARIES} - ${LEVELDB_LIB} - ${BRAFT_LIB} - ${BRPC_LIB} - dl - z - ) - -find_package(Protobuf REQUIRED) - - -ADD_LIBRARY(logservice STATIC - ${LOG_SOURCE_DIR}/src/log_instance.cpp - ${LOG_SOURCE_DIR}/src/log_server.cpp - ${LOG_SOURCE_DIR}/src/log_state_rocksdb_impl.cpp - ${LOG_SOURCE_DIR}/src/log_state_rocksdb_cloud_impl.cpp - ${LOG_SOURCE_DIR}/src/log_state_memory_impl.cpp - ${LOG_SOURCE_DIR}/src/fault_inject.cpp - ${LOG_SOURCE_DIR}/src/INIReader.cpp - ${LOG_SOURCE_DIR}/src/ini.c - ${TX_LOG_PROTOS_SOURCE_DIR}/log.pb.cc - ) - -target_include_directories(logservice PUBLIC ${LOG_INCLUDE_DIR}) -target_link_libraries(logservice PUBLIC ${LOG_LIB} ${PROTOBUF_LIBRARIES}) diff --git a/data_substrate/build_eloq_metrics.cmake b/data_substrate/build_eloq_metrics.cmake deleted file mode 100644 index 18df1d3c0b9..00000000000 --- a/data_substrate/build_eloq_metrics.cmake +++ /dev/null @@ -1,44 +0,0 @@ -set(ELOQ_METRICS_ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/eloq_metrics") -set(ELOQ_METRICS_SRC_DIR "${ELOQ_METRICS_ROOT_DIR}/src") -set(ELOQ_METRICS_INCLUDE_DIR "${ELOQ_METRICS_ROOT_DIR}/include") - -option(ELOQ_METRICS_ENABLE_BENCHMARK "Whether enable google benchmark" OFF) -option(ELOQ_METRICS_WITH_GLOG "Whether use glog" ON) -option(ENABLE_ELOQ_METRICS_APP "Whether enable mono metrics app" OFF) - -message(STATUS "ELOQ_METRICS_ENABLE_BENCHMARK ${ELOQ_METRICS_ENABLE_BENCHMARK}") -message(STATUS "ELOQ_METRICS_WITH_GLOG ${ELOQ_METRICS_WITH_GLOG}") -message(STATUS "ENABLE_ELOQ_METRICS_APP ${ENABLE_ELOQ_METRICS_APP}") -message(STATUS "ELOQ_METRICS_ROOT_DIR ${ELOQ_METRICS_ROOT_DIR}") - -if(ELOQ_METRICS_WITH_GLOG) - add_compile_definitions(WITH_GLOG=1) -endif() - -if(POLICY CMP0074) - cmake_policy(SET CMP0074 NEW) -endif() - -if(POLICY CMP0054) - cmake_policy(SET CMP0054 NEW) -endif() - -find_package(prometheus-cpp CONFIG REQUIRED) - -set(ELOQ_METRICS_TARGET_SOURCE_LIST - ${ELOQ_METRICS_INCLUDE_DIR}/metrics.h - ${ELOQ_METRICS_INCLUDE_DIR}/meter.h - ${ELOQ_METRICS_INCLUDE_DIR}/metrics_collector.h - ${ELOQ_METRICS_INCLUDE_DIR}/prometheus_collector.h - ${ELOQ_METRICS_INCLUDE_DIR}/metrics_manager.h - ${ELOQ_METRICS_INCLUDE_DIR}/metrics_registry_impl.h - ${ELOQ_METRICS_SRC_DIR}/metrics.cc - ${ELOQ_METRICS_SRC_DIR}/prometheus_collector.cc - ${ELOQ_METRICS_SRC_DIR}/metrics_manager.cc - ${ELOQ_METRICS_SRC_DIR}/metrics_registry_impl.cpp -) -add_library(${METRICS_LIB} STATIC ${ELOQ_METRICS_TARGET_SOURCE_LIST}) - -target_include_directories(${METRICS_LIB} PUBLIC ${ELOQ_METRICS_INCLUDE_DIR}) - -target_link_libraries(${METRICS_LIB} PUBLIC prometheus-cpp::pull) diff --git a/data_substrate/build_log_service.cmake b/data_substrate/build_log_service.cmake deleted file mode 100644 index 21cc010b861..00000000000 --- a/data_substrate/build_log_service.cmake +++ /dev/null @@ -1,158 +0,0 @@ -SET (LOG_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/log_service) -SET(TX_LOG_PROTOS_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/tx_service/tx-log-protos) - -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error") - -option(BRPC_WITH_GLOG "With glog" ON) - -find_path(BRPC_INCLUDE_PATH NAMES brpc/stream.h) -find_library(BRPC_LIB NAMES brpc) -if ((NOT BRPC_INCLUDE_PATH) OR (NOT BRPC_LIB)) - message(FATAL_ERROR "Fail to find brpc") -endif() -include_directories(${BRPC_INCLUDE_PATH}) - -find_path(BRAFT_INCLUDE_PATH NAMES braft/raft.h) -find_library(BRAFT_LIB NAMES braft) -if ((NOT BRAFT_INCLUDE_PATH) OR (NOT BRAFT_LIB)) - message (FATAL_ERROR "Fail to find braft") -endif() -include_directories(${BRAFT_INCLUDE_PATH}) - -find_path(GFLAGS_INCLUDE_PATH gflags/gflags.h) -find_library(GFLAGS_LIBRARY NAMES gflags libgflags) -if((NOT GFLAGS_INCLUDE_PATH) OR (NOT GFLAGS_LIBRARY)) - message(FATAL_ERROR "Fail to find gflags") -endif() -include_directories(${GFLAGS_INCLUDE_PATH}) - -if(BRPC_WITH_GLOG) - message(NOTICE "log service brpc with glog") - find_path(GLOG_INCLUDE_PATH NAMES glog/logging.h) - find_library(GLOG_LIB NAMES glog VERSION ">=0.6.0" REQUIRED) - if((NOT GLOG_INCLUDE_PATH) OR (NOT GLOG_LIB)) - message(FATAL_ERROR "Fail to find glog") - endif() - include_directories(${GLOG_INCLUDE_PATH}) - set(LOG_LIB ${LOG_LIB} ${GLOG_LIB}) -endif() - -execute_process( - COMMAND bash -c "grep \"namespace [_A-Za-z0-9]\\+ {\" ${GFLAGS_INCLUDE_PATH}/gflags/gflags_declare.h | head -1 | awk '{print $2}' | tr -d '\n'" - OUTPUT_VARIABLE GFLAGS_NS -) -if(${GFLAGS_NS} STREQUAL "GFLAGS_NAMESPACE") - execute_process( - COMMAND bash -c "grep \"#define GFLAGS_NAMESPACE [_A-Za-z0-9]\\+\" ${GFLAGS_INCLUDE_PATH}/gflags/gflags_declare.h | head -1 | awk '{print $3}' | tr -d '\n'" - OUTPUT_VARIABLE GFLAGS_NS - ) -else() - add_compile_definitions(OVERRIDE_GFLAGS_NAMESPACE) -endif() - -#find_path(GPERFTOOLS_INCLUDE_DIR NAMES gperftools/heap-profiler.h) -#find_library(GPERFTOOLS_LIBRARIES NAMES tcmalloc_and_profiler) -#if (GPERFTOOLS_INCLUDE_DIR AND GPERFTOOLS_LIBRARIES) -# set(CMAKE_CXX_FLAGS "-DBRPC_ENABLE_CPU_PROFILER") -# include_directories(${GPERFTOOLS_INCLUDE_DIR}) -#else () -# set (GPERFTOOLS_LIBRARIES "") -#endif () - -#set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_CPP_FLAGS} -DGFLAGS_NS=${GFLAGS_NS} -DNDEBUG -O2 -D__const__= -pipe -W -Wall -Wno-unused-parameter -fPIC -fno-omit-frame-pointer") -if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - # require at least gcc 4.8 - if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 4.8) - message(FATAL_ERROR "GCC is too old, please install a newer version supporting C++11") - endif() -elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") - # require at least clang 3.3 - if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 3.3) - message(FATAL_ERROR "Clang is too old, please install a newer version supporting C++11") - endif() -else() - message(WARNING "You are using an unsupported compiler! Compilation has only been tested with Clang and GCC.") -endif() - - -#if(CMAKE_VERSION VERSION_LESS "3.1.3") -# if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") -# set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17") -# endif() -# if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") -# set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17") -# endif() -#else() -# set(CMAKE_CXX_STANDARD 17) -# set(CMAKE_CXX_STANDARD_REQUIRED ON) -#endif() - -find_path(LEVELDB_INCLUDE_PATH NAMES leveldb/db.h) -find_library(LEVELDB_LIB NAMES leveldb) -if ((NOT LEVELDB_INCLUDE_PATH) OR (NOT LEVELDB_LIB)) - message(FATAL_ERROR "Fail to find leveldb") -endif() -include_directories(${LEVELDB_INCLUDE_PATH}) - -find_path(ROCKSDB_INCLUDE_PATH NAMES rocksdb/db.h) - -if (NOT ROCKSDB_INCLUDE_PATH) -message(FATAL_ERROR "Fail to find RocksDB include path") -endif () -message(STATUS "ROCKSDB_INCLUDE_PATH: ${ROCKSDB_INCLUDE_PATH}") - -find_library(ROCKSDB_LIB NAMES rocksdb) -if (NOT ROCKSDB_LIB) - message(FATAL_ERROR "Fail to find RocksDB lib path") -endif () -message(STATUS "ROCKSDB_LIB: ${ROCKSDB_LIB}") - -include_directories(${ROCKSDB_INCLUDE_PATH}) - -set(LOG_LIB - ${LOG_LIB} - ${ROCKSDB_LIB} - ) - -# one shipping thread is enough for rocksdb version log state -set(LOG_SHIPPING_THREADS_NUM 1) - -add_compile_definitions(LOG_SHIPPING_THREADS_NUM=${LOG_SHIPPING_THREADS_NUM}) - -# Add for compatiable reason -add_compile_definitions(LOG_STATE_TYPE_RKDB) - -set(LOG_INCLUDE_DIR - ${LOG_SOURCE_DIR}/include - ${TX_LOG_PROTOS_SOURCE_DIR} - ) - -set(LOG_LIB - ${LOG_LIB} - ${CMAKE_THREAD_LIBS_INIT} - ${GFLAGS_LIBRARY} - ${PROTOBUF_LIBRARY} - ${GPERFTOOLS_LIBRARIES} - ${LEVELDB_LIB} - ${BRAFT_LIB} - ${BRPC_LIB} - dl - z - ) - -find_package(Protobuf REQUIRED) - - -ADD_LIBRARY(logservice STATIC - ${LOG_SOURCE_DIR}/src/log_server.cpp - ${LOG_SOURCE_DIR}/src/log_state_rocksdb_impl.cpp - ${LOG_SOURCE_DIR}/src/fault_inject.cpp - ${LOG_SOURCE_DIR}/src/INIReader.cpp - ${LOG_SOURCE_DIR}/src/ini.c - ${TX_LOG_PROTOS_SOURCE_DIR}/log.pb.cc - ${LOG_SOURCE_DIR}/src/open_log_service.cpp - ${LOG_SOURCE_DIR}/src/open_log_task.cpp - ) - -target_include_directories(logservice PUBLIC ${LOG_INCLUDE_DIR}) -target_link_libraries(logservice PUBLIC ${LOG_LIB} ${PROTOBUF_LIBRARIES}) diff --git a/data_substrate/build_tx_service.cmake b/data_substrate/build_tx_service.cmake deleted file mode 100644 index be16ca79f24..00000000000 --- a/data_substrate/build_tx_service.cmake +++ /dev/null @@ -1,346 +0,0 @@ -SET(TX_SERVICE_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/tx_service) -SET(METRICS_SERVICE_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/eloq_metrics) - -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-parentheses -Wno-error") -set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -DFAULT_INJECTOR") - -option(BRPC_WITH_GLOG "With glog" ON) - -option(EXT_TX_PROC_ENABLED "Allows external threads to move forward the tx service." ON) - -if(EXT_TX_PROC_ENABLED) - add_definitions(-DEXT_TX_PROC_ENABLED) -endif() - -option(ELOQ_MODULE_ENABLED "Register the tx service as an ELOQ module." OFF) -message(NOTICE "Tx service ELOQ_MODULE_ENABLED : ${ELOQ_MODULE_ENABLED}") - -if(ELOQ_MODULE_ENABLED) - add_definitions(-DELOQ_MODULE_ENABLED) -endif() - -option(FORK_HM_PROCESS "Whether fork host manager process" OFF) -message(NOTICE "FORK_HM_PROCESS : ${FORK_HM_PROCESS}") - -option(STATISTICS "Whether enable table statistics" OFF) -message(NOTICE "STATISTICS : ${STATISTICS}") - -find_package(Protobuf REQUIRED) -find_package(GFLAGS REQUIRED) -find_package(MIMALLOC REQUIRED) - -find_path(GFLAGS_INCLUDE_PATH gflags/gflags.h) -find_library(GFLAGS_LIBRARY NAMES gflags libgflags) - -if((NOT GFLAGS_INCLUDE_PATH) OR(NOT GFLAGS_LIBRARY)) - message(FATAL_ERROR "Fail to find gflags") -endif() - -execute_process( - COMMAND bash -c "grep \"namespace [_A-Za-z0-9]\\+ {\" ${GFLAGS_INCLUDE_PATH}/gflags/gflags_declare.h | head -1 | awk '{print $2}' | tr -d '\n'" - OUTPUT_VARIABLE GFLAGS_NS -) - -if(${GFLAGS_NS} STREQUAL "GFLAGS_NAMESPACE") - execute_process( - COMMAND bash -c "grep \"#define GFLAGS_NAMESPACE [_A-Za-z0-9]\\+\" ${GFLAGS_INCLUDE_PATH}/gflags/gflags_declare.h | head -1 | awk '{print $3}' | tr -d '\n'" - OUTPUT_VARIABLE GFLAGS_NS - ) -else() - add_compile_definitions(OVERRIDE_GFLAGS_NAMESPACE) -endif() - -if(SMALL_RANGE) - add_compile_definitions(SMALL_RANGE) -endif() - -if(FORK_HM_PROCESS) - add_compile_definitions(FORK_HM_PROCESS) -endif() - -if(STATISTICS) - add_definitions(-DSTATISTICS) -endif() - -if(CMAKE_COMPILER_IS_GNUCC) - find_path(BRPC_INCLUDE_PATH NAMES brpc/stream.h) - find_library(BRPC_LIB NAMES brpc) - - if((NOT BRPC_INCLUDE_PATH) OR(NOT BRPC_LIB)) - message(FATAL_ERROR "Fail to find brpc") - endif() - - if(BRPC_WITH_GLOG) - message(NOTICE "TX BRPC WITH GLOG") - find_path(GLOG_INCLUDE_PATH NAMES glog/logging.h) - find_library(GLOG_LIB NAMES glog VERSION ">=0.6.0" REQUIRED) - - if((NOT GLOG_INCLUDE_PATH) OR(NOT GLOG_LIB)) - message(FATAL_ERROR "Fail to find glog") - endif() - - include_directories(${GLOG_INCLUDE_PATH}) - set(LINK_LIB ${LINK_LIB} ${GLOG_LIB}) - endif() - - find_path(LEVELDB_INCLUDE_PATH NAMES leveldb/db.h) - find_library(LEVELDB_LIB NAMES leveldb) - - if((NOT LEVELDB_INCLUDE_PATH) OR(NOT LEVELDB_LIB)) - message(FATAL_ERROR "Fail to find leveldb") - endif() -endif() - -# Compile all protobuf files under txservice proto directory. -SET(PROTO_SRC ${TX_SERVICE_SOURCE_DIR}/include/proto) -FILE(GLOB PROTO_FILES RELATIVE ${PROTO_SRC} ${PROTO_SRC}/*.proto) - -FOREACH(PROTO_FILE ${PROTO_FILES}) - STRING(REGEX REPLACE "[^/]proto" "" PROTO_NAME ${PROTO_FILE}) - LIST(APPEND PROTO_CC_FILES ${PROTO_SRC}/${PROTO_NAME}.pb.cc) - ADD_CUSTOM_COMMAND( - OUTPUT "${PROTO_SRC}/${PROTO_NAME}.pb.cc" "${PROTO_SRC}/${PROTO_NAME}.pb.h" - DEPENDS ${PROTO_SRC}/${PROTO_FILE} - COMMAND protoc ${PROTO_FILE} --proto_path=./ --cpp_out=./ - WORKING_DIRECTORY ${PROTO_SRC} - ) -ENDFOREACH(PROTO_FILE) - -set(LOG_PROTO_SRC ${CMAKE_CURRENT_SOURCE_DIR}/tx_service/tx-log-protos) -set(LOG_PROTO_NAME log) -execute_process( - COMMAND protoc ./${LOG_PROTO_NAME}.proto --cpp_out=./ --proto_path=./ - WORKING_DIRECTORY ${LOG_PROTO_SRC} -) - -add_subdirectory(tx_service/abseil-cpp) - -message(${TX_SERVICE_SOURCE_DIR}) -set(INCLUDE_DIR - ${TX_SERVICE_SOURCE_DIR}/include - ${TX_SERVICE_SOURCE_DIR}/include/cc - ${TX_SERVICE_SOURCE_DIR}/include/remote - ${TX_SERVICE_SOURCE_DIR}/include/fault - ${TX_SERVICE_SOURCE_DIR}/tx-log-protos - ${METRICS_SERVICE_SOURCE_DIR}/include - ${Protobuf_INCLUDE_DIR} - ${MIMALLOC_INCLUDE_DIR}) - -if(CMAKE_COMPILER_IS_GNUCC) - set(INCLUDE_DIR ${INCLUDE_DIR} - ${BRPC_INCLUDE_PATH} - ${GLOG_INCLUDE_PATH} - ${GFLAGS_INCLUDE_PATH}) -endif() - -set(LINK_LIB ${LINK_LIB} ${PROTOBUF_LIBRARY}) - -set(LINK_LIB ${LINK_LIB} - mimalloc - absl::btree - absl::flat_hash_map - ${GFLAGS_LIBRARY} - ${LEVELDB_LIB} -) - -if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin") - set(LINK_LIB ${LINK_LIB} - ${BRPC_LIB} - ) -endif() - -SET(ELOQ_SOURCES - ${TX_SERVICE_SOURCE_DIR}/src/tx_key.cpp - ${TX_SERVICE_SOURCE_DIR}/src/tx_execution.cpp - ${TX_SERVICE_SOURCE_DIR}/src/tx_operation.cpp - ${TX_SERVICE_SOURCE_DIR}/src/checkpointer.cpp - ${TX_SERVICE_SOURCE_DIR}/src/tx_trace.cpp - ${TX_SERVICE_SOURCE_DIR}/src/tx_start_ts_collector.cpp - ${TX_SERVICE_SOURCE_DIR}/src/tx_worker_pool.cpp - ${TX_SERVICE_SOURCE_DIR}/src/sharder.cpp - ${TX_SERVICE_SOURCE_DIR}/src/standby.cpp - ${TX_SERVICE_SOURCE_DIR}/src/catalog_key_record.cpp - ${TX_SERVICE_SOURCE_DIR}/src/range_record.cpp - ${TX_SERVICE_SOURCE_DIR}/src/range_bucket_key_record.cpp - ${TX_SERVICE_SOURCE_DIR}/src/cc/cc_entry.cpp - ${TX_SERVICE_SOURCE_DIR}/src/cc/cc_map.cpp - ${TX_SERVICE_SOURCE_DIR}/src/cc/cc_shard.cpp - ${TX_SERVICE_SOURCE_DIR}/src/cc/cc_handler_result.cpp - ${TX_SERVICE_SOURCE_DIR}/src/cc/local_cc_handler.cpp - ${TX_SERVICE_SOURCE_DIR}/src/cc/local_cc_shards.cpp - ${TX_SERVICE_SOURCE_DIR}/src/cc/non_blocking_lock.cpp - ${TX_SERVICE_SOURCE_DIR}/src/cc/cc_req_misc.cpp - ${TX_SERVICE_SOURCE_DIR}/src/cc/range_slice.cpp - ${TX_SERVICE_SOURCE_DIR}/src/cc/reader_writer_cntl.cpp - ${TX_SERVICE_SOURCE_DIR}/src/eloq_string_key_record.cpp - ${TX_SERVICE_SOURCE_DIR}/src/eloq_basic_catalog_factory.cpp - ${TX_SERVICE_SOURCE_DIR}/src/remote/remote_cc_handler.cpp - ${TX_SERVICE_SOURCE_DIR}/src/remote/remote_cc_request.cpp - ${TX_SERVICE_SOURCE_DIR}/src/remote/cc_node_service.cpp - ${TX_SERVICE_SOURCE_DIR}/src/remote/cc_stream_receiver.cpp - ${TX_SERVICE_SOURCE_DIR}/src/remote/cc_stream_sender.cpp - ${TX_SERVICE_SOURCE_DIR}/src/fault/log_replay_service.cpp - ${TX_SERVICE_SOURCE_DIR}/src/fault/cc_node.cpp - ${TX_SERVICE_SOURCE_DIR}/src/fault/fault_inject.cpp - ${TX_SERVICE_SOURCE_DIR}/src/dead_lock_check.cpp - ${TX_SERVICE_SOURCE_DIR}/src/tx_index_operation.cpp - ${TX_SERVICE_SOURCE_DIR}/src/sk_generator.cpp - ${TX_SERVICE_SOURCE_DIR}/src/data_sync_task.cpp - ${TX_SERVICE_SOURCE_DIR}/src/store/snapshot_manager.cpp - ${TX_SERVICE_SOURCE_DIR}/src/sequences/sequences.cpp - ${TX_SERVICE_SOURCE_DIR}/tx-log-protos/log_agent.cpp - ${TX_SERVICE_SOURCE_DIR}/tx-log-protos/log.pb.cc - ${METRICS_SERVICE_SOURCE_DIR}/src/metrics.cc -) - -set(INCLUDE_DIR ${INCLUDE_DIR} ${PROTO_SRC}) -set(ELOQ_SOURCES ${ELOQ_SOURCES} ${PROTO_CC_FILES}) - -ADD_LIBRARY(txservice STATIC - ${ELOQ_SOURCES}) - -target_include_directories(txservice PUBLIC ${INCLUDE_DIR}) - -target_link_libraries(txservice PUBLIC ${LINK_LIB} ${PROTOBUF_LIBRARIES}) - -# if(FORK_HM_PROCESS) -# include(FetchContent) -# # Import yaml-cpp library used by host manager -# FetchContent_Declare( -# yaml-cpp -# GIT_REPOSITORY https://github.com/jbeder/yaml-cpp.git -# GIT_TAG yaml-cpp-0.7.0 # Can be a tag (yaml-cpp-x.x.x), a commit hash, or a branch name (master) -# ) -# FetchContent_MakeAvailable(yaml-cpp) - -# SET(HOST_MANAGER_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/tx_service/raft_host_manager) -# set(HOST_MANAGER_INCLUDE_DIR -# ${HOST_MANAGER_SOURCE_DIR}/include -# ${TX_SERVICE_SOURCE_DIR}/tx-log-protos -# ${OPENSSL_INCLUDE_DIR} -# ${LOG_PROTO_SRC} -# ${PROTO_SRC}) - -# if(CMAKE_COMPILER_IS_GNUCC) -# set(HOST_MANAGER_INCLUDE_DIR ${HOST_MANAGER_INCLUDE_DIR} -# ${BRPC_INCLUDE_PATH} -# ${BRAFT_INCLUDE_PATH} -# ${GLOG_INCLUDE_PATH} -# ${GFLAGS_INCLUDE_PATH}) -# endif() - -# set(HOST_MANAGER_LINK_LIB ${HOST_MANAGER_LINK_LIB} ${PROTOBUF_LIBRARIES}) - -# if(CMAKE_COMPILER_IS_GNUCC) -# find_path(BRAFT_INCLUDE_PATH NAMES braft/raft.h) -# find_library(BRAFT_LIB NAMES braft) - -# if((NOT BRAFT_INCLUDE_PATH) OR(NOT BRAFT_LIB)) -# message(FATAL_ERROR "Fail to find braft") -# endif() - -# set(HOST_MANAGER_LINK_LIB ${HOST_MANAGER_LINK_LIB} -# ${GFLAGS_LIBRARY} -# ${LEVELDB_LIB} -# ${BRAFT_LIB} -# ${BRPC_LIB} -# ${OPENSSL_LIB}) -# find_path(GLOG_INCLUDE_PATH NAMES glog/logging.h) -# find_library(GLOG_LIB NAMES glog VERSION ">=0.6.0" REQUIRED) - -# if((NOT GLOG_INCLUDE_PATH) OR(NOT GLOG_LIB)) -# message(FATAL_ERROR "Fail to find glog") -# endif() - -# include_directories(${GLOG_INCLUDE_PATH}) -# set(HOST_MANAGER_LINK_LIB ${HOST_MANAGER_LINK_LIB} ${GLOG_LIB}) -# endif() - -# SET(RaftHM_SOURCES -# ${HOST_MANAGER_SOURCE_DIR}/src/main.cpp -# ${HOST_MANAGER_SOURCE_DIR}/src/raft_host_manager_service.cpp -# ${HOST_MANAGER_SOURCE_DIR}/src/raft_host_manager.cpp -# ${HOST_MANAGER_SOURCE_DIR}/src/ini.c -# ${HOST_MANAGER_SOURCE_DIR}/src/INIReader.cpp -# ${LOG_PROTO_SRC}/log_agent.cpp -# ${LOG_PROTO_SRC}/${LOG_PROTO_NAME}.pb.cc -# ${PROTO_CC_FILES} -# ) - -# include_directories(${HOST_MANAGER_INCLUDE_DIR}) -# MYSQL_ADD_EXECUTABLE(host_manager ${RaftHM_SOURCES} DESTINATION ${INSTALL_SBINDIR} COMPONENT Server) -# target_link_libraries(host_manager ${HOST_MANAGER_LINK_LIB} yaml-cpp::yaml-cpp) - -# endif() - -if (FORK_HM_PROCESS) - SET (HOST_MANAGER_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/tx_service/raft_host_manager) - set(HOST_MANAGER_INCLUDE_DIR - ${HOST_MANAGER_SOURCE_DIR}/include - ${TX_SERVICE_SOURCE_DIR}/tx-log-protos - ${OPENSSL_INCLUDE_DIR} - ${PROTO_SRC} - ${LOG_PROTO_SRC}) - - set(HOST_MANAGER_INCLUDE_DIR ${HOST_MANAGER_INCLUDE_DIR} - ${BRPC_INCLUDE_PATH} - ${BRAFT_INCLUDE_PATH} - ${GLOG_INCLUDE_PATH} - ${GFLAGS_INCLUDE_PATH}) - - set(HOST_MANAGER_LINK_LIB ${HOST_MANAGER_LINK_LIB} ${PROTOBUF_LIBRARIES}) - - find_path(BRAFT_INCLUDE_PATH NAMES braft/raft.h) - find_library(BRAFT_LIB NAMES braft) - if ((NOT BRAFT_INCLUDE_PATH) OR (NOT BRAFT_LIB)) - message (FATAL_ERROR "Fail to find braft") - endif() - set(HOST_MANAGER_LINK_LIB ${HOST_MANAGER_LINK_LIB} - ${GFLAGS_LIBRARY} - ${GPERFTOOLS_LIBRARIES} - ${LEVELDB_LIB} - ${BRAFT_LIB} - ${BRPC_LIB} - ${OPENSSL_LIB}) - find_path(GLOG_INCLUDE_PATH NAMES glog/logging.h) - find_library(GLOG_LIB NAMES glog VERSION ">=0.6.0" REQUIRED) - if((NOT GLOG_INCLUDE_PATH) OR (NOT GLOG_LIB)) - message(FATAL_ERROR "Fail to find glog") - endif() - include_directories(${GLOG_INCLUDE_PATH}) - set(HOST_MANAGER_LINK_LIB ${HOST_MANAGER_LINK_LIB} ${GLOG_LIB}) - - SET(RaftHM_SOURCES - ${HOST_MANAGER_SOURCE_DIR}/src/main.cpp - ${HOST_MANAGER_SOURCE_DIR}/src/raft_host_manager_service.cpp - ${HOST_MANAGER_SOURCE_DIR}/src/raft_host_manager.cpp - ${HOST_MANAGER_SOURCE_DIR}/src/ini.c - ${HOST_MANAGER_SOURCE_DIR}/src/INIReader.cpp - ${PROTO_SRC}/${PROTO_NAME}.pb.cc - ${LOG_PROTO_SRC}/log_agent.cpp - ${LOG_PROTO_SRC}/${LOG_PROTO_NAME}.pb.cc - ) - - include(FetchContent) - - # Import yaml-cpp library used by host manager - FetchContent_Declare( - yaml-cpp - GIT_REPOSITORY https://github.com/jbeder/yaml-cpp.git - GIT_TAG yaml-cpp-0.7.0 # Can be a tag (yaml-cpp-x.x.x), a commit hash, or a branch name (master) - ) - FetchContent_MakeAvailable(yaml-cpp) - set(HOST_MANAGER_LINK_LIB ${HOST_MANAGER_LINK_LIB} yaml-cpp::yaml-cpp) - - include_directories(${HOST_MANAGER_INCLUDE_DIR}) - add_executable(host_manager ${RaftHM_SOURCES}) - target_link_libraries(host_manager PRIVATE ${HOST_MANAGER_LINK_LIB} ) - - set_target_properties(host_manager PROPERTIES - BUILD_RPATH "$ORIGIN/../lib" - INSTALL_RPATH "$ORIGIN/../lib" - INSTALL_RPATH_USE_LINK_PATH TRUE) - - install(TARGETS host_manager RUNTIME DESTINATION bin) -endif() - diff --git a/data_substrate/core/include/data_substrate.h b/data_substrate/core/include/data_substrate.h deleted file mode 100644 index f9a801da4d3..00000000000 --- a/data_substrate/core/include/data_substrate.h +++ /dev/null @@ -1,250 +0,0 @@ -/** - * Copyright (C) 2025 EloqData Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under either of the following two licenses: - * 1. GNU Affero General Public License, version 3, as published by the Free - * Software Foundation. - * 2. GNU General Public License as published by the Free Software - * Foundation; version 2 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License or GNU General Public License for more - * details. - * - * You should have received a copy of the GNU Affero General Public License - * and GNU General Public License V2 along with this program. If not, see - * . - * - */ -#pragma once - -#include - -#include -#include -#include -#include -#include - -#if (defined(DATA_STORE_TYPE_ELOQDSS_ROCKSDB_CLOUD_S3) || \ - defined(DATA_STORE_TYPE_ELOQDSS_ROCKSDB_CLOUD_GCS) || \ - defined(DATA_STORE_TYPE_ELOQDSS_ROCKSDB) || \ - defined(DATA_STORE_TYPE_ELOQDSS_ELOQSTORE)) -#define ELOQDS 1 -#endif - -// Log state type -#if !defined(LOG_STATE_TYPE_RKDB_CLOUD) - -// Only if LOG_STATE_TYPE_RKDB_CLOUD undefined -#if ((defined(LOG_STATE_TYPE_RKDB_S3) || defined(LOG_STATE_TYPE_RKDB_GCS)) && \ - !defined(LOG_STATE_TYPE_RKDB)) -#define LOG_STATE_TYPE_RKDB_CLOUD 1 -#endif - -#endif - -#if !defined(LOG_STATE_TYPE_RKDB_ALL) - -// Only if LOG_STATE_TYPE_RKDB_ALL undefined -#if (defined(LOG_STATE_TYPE_RKDB_S3) || defined(LOG_STATE_TYPE_RKDB_GCS) || \ - defined(LOG_STATE_TYPE_RKDB)) -#define LOG_STATE_TYPE_RKDB_ALL 1 -#endif - -#endif - -#if defined(DATA_STORE_TYPE_DYNAMODB) || \ - (ROCKSDB_CLOUD_FS_TYPE == ROCKSDB_CLOUD_FS_TYPE_S3) -#include -#endif -#include "metrics.h" -#include "meter.h" -#include "type.h" - -// Forward declaration for INIReader -class INIReader; - -// Forward declarations -namespace txservice -{ -class TxService; -struct NodeConfig; -class CatalogFactory; -class SystemHandler; -namespace store -{ -class DataStoreHandler; -} -} // namespace txservice - -#ifdef ELOQDS -namespace EloqDS -{ -class DataStoreService; -} // namespace EloqDS -#endif - -namespace txlog -{ -class LogServer; -class LogAgent; -} // namespace txlog - -namespace metrics -{ -class MetricsRegistry; -} // namespace metrics - -class DataSubstrate -{ -public: - struct EngineConfig - { - txservice::CatalogFactory *catalog_factory; - txservice::TableEngine engine; - bool enable_engine; - }; - - // Core data substrate configuration - struct CoreConfig - { - std::string data_path; - uint32_t core_num; - bool enable_heap_defragment; - bool enable_wal; - bool enable_data_store; - bool bootstrap; - bool enable_cache_replacement; - bool enable_mvcc; - uint32_t maxclients; - }; - - // Network and cluster configuration - struct NetworkConfig - { - std::string local_ip; - uint32_t local_port; - std::string ip_list; - std::string standby_ip_list; - std::string voter_ip_list; - uint32_t node_group_replica_num; - bool bind_all; - // Parsed service lists - std::unordered_map> - ng_configs; - uint64_t cluster_config_version; - uint32_t node_id; - uint32_t native_ng_id; - std::string cluster_config_file_path; - }; - - struct LogServiceConfig - { - std::vector txlog_ips; - std::vector txlog_ports; - }; - - static std::unique_ptr Initialize( - const std::string &config_file_path); - - // Global initialization function - static bool InitializeGlobal(const std::string &config_file_path); - - // Get global DataSubstrate instance - static DataSubstrate *GetGlobal(); - - // Shutdown DataSubstrate instance - void Shutdown(); - - // Component accessors - txservice::TxService *GetTxService() const - { - return tx_service_.get(); - } - txservice::store::DataStoreHandler *GetStoreHandler() const - { - return store_hd_.get(); - } - txlog::LogServer *GetLogServer() const - { - return log_server_.get(); - } - metrics::MetricsRegistry *GetMetricsRegistry() const - { - return metrics_registry_.get(); - } - // Configuration accessors - const CoreConfig &GetCoreConfig() const - { - return core_config_; - } - const NetworkConfig &GetNetworkConfig() const - { - return network_config_; - } - - ~DataSubstrate(); - - // Constructor (public for static factory method) - DataSubstrate() = default; - -private: - // Private methods that call component initialization files - bool LoadCoreAndNetworkConfig(const INIReader &config_file_reader); - bool InitializeStorageHandler(const INIReader &config_file_reader); - bool InitializeLogService(const INIReader &config_file_reader); - bool InitializeTxService(const INIReader &config_file_reader); - bool InitializeMetrics(const INIReader &config_file_reader); - bool RegisterEngines(); - - // Configuration storage - CoreConfig core_config_; - NetworkConfig network_config_; - LogServiceConfig log_service_config_; - metrics::CommonLabels tx_service_common_labels_{}; - std::vector>> - external_metrics = {}; - // TODO(liunyl): system handler is used to refresh auth related cache. In - // converged db, there should only be one system handler. - txservice::SystemHandler *system_handler_{nullptr}; - - // Component instances - std::unique_ptr tx_service_; -#if defined(DATA_STORE_TYPE_DYNAMODB) || \ - defined(DATA_STORE_TYPE_ELOQDSS_ROCKSDB_CLOUD_S3) || \ - defined(LOG_STATE_TYPE_RKDB_S3) - Aws::SDKOptions aws_options_; -#endif - - std::unique_ptr store_hd_; -#ifdef ELOQDS - std::unique_ptr data_store_service_; -#endif - std::unique_ptr log_server_; - std::unique_ptr metrics_registry_{nullptr}; - - // Engine registry - EngineConfig engines_[NUM_EXTERNAL_ENGINES]; - - std::unordered_map prebuilt_tables_; -}; - -// Helper function to check if a gflag was set from command line -static inline bool CheckCommandLineFlagIsDefault(const char *name) -{ - gflags::CommandLineFlagInfo flag_info; - bool flag_found = gflags::GetCommandLineFlagInfo(name, &flag_info); - // Make sure the flag is declared. - assert(flag_found); - (void) flag_found; - - // Return `true` if the flag has the default value and has not been set - // explicitly from the cmdline or via SetCommandLineOption - return flag_info.is_default; -} \ No newline at end of file diff --git a/data_substrate/core/include/glog_error_logging.h b/data_substrate/core/include/glog_error_logging.h deleted file mode 100644 index 46a8b8c42f7..00000000000 --- a/data_substrate/core/include/glog_error_logging.h +++ /dev/null @@ -1,139 +0,0 @@ -/** - * Copyright (C) 2025 EloqData Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under either of the following two licenses: - * 1. GNU Affero General Public License, version 3, as published by the Free - * Software Foundation. - * 2. GNU General Public License as published by the Free Software - * Foundation; version 2 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License or GNU General Public License for more - * details. - * - * You should have received a copy of the GNU Affero General Public License - * and GNU General Public License V2 along with this program. If not, see - * . - * - */ -#pragma once -#include -#include - -#include -#include -#include -#include - -DECLARE_string(log_file_name_prefix); - -inline void CustomPrefix(std::ostream &s, - const google::LogMessageInfo &l, - void *) -{ - s << "[" // - << std::setw(4) << 1900 + l.time.year() // YY - << '-' // - - << std::setw(2) << 1 + l.time.month() // MM - << '-' // - - << std::setw(2) << l.time.day() // DD - << 'T' // T - << std::setw(2) << l.time.hour() // hh - << ':' // : - << std::setw(2) << l.time.min() // mm - << ':' // : - << std::setw(2) << l.time.sec() // ss - << '.' // . - << std::setfill('0') << std::setw(6) // - << l.time.usec() // usec - << " " << l.severity[0] << " " - << "" << l.thread_id << "] " -#ifndef DISABLE_CODE_LINE_IN_LOG - << "[" << l.filename << ':' << l.line_number << "]"; -#else - ; -#endif -}; - -inline void InitGoogleLogging(char **argv) -{ - // If `GLOG_logtostderr` is specified then log to `stderr` only . - // - // NOTE: This is for cases where disk space needs protection, like when - // deployed in the cloud. - if (FLAGS_logtostderr && FLAGS_log_dir.empty()) - { - FLAGS_alsologtostderr = false; - FLAGS_logtostdout = false; - } - else - { - // Log to `stderr` and `GLOG_log_dir/logfiles`. - // - // NOTE: If `GLOG_log_dir` is not specified then it will be default to - // `path/to/eloqkv/logs` - if (FLAGS_log_dir.empty()) - { - // Get the absolute path of the bin directory - char bin_path[PATH_MAX]; - ssize_t len = readlink("/proc/self/exe", bin_path, PATH_MAX); - std::filesystem::path fullPath(std::string(bin_path, len)); - std::filesystem::path dir_path = - fullPath.parent_path().parent_path(); - FLAGS_log_dir = dir_path.string() + "/logs"; - } - - if (!std::filesystem::exists(FLAGS_log_dir)) - { - std::filesystem::create_directories(FLAGS_log_dir); - } - - // Log to stderr and logfiles - // FLAGS_alsologtostderr = true; - - // NOTE: Enable this will log to `stdout` instead of logfiles. - FLAGS_logtostdout = false; - - // NOTE: Enable this will log to `stderr` instead of logfiles. - FLAGS_logtostderr = false; - - // Log INFO/WARNING/ERROR/FATAL - FLAGS_minloglevel = 0; - - // stderrthreshold (log messages at or above this level are copied to - // stderr in addition to logfiles.) default: 2. - FLAGS_stderrthreshold = google::GLOG_FATAL; - - // Don't buffer anything. NOTE: If `logtostderr` or `logtostdout` is - // `true` then glog will force this value to -1. - FLAGS_logbuflevel = -1; - - FLAGS_log_file_header = false; - - auto log_file_name_prefix = std::getenv("GLOG_log_file_name_prefix"); - FLAGS_log_file_name_prefix = log_file_name_prefix == NULL - ? FLAGS_log_file_name_prefix - : log_file_name_prefix; - auto log_file_prefix = - FLAGS_log_dir + "/" + FLAGS_log_file_name_prefix + "."; - - // Configure log destinations. - google::SetLogDestination(google::INFO, - (log_file_prefix + "INFO.").c_str()); - google::SetLogDestination(google::WARNING, - (log_file_prefix + "WARNING.").c_str()); - google::SetLogDestination(google::ERROR, - (log_file_prefix + "ERROR.").c_str()); - - // Configure symlink for logfiles. - google::SetLogSymlink(google::INFO, FLAGS_log_file_name_prefix.c_str()); - google::SetLogSymlink(google::WARNING, - FLAGS_log_file_name_prefix.c_str()); - google::SetLogSymlink(google::ERROR, - FLAGS_log_file_name_prefix.c_str()); - } - google::InitGoogleLogging(argv[0], &CustomPrefix); -} diff --git a/data_substrate/core/include/mysql_metrics.h b/data_substrate/core/include/mysql_metrics.h deleted file mode 100644 index b7b0f427f9b..00000000000 --- a/data_substrate/core/include/mysql_metrics.h +++ /dev/null @@ -1,81 +0,0 @@ -#pragma once -#include -#include "metrics.h" -#include "meter.h" - -namespace metrics -{ -extern std::unique_ptr mysql_meter; -inline bool enable_mysql_tx_metrics{false}; -inline bool enable_mysql_dml_metrics{false}; -extern Map cmd_to_string_map; - -inline const Name NAME_MYSQL_TX_DURATION{"mysql_tx_duration"}; -inline const Name NAME_MYSQL_PROCESSED_TX_TOTAL{"mysql_processed_tx_total"}; -inline const Name NAME_MYSQL_PROCESSED_QUERY_TOTAL{ - "mysql_processed_query_total"}; -inline const Name NAME_MYSQL_CONNECTION_COUNT{"mysql_connection_count"}; -inline const Name NAME_MAX_CONNECTIONS{"mysql_max_connections"}; -inline const Name NAME_MYSQL_DML_TOTAL{"mysql_dml_total"}; -inline const Name NAME_MYSQL_DML_DURATION{"mysql_dml_duration"}; - -void register_mysql_metrics(MetricsRegistry *metrics_registry, - CommonLabels common_labels= {}); - -#define TRACK_TX_METRICS(thd) \ - if (metrics::enable_mysql_tx_metrics && \ - !(thd)->transaction->tx_registered_) \ - { \ - (thd)->transaction->tx_start_= metrics::Clock::now(); \ - (thd)->transaction->tx_registered_= true; \ - } - -#define COLLECT_TX_METRICS(thd) \ - if (metrics::enable_mysql_tx_metrics && (thd)->transaction->tx_registered_) \ - { \ - (thd)->transaction->tx_registered_= false; \ - metrics::mysql_meter->CollectDuration(metrics::NAME_MYSQL_TX_DURATION, \ - (thd)->transaction->tx_start_); \ - metrics::mysql_meter->Collect(metrics::NAME_MYSQL_PROCESSED_TX_TOTAL, 1); \ - } - -#define IS_MONITORED_DML(dml) \ - ((dml) == SQLCOM_SELECT || \ - (dml) == SQLCOM_INSERT || \ - (dml) == SQLCOM_INSERT_SELECT || \ - (dml) == SQLCOM_UPDATE || \ - (dml) == SQLCOM_UPDATE_MULTI || \ - (dml) == SQLCOM_DELETE || \ - (dml) == SQLCOM_DELETE_MULTI || \ - (dml) == SQLCOM_LOAD) - -/* - Usage: - metrics::TimePoint dml_start; - TRACK_DML_METRICS(dml, dml_start) - */ -#define TRACK_DML_METRICS(dml, dml_start) \ - { \ - if (metrics::enable_mysql_dml_metrics && IS_MONITORED_DML(dml)) \ - { \ - dml_start= metrics::Clock::now(); \ - } \ - } - -/* - Usage: - TRACK_DML_METRICS_DONE(dml, dml_start) - */ -#define TRACK_DML_METRICS_DONE(dml, dml_start) \ - { \ - int dml_type= static_cast(dml); \ - if (metrics::enable_mysql_dml_metrics && IS_MONITORED_DML(dml)) \ - { \ - std::string_view dml_sv= metrics::cmd_to_string_map.at(dml_type); \ - metrics::mysql_meter->CollectDuration(metrics::NAME_MYSQL_DML_DURATION, \ - dml_start, dml_sv); \ - metrics::mysql_meter->Collect(metrics::NAME_MYSQL_DML_TOTAL, 1, \ - dml_sv); \ - } \ - } -} // namespace metrics diff --git a/data_substrate/core/include/redis_metrics.h b/data_substrate/core/include/redis_metrics.h deleted file mode 100644 index 3d928fb3611..00000000000 --- a/data_substrate/core/include/redis_metrics.h +++ /dev/null @@ -1,69 +0,0 @@ -/** - * Copyright (C) 2025 EloqData Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under either of the following two licenses: - * 1. GNU Affero General Public License, version 3, as published by the Free - * Software Foundation. - * 2. GNU General Public License as published by the Free Software - * Foundation; version 2 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License or GNU General Public License for more - * details. - * - * You should have received a copy of the GNU Affero General Public License - * and GNU General Public License V2 along with this program. If not, see - * . - * - */ -#pragma once - -#include -#include -#include - -#include "metrics.h" -#include "meter.h" - -namespace metrics -{ -inline const Name NAME_REDIS_CONNECTION_COUNT{"redis_connection_count"}; -inline const Name NAME_MAX_CONNECTION{"redis_max_connections"}; - -inline const Name NAME_REDIS_COMMAND_TOTAL{"redis_command_total"}; -inline const Name NAME_REDIS_COMMAND_DURATION{"redis_command_duration"}; - -inline const Name NAME_REDIS_COMMAND_AGGREGATED_TOTAL{ - "redis_command_aggregated_total"}; -inline const Name NAME_REDIS_COMMAND_AGGREGATED_DURATION{ - "redis_command_aggregated_duration"}; - -inline const Name NAME_REDIS_SLOW_LOG_LEN{"redis_slow_log_len"}; - -inline size_t collect_redis_command_duration_round{0}; -inline std::unique_ptr redis_meter{nullptr}; - -inline void register_redis_metrics(MetricsRegistry *metrics_registry, - CommonLabels &common_labels, - size_t core_num) -{ - redis_meter = std::make_unique(metrics_registry, common_labels); - redis_meter->Register(metrics::NAME_REDIS_CONNECTION_COUNT, - metrics::Type::Gauge); - redis_meter->Register(metrics::NAME_MAX_CONNECTION, metrics::Type::Gauge); - std::vector labels; - labels.emplace_back("core_id", std::vector()); - for (size_t idx = 0; idx < core_num; ++idx) - { - labels[0].second.push_back(std::to_string(idx)); - } - - redis_meter->Register(metrics::NAME_REDIS_SLOW_LOG_LEN, - metrics::Type::Gauge, - std::move(labels)); - -} -} // namespace metrics diff --git a/data_substrate/core/src/data_substrate.cpp b/data_substrate/core/src/data_substrate.cpp deleted file mode 100644 index cef219f313f..00000000000 --- a/data_substrate/core/src/data_substrate.cpp +++ /dev/null @@ -1,530 +0,0 @@ -/** - * Copyright (C) 2025 EloqData Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under either of the following two licenses: - * 1. GNU Affero General Public License, version 3, as published by the Free - * Software Foundation. - * 2. GNU General Public License as published by the Free Software - * Foundation; version 2 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License or GNU General Public License for more - * details. - * - * You should have received a copy of the GNU Affero General Public License - * and GNU General Public License V2 along with this program. If not, see - * . - * - */ - -#include "data_substrate.h" - -#include -#include - -#include -#include - -#include "INIReader.h" -#include "log_server.h" -#include "store/data_store_handler.h" -#include "tx_service.h" -#include "sequences/sequences.h" - -#ifdef ELOQDS -#include "eloq_data_store_service/data_store_service.h" -#endif - - -// Data Substrate gflags definitions -DEFINE_string(eloq_data_path, "/tmp/eloq_data", "Data substrate data path"); -DEFINE_int32(core_number, 8, "Number of cores for data substrate"); -DEFINE_bool(enable_heap_defragment, false, "Enable heap defragmentation"); -DEFINE_bool(enable_wal, true, "Enable WAL"); -DEFINE_bool(enable_data_store, true, "Enable data store"); -DEFINE_bool(enable_io_uring, false, "Enable io_uring as the IO engine"); -DEFINE_bool(raft_log_async_fsync, false, - "Whether raft log fsync is performed asynchronously (blocking the " - "bthread) instead of blocking the worker thread"); -DEFINE_bool(bootstrap, false, "init system tables and exit"); -DEFINE_bool(enable_cache_replacement, true, "Enable cache replacement"); -DEFINE_bool(enable_mvcc, false, "Enable mvcc"); - -// Network configuration gflags -DEFINE_string(tx_ip, "127.0.0.1", "Local tx service IP address"); -DEFINE_int32(tx_port, 16379, "Local tx service port"); -DEFINE_string(ip_port_list, "", "IP list for cluster"); -DEFINE_string(standby_ip_port_list, "", "Standby IP list"); -DEFINE_string(voter_ip_port_list, "", "Voter IP list"); - -DEFINE_string(cluster_config_file, "", "Cluster configuration file"); -DEFINE_int32(node_group_replica_num, 3, "Node group replica number"); -DEFINE_bool( - bind_all, false, - "Listen on all interfaces if enabled, otherwise listen on local.ip"); -DEFINE_uint32(maxclients, 10000, "maxclients"); -DEFINE_string(log_file_name_prefix, - "eloqdb.log", - "Sets the prefix for log files. Default is 'eloqdb.log'"); -// Declare as weak symbols to allow optional linking -extern txservice::CatalogFactory *eloqsql_catalog_factory __attribute__((weak)); -extern txservice::CatalogFactory *eloqkv_catalog_factory __attribute__((weak)); -extern txservice::SystemHandler *eloqsql_system_handler __attribute__((weak)); - -namespace brpc -{ -DECLARE_int32(event_dispatcher_num); -} -const auto NUM_VCPU= std::thread::hardware_concurrency(); - -// Global DataSubstrate instance -std::unique_ptr g_data_substrate= nullptr; - -std::unique_ptr -DataSubstrate::Initialize(const std::string &config_file_path) -{ - auto data_substrate= std::make_unique(); - - INIReader config_file_reader(config_file_path); - if (!config_file_path.empty() && config_file_reader.ParseError() != 0) - { - LOG(ERROR) << "Failed to parse config file: " << config_file_path; - return nullptr; - } - // Phase 1: Load core and network configuration - if (!data_substrate->LoadCoreAndNetworkConfig(config_file_reader)) - { - return nullptr; - } - - // Phase 2: Register engines - if (!data_substrate->RegisterEngines()) - { - return nullptr; - } - - // Phase 3: Initialize log service - if (!data_substrate->InitializeLogService(config_file_reader)) - { - return nullptr; - } - - // Phase 4: Initialize storage handler - if (data_substrate->GetCoreConfig().enable_data_store && - !data_substrate->InitializeStorageHandler(config_file_reader)) - { - return nullptr; - } - - // Phase 5: Initialize metrics - if (!data_substrate->InitializeMetrics(config_file_reader)) - { - return nullptr; - } - - - // Phase 6: Initialize TxService - if (!data_substrate->InitializeTxService(config_file_reader)) - { - return nullptr; - } - return data_substrate; -} - -bool DataSubstrate::InitializeGlobal(const std::string &config_file_path) -{ - if (g_data_substrate != nullptr) - { - LOG(WARNING) << "Global DataSubstrate already initialized"; - return true; - } - - g_data_substrate= Initialize(config_file_path); - if (g_data_substrate == nullptr) - { - LOG(ERROR) << "Failed to initialize global DataSubstrate"; - return false; - } - - LOG(INFO) << "Global DataSubstrate initialized successfully"; - return true; -} - -DataSubstrate *DataSubstrate::GetGlobal() { return g_data_substrate.get(); } - -DataSubstrate::~DataSubstrate() -{ - // Note: We can't call Shutdown() here because TxService is an incomplete - // type The actual shutdown will be handled by the component that owns the - // TxService For now, just reset the pointers - tx_service_= nullptr; - store_hd_= nullptr; -#ifdef ELOQDS - data_store_service_= nullptr; -#endif - log_server_= nullptr; - metrics_registry_= nullptr; -} - -void DataSubstrate::Shutdown() -{ -#ifdef ELOQ_MODULE_ELOQSQL - system_handler_->Shutdown(); - system_handler_ = nullptr; -#endif - if (tx_service_ != nullptr) - { - LOG(INFO) << "Shutting down the tx service."; - tx_service_->Shutdown(); - LOG(INFO) << "Tx service shut down."; - } - - if (store_hd_ != nullptr) - { - LOG(INFO) << "Shutting down the storage handler."; - store_hd_= nullptr; // Wait for all in-fight requests complete. -#if ELOQDS - if (data_store_service_ != nullptr) - { - data_store_service_= nullptr; - } -#endif - LOG(INFO) << "Storage handler shut down."; - } - -#if (WITH_LOG_SERVICE) - if (log_server_ != nullptr) - { - LOG(INFO) << "Shutting down the internal logservice."; - log_server_= nullptr; - LOG(INFO) << "Internal logservice shut down."; - } -#endif - txservice::Sequences::Destory(); - -#if defined(DATA_STORE_TYPE_DYNAMODB) || \ - defined(DATA_STORE_TYPE_ELOQDSS_ROCKSDB_CLOUD_S3) || \ - defined(LOG_STATE_TYPE_RKDB_S3) - Aws::ShutdownAPI(aws_options_); -#endif - - tx_service_= nullptr; -} - -bool DataSubstrate::RegisterEngines() -{ - engines_[0]= {nullptr, txservice::TableEngine::EloqSql, false}; - engines_[1]= {nullptr, txservice::TableEngine::EloqKv, false}; - engines_[2]= {nullptr, txservice::TableEngine::EloqDoc, false}; - - prebuilt_tables_.try_emplace(txservice::Sequences::table_name_, std::string(txservice::Sequences::table_name_sv_)); -#ifdef ELOQ_MODULE_ELOQSQL - engines_[0].enable_engine= true; - if (eloqsql_catalog_factory) { - engines_[0].catalog_factory= eloqsql_catalog_factory; - } - if (eloqsql_system_handler) { - system_handler_ = eloqsql_system_handler; - } -#endif - -#ifdef ELOQ_MODULE_ELOQKV - engines_[1].enable_engine= true; - if (eloqkv_catalog_factory) { - engines_[1].catalog_factory= eloqkv_catalog_factory; - } - for (uint16_t i= 0; i < 16; i++) - { - std::string table_name("data_table_"); - table_name.append(std::to_string(i)); - txservice::TableName redis_table_name(std::move(table_name), - txservice::TableType::Primary, - txservice::TableEngine::EloqKv); - std::string image= redis_table_name.String(); - prebuilt_tables_.try_emplace(redis_table_name, image); - } -#endif - - return true; -} - -bool DataSubstrate::LoadCoreAndNetworkConfig(const INIReader &config_reader) -{ - // Load core data substrate configuration with gflags priority - core_config_.data_path= - !CheckCommandLineFlagIsDefault("eloq_data_path") - ? FLAGS_eloq_data_path - : config_reader.Get("local", "eloq_data_path", FLAGS_eloq_data_path); - - core_config_.enable_heap_defragment= - !CheckCommandLineFlagIsDefault("enable_heap_defragment") - ? FLAGS_enable_heap_defragment - : config_reader.GetBoolean("local", "enable_heap_defragment", - FLAGS_enable_heap_defragment); - - core_config_.enable_cache_replacement= - !CheckCommandLineFlagIsDefault("enable_cache_replacement") - ? FLAGS_enable_cache_replacement - : config_reader.GetBoolean("local", "enable_cache_replacement", - FLAGS_enable_cache_replacement); - - core_config_.maxclients= - !CheckCommandLineFlagIsDefault("maxclients") - ? FLAGS_maxclients - : config_reader.GetInteger("local", "maxclients", FLAGS_maxclients); - struct rlimit ulimit; - ulimit.rlim_cur= FLAGS_maxclients; - ulimit.rlim_max= FLAGS_maxclients; - if (setrlimit(RLIMIT_NOFILE, &ulimit) == -1) - { - LOG(ERROR) << "Failed to set maxclients."; - return false; - } - core_config_.bootstrap= - !CheckCommandLineFlagIsDefault("bootstrap") - ? FLAGS_bootstrap - : config_reader.GetBoolean("local", "bootstrap", FLAGS_bootstrap); - - core_config_.enable_wal= - !CheckCommandLineFlagIsDefault("enable_wal") - ? FLAGS_enable_wal - : config_reader.GetBoolean("local", "enable_wal", FLAGS_enable_wal); - - core_config_.enable_data_store= - !CheckCommandLineFlagIsDefault("enable_data_store") - ? FLAGS_enable_data_store - : config_reader.GetBoolean("local", "enable_data_store", - FLAGS_enable_data_store); - if (core_config_.enable_wal && !core_config_.enable_data_store) - { - LOG(ERROR) << "When set enable_wal, should also set enable_data_store"; - return false; - } - - if (!core_config_.enable_data_store && core_config_.enable_cache_replacement) - { - LOG(WARNING) << "When set enable_cache_replacement, should also set " - "enable_data_store, reset to false"; - core_config_.enable_cache_replacement= false; - } - - const char *field_core= "core_number"; - core_config_.core_num= FLAGS_core_number; - if (CheckCommandLineFlagIsDefault(field_core)) - { - if (config_reader.HasValue("local", field_core)) - { - core_config_.core_num= config_reader.GetInteger("local", field_core, 0); - assert(core_config_.core_num); - } - else - { - if (!NUM_VCPU) - { - LOG(ERROR) << "config is missing: " << field_core; - return false; - } - const uint min= 1; - if (core_config_.enable_data_store) - { - core_config_.core_num= std::max(min, (NUM_VCPU * 3) / 5); - LOG(INFO) << "give cpus to checkpointer " << core_config_.core_num; - } - else - { - core_config_.core_num= std::max(min, (NUM_VCPU * 7) / 10); - } - LOG(INFO) << "config is automatically set: " << field_core << "=" - << core_config_.core_num << ", vcpu=" << NUM_VCPU; - } - } - - // Load network and cluster configuration with gflags priority - network_config_.local_ip= - !CheckCommandLineFlagIsDefault("tx_ip") - ? FLAGS_tx_ip - : config_reader.Get("local", "tx_ip", FLAGS_tx_ip); - - network_config_.local_port= - !CheckCommandLineFlagIsDefault("tx_port") - ? FLAGS_tx_port - : config_reader.GetInteger("local", "tx_port", FLAGS_tx_port); - std::string local_ip_port= network_config_.local_ip + ":" + - std::to_string(network_config_.local_port); - DLOG(INFO) << "Local tx service ip port: " << local_ip_port; - - network_config_.ip_list= - !CheckCommandLineFlagIsDefault("ip_port_list") - ? FLAGS_ip_port_list - : config_reader.Get("cluster", "ip_port_list", local_ip_port); - - network_config_.standby_ip_list= - !CheckCommandLineFlagIsDefault("standby_ip_port_list") - ? FLAGS_standby_ip_port_list - : config_reader.Get("cluster", "standby_ip_port_list", ""); - - network_config_.voter_ip_list= - !CheckCommandLineFlagIsDefault("voter_ip_port_list") - ? FLAGS_voter_ip_port_list - : config_reader.Get("cluster", "voter_ip_port_list", ""); - - network_config_.cluster_config_file_path= - !CheckCommandLineFlagIsDefault("cluster_config_file") - ? FLAGS_cluster_config_file - : config_reader.Get("cluster", "cluster_config_file", - FLAGS_cluster_config_file); - - network_config_.node_group_replica_num= - !CheckCommandLineFlagIsDefault("node_group_replica_num") - ? FLAGS_node_group_replica_num - : config_reader.GetInteger("cluster", "node_group_replica_num", - FLAGS_node_group_replica_num); - - network_config_.bind_all= - !CheckCommandLineFlagIsDefault("bind_all") - ? FLAGS_bind_all - : config_reader.GetBoolean("local", "bind_all", FLAGS_bind_all); - - core_config_.enable_mvcc= - !CheckCommandLineFlagIsDefault("enable_mvcc") - ? FLAGS_enable_mvcc - : config_reader.GetBoolean("local", "enable_mvcc", - FLAGS_enable_mvcc); - - // Try to read cluster config from file. Cluster config file is written by - // host manager when cluster config updates. - if (network_config_.cluster_config_file_path.empty()) - { - network_config_.cluster_config_file_path= - core_config_.data_path + "/tx_service/cluster_config"; - } - - network_config_.cluster_config_version= 2; - if (!txservice::ReadClusterConfigFile( - network_config_.cluster_config_file_path, network_config_.ng_configs, - network_config_.cluster_config_version)) - { - // Read cluster topology from general config file in this case - auto parse_res= txservice::ParseNgConfig( - network_config_.ip_list, network_config_.standby_ip_list, - network_config_.voter_ip_list, network_config_.ng_configs, - network_config_.node_group_replica_num, 0); - if (!parse_res) - { - LOG(ERROR) << "Failed to extract cluster configs from ip_port_list."; - return false; - } - } - - // print out ng_configs - for (auto &pair : network_config_.ng_configs) - { - DLOG(INFO) << "ng_id: " << pair.first; - for (auto &node : pair.second) - { - DLOG(INFO) << "node_id: " << node.node_id_ - << ", host_name: " << node.host_name_ - << ", port: " << node.port_; - } - } - - // check whether this node is in cluster. - bool found= false; - network_config_.node_id= 0; - network_config_.native_ng_id= 0; - for (auto &pair : network_config_.ng_configs) - { - auto &ng_nodes= pair.second; - for (size_t i= 0; i < ng_nodes.size(); i++) - { - if (ng_nodes[i].host_name_ == network_config_.local_ip && - ng_nodes[i].port_ == network_config_.local_port) - { - network_config_.node_id= ng_nodes[i].node_id_; - found= true; - if (ng_nodes[i].is_candidate_) - { - // found native_ng_id. - network_config_.native_ng_id= pair.first; - break; - } - } - } - } - - if (!found) - { - LOG(ERROR) << "!!!!!!!! Current node does not belong to the " - "cluster, startup is terminated !!!!!!!!"; - return false; - } - const char *field_ed= "event_dispatcher_num"; - uint num_iothreads= brpc::FLAGS_event_dispatcher_num; - if (CheckCommandLineFlagIsDefault(field_ed)) - { - if (config_reader.HasValue("local", field_ed)) - { - num_iothreads= config_reader.GetInteger("local", field_ed, 0); - assert(num_iothreads); - } - else - { - if (!NUM_VCPU) - { - LOG(ERROR) << "config is missing: " << field_ed; - return false; - } - num_iothreads= std::max(uint(1), (core_config_.core_num + 5) / 6); - LOG(INFO) << "config is automatically set: " << field_ed << "=" - << num_iothreads << ", vcpu=" << NUM_VCPU; - } - } - GFLAGS_NAMESPACE::SetCommandLineOption( - field_ed, std::to_string(num_iothreads).c_str()); - - // Set bthread concurrency - GFLAGS_NAMESPACE::SetCommandLineOption( - "bthread_concurrency", std::to_string(core_config_.core_num).c_str()); -#ifdef ELOQ_MODULE_ENABLED - GFLAGS_NAMESPACE::SetCommandLineOption("worker_polling_time_us", "1000"); - GFLAGS_NAMESPACE::SetCommandLineOption("brpc_worker_as_ext_processor", - "true"); -#endif - GFLAGS_NAMESPACE::SetCommandLineOption("use_pthread_event_dispatcher", - "true"); - GFLAGS_NAMESPACE::SetCommandLineOption("max_body_size", "536870912"); - GFLAGS_NAMESPACE::SetCommandLineOption("graceful_quit_on_sigterm", "true"); - - bool enable_io_uring= - !CheckCommandLineFlagIsDefault("enable_io_uring") - ? FLAGS_enable_io_uring - : config_reader.GetBoolean("local", "enable_io_uring", - FLAGS_enable_io_uring); - - bool raft_log_async_fsync= - !CheckCommandLineFlagIsDefault("raft_log_async_fsync") - ? FLAGS_raft_log_async_fsync - : config_reader.GetBoolean("local", "raft_log_async_fsync", - FLAGS_raft_log_async_fsync); - if (raft_log_async_fsync && !enable_io_uring) - { - LOG(ERROR) << "Invalid config: when set `enable_io_uring`, " - "should also set `enable_io_uring`."; - return false; - } - GFLAGS_NAMESPACE::SetCommandLineOption("use_io_uring", - enable_io_uring ? "true" : "false"); - GFLAGS_NAMESPACE::SetCommandLineOption( - "raft_use_bthread_fsync", raft_log_async_fsync ? "true" : "false"); -#if defined(DATA_STORE_TYPE_DYNAMODB) || \ - defined(DATA_STORE_TYPE_ELOQDSS_ROCKSDB_CLOUD_S3) || \ - defined(LOG_STATE_TYPE_RKDB_S3) - aws_options_.loggingOptions.logLevel= Aws::Utils::Logging::LogLevel::Info; - Aws::InitAPI(aws_options_); -#endif - - return true; -} \ No newline at end of file diff --git a/data_substrate/core/src/log_init.cpp b/data_substrate/core/src/log_init.cpp deleted file mode 100644 index cf45f43ca50..00000000000 --- a/data_substrate/core/src/log_init.cpp +++ /dev/null @@ -1,541 +0,0 @@ -#include "INIReader.h" -#include "data_substrate.h" -#include "log_server.h" -#include "log_utils.h" -#include "sharder.h" -#if defined(LOG_STATE_TYPE_RKDB_CLOUD) -#include "rocksdb_cloud_config.h" -#endif - -DEFINE_string(txlog_service_list, "", "Log group servers configuration"); -DEFINE_int32(txlog_group_replica_num, 3, "Replica number of one log group"); -DEFINE_string(log_service_data_path, "", "path for log_service data"); - -DEFINE_string(txlog_rocksdb_storage_path, - "", - "Storage path for txlog rocksdb state"); -DEFINE_uint32(txlog_rocksdb_max_write_buffer_number, - 8, - "Max write buffer number"); -DEFINE_string(txlog_rocksdb_sst_files_size_limit, - "500MB", - "The total RocksDB sst files size before purge"); -DEFINE_uint32(txlog_rocksdb_scan_threads, - 1, - "The number of rocksdb scan threads"); - -DEFINE_uint32(txlog_rocksdb_max_background_jobs, 12, "Max background jobs"); -DEFINE_string(txlog_rocksdb_target_file_size_base, - "64MB", - "Target file size base for rocksdb"); - -DEFINE_uint32(logserver_snapshot_interval, 600, "logserver_snapshot interval"); - -DEFINE_bool(enable_txlog_request_checkpoint, - true, - "Enable txlog server sending checkpoint requests when the criteria " - "are met."); - -#if defined(LOG_STATE_TYPE_RKDB_CLOUD) -DEFINE_string(txlog_rocksdb_cloud_region, - "ap-northeast-1", - "Cloud service regin"); -DEFINE_string(txlog_rocksdb_cloud_bucket_name, - "rocksdb-cloud-test", - "Cloud storage bucket name"); -DEFINE_string(txlog_rocksdb_cloud_bucket_prefix, - "txlog-", - "Cloud storage bucket prefix"); -DEFINE_string(txlog_rocksdb_cloud_object_path, - "eloqkv_txlog", - "Cloud storage object path, if not set, will use bucket name and " - "prefix"); -DEFINE_uint32( - txlog_rocksdb_cloud_ready_timeout, - 10, - "Timeout before rocksdb cloud becoming ready on new log group leader"); -DEFINE_uint32(txlog_rocksdb_cloud_file_deletion_delay, - 3600, - "The file deletion delay for rocksdb cloud file"); -DEFINE_uint32(txlog_rocksdb_cloud_log_retention_days, - 90, - "The number of days for which logs should be retained"); -DEFINE_string(txlog_rocksdb_cloud_log_purger_schedule, - "00:00:01", - "Time (in regular format: HH:MM:SS) to run log purger daily, " - "deleting logs older than log_retention_days."); -DEFINE_uint32(txlog_in_mem_data_log_queue_size_high_watermark, - 50 * 10000, - "In memory data log queue max size"); -DEFINE_string(txlog_rocksdb_cloud_endpoint_url, - "", - "Endpoint url of cloud storage service"); -DEFINE_string(txlog_rocksdb_cloud_sst_file_cache_size, - "1GB", - "Local sst cache size for txlog"); -DEFINE_uint32(txlog_rocksdb_cloud_sst_file_cache_num_shard_bits, - 5, - "TxLog RocksDB Cloud SST file cache num shard bits"); -#endif -#ifdef WITH_CLOUD_AZ_INFO -DEFINE_string(txlog_rocksdb_cloud_prefer_zone, - "", - "user preferred deployed availability zone"); -DEFINE_string(txlog_rocksdb_cloud_current_zone, - "", - "the log service server node deployed on currently"); -#endif - -DEFINE_uint32(check_replay_log_size_interval_sec, - 10, - "The interval for checking replay log size."); - -DEFINE_string(notify_checkpointer_threshold_size, - "1GB", - "When the replay log size reaches this threshold the txlog " - "server sends a checkpoint request to tx_service."); - - -bool DataSubstrate::InitializeLogService(const INIReader &config_reader) -{ - std::string txlog_service = - !CheckCommandLineFlagIsDefault("txlog_service_list") - ? FLAGS_txlog_service_list - : config_reader.GetString( - "cluster", "txlog_service_list", FLAGS_txlog_service_list); - int txlog_group_replica_num = - !CheckCommandLineFlagIsDefault("txlog_group_replica_num") - ? FLAGS_txlog_group_replica_num - : config_reader.GetInteger("cluster", - "txlog_group_replica_num", - FLAGS_txlog_group_replica_num); - - std::string log_service_data_path = - !CheckCommandLineFlagIsDefault("log_service_data_path") - ? FLAGS_log_service_data_path - : config_reader.GetString("local", "log_service_data_path", ""); - std::string log_path("local://"); - if (log_service_data_path.empty()) - { - log_path.append(core_config_.data_path); - } - else - { - log_path.append(log_service_data_path); - } - log_path.append("/log_service"); - // parse standalone txlog_service_list - bool is_standalone_txlog_service = false; - std::vector &txlog_ips = log_service_config_.txlog_ips; - std::vector &txlog_ports = log_service_config_.txlog_ports; - if (!txlog_service.empty()) - { - is_standalone_txlog_service = true; - std::string token; - std::istringstream txlog_ip_port_list_stream(txlog_service); - while (std::getline(txlog_ip_port_list_stream, token, ',')) - { - size_t c_idx = token.find_first_of(':'); - if (c_idx != std::string::npos) - { - txlog_ips.emplace_back(token.substr(0, c_idx)); - uint16_t pt = std::stoi(token.substr(c_idx + 1)); - txlog_ports.emplace_back(pt); - } - } - } -#if defined(WITH_LOG_SERVICE) - else - { - uint32_t txlog_node_id = 0; - uint32_t next_txlog_node_id = 0; - uint16_t log_server_port = network_config_.local_port + 2; - std::unordered_set tx_node_ids; - for (uint32_t ng = 0; ng < network_config_.ng_configs.size(); ng++) - { - for (uint32_t nidx = 0; - nidx < network_config_.ng_configs[ng].size(); - nidx++) - { - if (tx_node_ids.count( - network_config_.ng_configs[ng][nidx].node_id_) == 0) - { - tx_node_ids.insert( - network_config_.ng_configs[ng][nidx].node_id_); - if (network_config_.ng_configs[ng][nidx].port_ == - network_config_.local_port && - network_config_.ng_configs[ng][nidx].host_name_ == - network_config_.local_ip) - { - // is local node, set txlog_node_id - txlog_node_id = next_txlog_node_id; - } - next_txlog_node_id++; - txlog_ports.emplace_back( - network_config_.ng_configs[ng][nidx].port_ + 2); - txlog_ips.emplace_back( - network_config_.ng_configs[ng][nidx].host_name_); - } - } - } - - // Init local txlog service if it is not standalone txlog service - // Init before store handler so they can init in parallel - if (!is_standalone_txlog_service) - { - if (txlog_ips.empty() || txlog_ports.empty()) - { - LOG(ERROR) - << "WAL is enabled but `txlog_service_list` is empty and " - "built-in log server initialization failed, " - "unable to proceed."; - return false; - } - - [[maybe_unused]] size_t txlog_rocksdb_scan_threads = - !CheckCommandLineFlagIsDefault("txlog_rocksdb_scan_threads") - ? FLAGS_txlog_rocksdb_scan_threads - : config_reader.GetInteger( - "local", - "txlog_rocksdb_scan_threads", - FLAGS_txlog_rocksdb_scan_threads); - - size_t txlog_rocksdb_max_write_buffer_number = - !CheckCommandLineFlagIsDefault( - "txlog_rocksdb_max_write_buffer_number") - ? FLAGS_txlog_rocksdb_max_write_buffer_number - : config_reader.GetInteger( - "local", - "txlog_rocksdb_max_write_buffer_number", - FLAGS_txlog_rocksdb_max_write_buffer_number); - - size_t txlog_rocksdb_max_background_jobs = - !CheckCommandLineFlagIsDefault( - "txlog_rocksdb_max_background_jobs") - ? FLAGS_txlog_rocksdb_max_background_jobs - : config_reader.GetInteger( - "local", - "txlog_rocksdb_max_background_jobs", - FLAGS_txlog_rocksdb_max_background_jobs); - - size_t txlog_rocksdb_target_file_size_base_val = - !CheckCommandLineFlagIsDefault( - "txlog_rocksdb_target_file_size_base") - ? txlog::parse_size( - FLAGS_txlog_rocksdb_target_file_size_base) - : txlog::parse_size(config_reader.GetString( - "local", - "txlog_rocksdb_target_file_size_base", - FLAGS_txlog_rocksdb_target_file_size_base)); - - [[maybe_unused]] size_t logserver_snapshot_interval = - !CheckCommandLineFlagIsDefault("logserver_snapshot_interval") - ? FLAGS_logserver_snapshot_interval - : config_reader.GetInteger( - "local", - "logserver_snapshot_interval", - FLAGS_logserver_snapshot_interval); - - [[maybe_unused]] bool enable_txlog_request_checkpoint = - !CheckCommandLineFlagIsDefault( - "enable_txlog_request_checkpoint") - ? FLAGS_enable_txlog_request_checkpoint - : config_reader.GetBoolean( - "local", - "enable_txlog_request_checkpoint", - FLAGS_enable_txlog_request_checkpoint); - - [[maybe_unused]] size_t check_replay_log_size_interval_sec = - !CheckCommandLineFlagIsDefault( - "check_replay_log_size_interval_sec") - ? FLAGS_check_replay_log_size_interval_sec - : config_reader.GetInteger( - "local", - "check_replay_log_size_interval_sec", - FLAGS_check_replay_log_size_interval_sec); - - [[maybe_unused]] size_t notify_checkpointer_threshold_size_val = - !CheckCommandLineFlagIsDefault( - "notify_checkpointer_threshold_size") - ? txlog::parse_size( - FLAGS_notify_checkpointer_threshold_size) - : txlog::parse_size(config_reader.GetString( - "local", - "notify_checkpointer_threshold_size", - FLAGS_notify_checkpointer_threshold_size)); - -#if defined(LOG_STATE_TYPE_RKDB_ALL) - std::string txlog_rocksdb_storage_path = - !CheckCommandLineFlagIsDefault("txlog_rocksdb_storage_path") - ? FLAGS_txlog_rocksdb_storage_path - : config_reader.GetString( - "local", "txlog_rocksdb_storage_path", ""); - if (txlog_rocksdb_storage_path.empty()) - { - // remove "local://" prefix from log_path - txlog_rocksdb_storage_path = log_path.substr(8) + "/rocksdb"; - } - -#if defined(LOG_STATE_TYPE_RKDB_CLOUD) - txlog::RocksDBCloudConfig txlog_rocksdb_cloud_config; -#if defined(LOG_STATE_TYPE_RKDB_S3) - txlog_rocksdb_cloud_config.aws_access_key_id_ = - !CheckCommandLineFlagIsDefault("aws_access_key_id") - ? FLAGS_aws_access_key_id - : config_reader.GetString("store", "aws_access_key_id", ""); - txlog_rocksdb_cloud_config.aws_secret_key_ = - !CheckCommandLineFlagIsDefault("aws_secret_key") - ? FLAGS_aws_secret_key - : config_reader.GetString("store", "aws_secret_key", ""); -#endif /* LOG_STATE_TYPE_RKDB_S3 */ - txlog_rocksdb_cloud_config.endpoint_url_ = - !CheckCommandLineFlagIsDefault( - "txlog_rocksdb_cloud_endpoint_url") - ? FLAGS_txlog_rocksdb_cloud_endpoint_url - : config_reader.GetString( - "local", - "txlog_rocksdb_cloud_endpoint_url", - FLAGS_txlog_rocksdb_cloud_endpoint_url); - txlog_rocksdb_cloud_config.bucket_name_ = - !CheckCommandLineFlagIsDefault( - "txlog_rocksdb_cloud_bucket_name") - ? FLAGS_txlog_rocksdb_cloud_bucket_name - : config_reader.GetString( - "local", - "txlog_rocksdb_cloud_bucket_name", - FLAGS_txlog_rocksdb_cloud_bucket_name); - txlog_rocksdb_cloud_config.bucket_prefix_ = - !CheckCommandLineFlagIsDefault( - "txlog_rocksdb_cloud_bucket_prefix") - ? FLAGS_txlog_rocksdb_cloud_bucket_prefix - : config_reader.GetString( - "local", - "txlog_rocksdb_cloud_bucket_prefix", - FLAGS_txlog_rocksdb_cloud_bucket_prefix); - txlog_rocksdb_cloud_config.object_path_ = - !CheckCommandLineFlagIsDefault( - "txlog_rocksdb_cloud_object_path") - ? FLAGS_txlog_rocksdb_cloud_object_path - : config_reader.GetString( - "local", - "txlog_rocksdb_cloud_object_path", - FLAGS_txlog_rocksdb_cloud_object_path); - txlog_rocksdb_cloud_config.region_ = - !CheckCommandLineFlagIsDefault("txlog_rocksdb_cloud_region") - ? FLAGS_txlog_rocksdb_cloud_region - : config_reader.GetString("local", - "txlog_rocksdb_cloud_region", - FLAGS_txlog_rocksdb_cloud_region); - uint32_t db_ready_timeout_us = - !CheckCommandLineFlagIsDefault( - "txlog_rocksdb_cloud_ready_timeout") - ? FLAGS_txlog_rocksdb_cloud_ready_timeout - : config_reader.GetInteger( - "local", - "txlog_rocksdb_cloud_ready_timeout", - FLAGS_txlog_rocksdb_cloud_ready_timeout); - txlog_rocksdb_cloud_config.db_ready_timeout_us_ = - db_ready_timeout_us * 1000 * 1000; - txlog_rocksdb_cloud_config.db_file_deletion_delay_ = - !CheckCommandLineFlagIsDefault( - "txlog_rocksdb_cloud_file_deletion_delay") - ? FLAGS_txlog_rocksdb_cloud_file_deletion_delay - : config_reader.GetInteger( - "local", - "txlog_rocksdb_cloud_file_deletion_delay", - FLAGS_txlog_rocksdb_cloud_file_deletion_delay); - txlog_rocksdb_cloud_config.log_retention_days_ = - !CheckCommandLineFlagIsDefault( - "txlog_rocksdb_cloud_log_retention_days") - ? FLAGS_txlog_rocksdb_cloud_log_retention_days - : config_reader.GetInteger( - "local", - "txlog_rocksdb_cloud_log_retention_days", - FLAGS_txlog_rocksdb_cloud_log_retention_days); - txlog_rocksdb_cloud_config.sst_file_cache_num_shard_bits_ = - !CheckCommandLineFlagIsDefault( - "txlog_rocksdb_cloud_sst_file_cache_num_shard_bits") - ? FLAGS_txlog_rocksdb_cloud_sst_file_cache_num_shard_bits - : config_reader.GetInteger( - "local", - "txlog_rocksdb_cloud_sst_file_cache_num_shard_bits", - FLAGS_txlog_rocksdb_cloud_sst_file_cache_num_shard_bits); - txlog_rocksdb_cloud_config.sst_file_cache_size_ = - !CheckCommandLineFlagIsDefault( - "txlog_rocksdb_cloud_sst_file_cache_size") - ? txlog::parse_size( - FLAGS_txlog_rocksdb_cloud_sst_file_cache_size) - : txlog::parse_size(config_reader.GetString( - "local", - "txlog_rocksdb_cloud_sst_file_cache_size", - FLAGS_txlog_rocksdb_cloud_sst_file_cache_size)); - std::tm log_purger_tm{}; - std::string log_purger_schedule = - !CheckCommandLineFlagIsDefault( - "txlog_rocksdb_cloud_log_purger_schedule") - ? FLAGS_txlog_rocksdb_cloud_log_purger_schedule - : config_reader.GetString( - "local", - "txlog_rocksdb_cloud_log_purger_schedule", - FLAGS_txlog_rocksdb_cloud_log_purger_schedule); - std::istringstream iss(log_purger_schedule); - iss >> std::get_time(&log_purger_tm, "%H:%M:%S"); - - if (iss.fail()) - { - LOG(ERROR) << "Invalid time format." << std::endl; - } - else - { - txlog_rocksdb_cloud_config.log_purger_starting_hour_ = - log_purger_tm.tm_hour; - txlog_rocksdb_cloud_config.log_purger_starting_minute_ = - log_purger_tm.tm_min; - txlog_rocksdb_cloud_config.log_purger_starting_second_ = - log_purger_tm.tm_sec; - } - if (core_config_.bootstrap) - { - log_server_ = std::make_unique<::txlog::LogServer>( - txlog_node_id, - log_server_port, - txlog_ips, - txlog_ports, - log_path, - 0, - txlog_group_replica_num, -#ifdef WITH_CLOUD_AZ_INFO - FLAGS_txlog_rocksdb_cloud_prefer_zone, - FLAGS_txlog_rocksdb_cloud_current_zone, -#endif - txlog_rocksdb_storage_path, - txlog_rocksdb_scan_threads, - txlog_rocksdb_cloud_config, - FLAGS_txlog_in_mem_data_log_queue_size_high_watermark, - txlog_rocksdb_max_write_buffer_number, - txlog_rocksdb_max_background_jobs, - txlog_rocksdb_target_file_size_base_val, - logserver_snapshot_interval); - } - else - { - log_server_ = std::make_unique<::txlog::LogServer>( - txlog_node_id, - log_server_port, - txlog_ips, - txlog_ports, - log_path, - 0, - txlog_group_replica_num, -#ifdef WITH_CLOUD_AZ_INFO - FLAGS_txlog_rocksdb_cloud_prefer_zone, - FLAGS_txlog_rocksdb_cloud_current_zone, -#endif - txlog_rocksdb_storage_path, - txlog_rocksdb_scan_threads, - txlog_rocksdb_cloud_config, - FLAGS_txlog_in_mem_data_log_queue_size_high_watermark, - txlog_rocksdb_max_write_buffer_number, - txlog_rocksdb_max_background_jobs, - txlog_rocksdb_target_file_size_base_val, - logserver_snapshot_interval, - enable_txlog_request_checkpoint, - check_replay_log_size_interval_sec, - notify_checkpointer_threshold_size_val); - } -#else - size_t txlog_rocksdb_sst_files_size_limit_val = - !CheckCommandLineFlagIsDefault( - "txlog_rocksdb_sst_files_size_limit") - ? txlog::parse_size( - FLAGS_txlog_rocksdb_sst_files_size_limit) - : txlog::parse_size(config_reader.GetString( - "local", - "txlog_rocksdb_sst_files_size_limit", - FLAGS_txlog_rocksdb_sst_files_size_limit)); - - // Start internal logserver. -#if defined(OPEN_LOG_SERVICE) - if (core_config_.bootstrap) - { - log_server_ = std::make_unique<::txlog::LogServer>( - txlog_node_id, - log_server_port, - log_path, - 1, - txlog_rocksdb_sst_files_size_limit_val, - txlog_rocksdb_max_write_buffer_number, - txlog_rocksdb_max_background_jobs, - txlog_rocksdb_target_file_size_base_val); - } - else - { - log_server_ = std::make_unique<::txlog::LogServer>( - txlog_node_id, - log_server_port, - log_path, - 1, - txlog_rocksdb_sst_files_size_limit_val, - txlog_rocksdb_max_write_buffer_number, - txlog_rocksdb_max_background_jobs, - txlog_rocksdb_target_file_size_base_val); - } -#else - if (core_config_.bootstrap) - { - log_server_ = std::make_unique<::txlog::LogServer>( - txlog_node_id, - log_server_port, - txlog_ips, - txlog_ports, - log_path, - 0, - txlog_group_replica_num, - txlog_rocksdb_storage_path, - txlog_rocksdb_scan_threads, - txlog_rocksdb_sst_files_size_limit_val, - txlog_rocksdb_max_write_buffer_number, - txlog_rocksdb_max_background_jobs, - txlog_rocksdb_target_file_size_base_val, - logserver_snapshot_interval); - } - else - { - log_server_ = std::make_unique<::txlog::LogServer>( - txlog_node_id, - log_server_port, - txlog_ips, - txlog_ports, - log_path, - 0, - txlog_group_replica_num, - txlog_rocksdb_storage_path, - txlog_rocksdb_scan_threads, - txlog_rocksdb_sst_files_size_limit_val, - txlog_rocksdb_max_write_buffer_number, - txlog_rocksdb_max_background_jobs, - txlog_rocksdb_target_file_size_base_val, - logserver_snapshot_interval, - enable_txlog_request_checkpoint, - check_replay_log_size_interval_sec, - notify_checkpointer_threshold_size_val); - } -#endif -#endif -#endif - DLOG(INFO) << "Log server started, node_id: " << txlog_node_id - << ", log_server_port: " << log_server_port - << ", txlog_group_replica_num: " - << txlog_group_replica_num << ", log_path: " << log_path; - int err = log_server_->Start(); - - if (err != 0) - { - LOG(ERROR) << "Failed to start the log service."; - return false; - } - } - } -#endif - return true; -} diff --git a/data_substrate/core/src/metrics_init.cpp b/data_substrate/core/src/metrics_init.cpp deleted file mode 100644 index 713ef064f32..00000000000 --- a/data_substrate/core/src/metrics_init.cpp +++ /dev/null @@ -1,216 +0,0 @@ -#include "INIReader.h" -#include "data_substrate.h" -#include "gflags/gflags.h" -#include "glog/logging.h" -#include "metrics.h" -#include "metrics_registry_impl.h" -#include "store/data_store_handler.h" -#include "tx_service_metrics.h" - -#if (WITH_LOG_SERVICE) -#include "log_service_metrics.h" -#endif -#ifdef ELOQ_MODULE_ELOQSQL -#include "mysql_metrics.h" //TDOO(liunyl): move this file to correct location -#endif -#ifdef ELOQ_MODULE_ELOQKV -#include "redis_metrics.h" -#endif - -DEFINE_bool(enable_metrics, false, "Enable metrics"); -DEFINE_string(metrics_port, "18081", "Metrics port"); -namespace EloqKV -{ -enum struct RedisCommandType; -extern const std::vector> - command_types; -} // namespace EloqKV - -bool DataSubstrate::InitializeMetrics(const INIReader &config_reader) -{ - /* Parse metrics config */ - metrics::enable_metrics = - config_reader.GetBoolean("metrics", "enable_metrics", false); - DLOG(INFO) << "enable_metrics: " - << (metrics::enable_metrics ? "ON" : "OFF"); - if (metrics::enable_metrics) - { - FLAGS_metrics_port = std::to_string(config_reader.GetInteger( - "metrics", "metrics_port", std::stoi(FLAGS_metrics_port))); - - LOG(INFO) << "metrics_port: " << FLAGS_metrics_port; - - // global metrics - metrics::enable_memory_usage = - config_reader.GetBoolean("metrics", "enable_memory_usage", true); - LOG(INFO) << "enable_memory_usage: " - << (metrics::enable_memory_usage ? "ON" : "OFF"); - if (metrics::enable_memory_usage) - { - metrics::collect_memory_usage_round = config_reader.GetInteger( - "metrics", "collect_memory_usage_round", 10000); - LOG(INFO) << "collect memory usage every " - << metrics::collect_memory_usage_round << " round(s)"; - } - - metrics::enable_cache_hit_rate = - config_reader.GetBoolean("metrics", "enable_cache_hit_rate", true); - LOG(INFO) << "enable_cache_hit_rate: " - << (metrics::enable_cache_hit_rate ? "ON" : "OFF"); - - // tx metrics - metrics::enable_tx_metrics = - config_reader.GetBoolean("metrics", "enable_tx_metrics", true); - LOG(INFO) << "enable_tx_metrics: " - << (metrics::enable_tx_metrics ? "ON" : "OFF"); - if (metrics::enable_tx_metrics) - { - metrics::collect_tx_duration_round = - config_reader.GetInteger("metrics", - "collect_tx_duration_round", - metrics::collect_tx_duration_round); - LOG(INFO) << "collect tx duration every " - << metrics::collect_tx_duration_round << " round(s)"; - } - - // busy round metrics - metrics::enable_busy_round_metrics = config_reader.GetBoolean( - "metrics", "enable_busy_round_metrics", true); - LOG(INFO) << "enable_busy_round_metrics: " - << (metrics::enable_busy_round_metrics ? "ON" : "OFF"); - if (metrics::enable_busy_round_metrics) - { - metrics::busy_round_threshold = - config_reader.GetInteger("metrics", "busy_round_threshold", 10); - LOG(INFO) << "collect busy round metrics when reaching the " - "busy round " - "threshold " - << metrics::busy_round_threshold; - } - - // remote request metrics - metrics::enable_remote_request_metrics = config_reader.GetBoolean( - "metrics", "enable_busy_round_metrics", metrics::enable_tx_metrics); - LOG(INFO) << "enable_remote_request_metrics: " - << (metrics::enable_remote_request_metrics ? "ON" : "OFF"); - if (core_config_.enable_data_store) - { - metrics::enable_kv_metrics = - config_reader.GetBoolean("metrics", "enable_kv_metrics", true); - LOG(INFO) << "enable_kv_metrics: " - << (metrics::enable_kv_metrics ? "ON" : "OFF"); - } - -#if (WITH_LOG_SERVICE) - if (core_config_.enable_wal) - { - // log_service metrics - metrics::enable_log_service_metrics = config_reader.GetBoolean( - "metrics", "enable_log_service_metrics", true); - LOG(INFO) << "enable_log_service_metrics: " - << (metrics::enable_log_service_metrics ? "ON" : "OFF"); - } -#endif - - // failed forward msgs metrics - metrics::enable_standby_metrics = - config_reader.GetBoolean("metrics", "enable_standby_metrics", true); - LOG(INFO) << "enable standby metrics: " - << (metrics::enable_standby_metrics ? "ON" : "OFF"); - if (metrics::enable_standby_metrics) - { - metrics::collect_standby_metrics_round = config_reader.GetInteger( - "metrics", "collect_standby_metrics_round", 10000); - LOG(INFO) << "collect standby metrics every " - << metrics::collect_standby_metrics_round << " round(s)"; - } -#ifdef ELOQ_MODULE_ELOQSQL - metrics::enable_mysql_tx_metrics = config_reader.GetBoolean( - "metrics", "enable_mysql_tx_metrics", true); - metrics::enable_mysql_dml_metrics = config_reader.GetBoolean( - "metrics", "enable_mysql_dml_metrics", true); -#endif - setenv("ELOQ_METRICS_PORT", FLAGS_metrics_port.c_str(), false); - eloq_metrics_app::MetricsRegistryImpl::MetricsRegistryResult - metrics_registry_result = - eloq_metrics_app::MetricsRegistryImpl::GetRegistry(); - - if (metrics_registry_result.not_ok_ != nullptr) - { - LOG(ERROR) - << "!!!!!!!! Failed to initialize MetricsRegristry !!!!!!!!"; - return false; - } - - metrics_registry_ = - std::move(metrics_registry_result.metrics_registry_); - - if (core_config_.enable_data_store) - { - metrics::CommonLabels kv_common_common_labels{}; - kv_common_common_labels["node_ip"] = network_config_.local_ip; - kv_common_common_labels["node_port"] = - std::to_string(network_config_.local_port); - store_hd_->RegisterKvMetrics(metrics_registry_.get(), - kv_common_common_labels); - } - -#ifdef ELOQ_MODULE_ELOQSQL - metrics::CommonLabels mysql_common_labels{}; - mysql_common_labels["node_ip"] = network_config_.local_ip; - mysql_common_labels["node_port"] = - std::to_string(network_config_.local_port); - metrics::register_mysql_metrics(metrics_registry_.get(), - mysql_common_labels); -#endif - -#ifdef ELOQ_MODULE_ELOQKV - metrics::CommonLabels redis_common_labels{}; - redis_common_labels["node_ip"] = network_config_.local_ip; - redis_common_labels["node_port"] = - std::to_string(network_config_.local_port); - redis_common_labels["node_id"] = - std::to_string(network_config_.node_id); - metrics::register_redis_metrics(metrics_registry_.get(), - redis_common_labels, - core_config_.core_num); - metrics::redis_meter->Collect( - metrics::NAME_MAX_CONNECTION, - DataSubstrate::GetGlobal()->GetCoreConfig().maxclients); - for (const auto &[cmd, _] : EloqKV::command_types) - { - std::vector label_groups = {{"type", {cmd}}}; - - external_metrics.push_back( - std::make_tuple(metrics::NAME_REDIS_COMMAND_DURATION, - metrics::Type::Histogram, - label_groups)); - external_metrics.push_back( - std::make_tuple(metrics::NAME_REDIS_COMMAND_TOTAL, - metrics::Type::Counter, - label_groups)); - } - for (const auto &access_type : {"read", "write"}) - { - external_metrics.push_back( - std::make_tuple(metrics::NAME_REDIS_COMMAND_AGGREGATED_TOTAL, - metrics::Type::Counter, - std::vector{ - {"access_type", {access_type}}})); - external_metrics.push_back( - std::make_tuple(metrics::NAME_REDIS_COMMAND_AGGREGATED_DURATION, - metrics::Type::Histogram, - std::vector{ - {"access_type", {access_type}}})); - } -#endif - - tx_service_common_labels_["node_ip"] = network_config_.local_ip; - tx_service_common_labels_["node_port"] = - std::to_string(network_config_.local_port); - tx_service_common_labels_["node_id"] = - std::to_string(network_config_.node_id); - } - - return true; -} \ No newline at end of file diff --git a/data_substrate/core/src/storage_init.cpp b/data_substrate/core/src/storage_init.cpp deleted file mode 100644 index d33ec1a5854..00000000000 --- a/data_substrate/core/src/storage_init.cpp +++ /dev/null @@ -1,318 +0,0 @@ -/** - * Copyright (C) 2025 EloqData Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under either of the following two licenses: - * 1. GNU Affero General Public License, version 3, as published by the Free - * Software Foundation. - * 2. GNU General Public License as published by the Free Software - * Foundation; version 2 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License or GNU General Public License for more - * details. - * - * You should have received a copy of the GNU Affero General Public License - * and GNU General Public License V2 along with this program. If not, see - * . - * - */ -#include - -#include "INIReader.h" -#include "data_store_handler.h" -#include "data_substrate.h" -#include "kv_store.h" -#if ELOQDS -#include - -#include "eloq_data_store_service/data_store_service.h" -#include "eloq_data_store_service/data_store_service_config.h" -#include "data_store_service_client.h" -#endif -#if defined(DATA_STORE_TYPE_ROCKSDB) -#include "store_handler/rocksdb_handler.h" -#endif - -#if defined(DATA_STORE_TYPE_ELOQDSS_ROCKSDB_CLOUD_S3) || \ - defined(DATA_STORE_TYPE_ELOQDSS_ROCKSDB_CLOUD_GCS) -#include "eloq_data_store_service/rocksdb_cloud_data_store_factory.h" -#elif defined(DATA_STORE_TYPE_ELOQDSS_ROCKSDB) -#include "eloq_data_store_service/rocksdb_data_store_factory.h" -#elif defined(DATA_STORE_TYPE_ELOQDSS_ELOQSTORE) -#include "eloq_data_store_service/eloq_store_data_store_factory.h" -#endif - -#ifdef DATA_STORE_TYPE_DYNAMODB -#include "store_handler/dynamo_handler.h" -#endif - -#ifdef DATA_STORE_TYPE_BIGTABLE -#include "store_handler/bigtable_handler.h" -#endif - -#if defined(DATA_STORE_TYPE_DYNAMODB) -DEFINE_string(dynamodb_endpoint, "", "Endpoint of KvStore Dynamodb"); -DEFINE_string(dynamodb_keyspace, "eloq", "KeySpace of Dynamodb KvStore"); -DEFINE_string(dynamodb_region, - "ap-northeast-1", - "Region of the used trable in DynamoDB"); -#endif - -#ifdef DATA_STORE_TYPE_BIGTABLE -DEFINE_string(bigtable_project_id, "", "Project id of BigTable"); -DEFINE_string(bigtable_instance_id, "", "Instance id of BigTable"); -DEFINE_string(bigtable_keyspace, "eloq", "KeySpace of BigTable"); -#endif -// aws_secret_key -#if defined(DATA_STORE_TYPE_ELOQDSS_ROCKSDB_CLOUD_S3) -DECLARE_string(aws_access_key_id); -DECLARE_string(aws_secret_key); -#elif defined(DATA_STORE_TYPE_DYNAMODB) || \ - (defined(USE_ROCKSDB_LOG_STATE) && defined(WITH_ROCKSDB_CLOUD)) -DEFINE_string(aws_access_key_id, "", "AWS SDK access key id"); -DEFINE_string(aws_secret_key, "", "AWS SDK secret key"); -#endif - -#if ELOQDS -DEFINE_string(eloq_dss_peer_node, - "", - "EloqDataStoreService peer node address. Used to fetch eloq-dss " - "topology from a working eloq-dss server."); -DEFINE_string(eloq_dss_branch_name, - "development", - "Branch name of EloqDataStore"); -DEFINE_string(eloq_dss_config_file_path, - "", - "EloqDataStoreService config file path. Used to load eloq-dss " - "config from a file."); -#endif - -DEFINE_uint32(snapshot_sync_worker_num, 0, "Snpashot sync worker num"); - -bool DataSubstrate::InitializeStorageHandler(const INIReader &config_reader) -{ - if (CheckCommandLineFlagIsDefault("snapshot_sync_worker_num") && - !config_reader.HasValue("store", "snapshot_sync_worker_num")) - { - FLAGS_snapshot_sync_worker_num = - std::max(core_config_.core_num / 4, static_cast(1)); - } -#if defined(DATA_STORE_TYPE_DYNAMODB) - std::string dynamodb_endpoint = - !CheckCommandLineFlagIsDefault("dynamodb_endpoint") - ? FLAGS_dynamodb_endpoint - : config_reader.GetString( - "store", "dynamodb_endpoint", FLAGS_dynamodb_endpoint); - std::string dynamodb_keyspace = - !CheckCommandLineFlagIsDefault("dynamodb_keyspace") - ? FLAGS_dynamodb_keyspace - : config_reader.GetString( - "store", "dynamodb_keyspace", FLAGS_dynamodb_keyspace); - std::string dynamodb_region = - !CheckCommandLineFlagIsDefault("dynamodb_region") - ? FLAGS_dynamodb_region - : config_reader.GetString( - "store", "dynamodb_region", FLAGS_dynamodb_region); - std::string aws_access_key_id = - !CheckCommandLineFlagIsDefault("aws_access_key_id") - ? FLAGS_aws_access_key_id - : config_reader.GetString( - "store", "aws_access_key_id", FLAGS_aws_access_key_id); - std::string aws_secret_key = - !CheckCommandLineFlagIsDefault("aws_secret_key") - ? FLAGS_aws_secret_key - : config_reader.GetString( - "store", "aws_secret_key", FLAGS_aws_secret_key); - bool ddl_skip_kv = false; - uint16_t worker_pool_size = core_num_ * 2; - - store_hd_ = std::make_unique(dynamodb_keyspace, - dynamodb_endpoint, - dynamodb_region, - aws_access_key_id, - aws_secret_key, - core_config_.bootstrap, - ddl_skip_kv, - worker_pool_size, - false); - -#elif defined(DATA_STORE_TYPE_BIGTABLE) - std::string bigtable_keyspace = - !CheckCommandLineFlagIsDefault("bigtable_keyspace") - ? FLAGS_bigtable_keyspace - : config_reader.GetString( - "store", "bigtable_keyspace", FLAGS_bigtable_keyspace); - std::string bigtable_project_id = - !CheckCommandLineFlagIsDefault("bigtable_project_id") - ? FLAGS_bigtable_project_id - : config_reader.GetString( - "store", "bigtable_project_id", FLAGS_bigtable_project_id); - std::string bigtable_instance_id = - !CheckCommandLineFlagIsDefault("bigtable_instance_id") - ? FLAGS_bigtable_instance_id - : config_reader.GetString( - "store", "bigtable_instance_id", FLAGS_bigtable_instance_id); - bool ddl_skip_kv = false; - store_hd_ = std::make_unique( - bigtable_keyspace, bigtable_project_id, bigtable_instance_id, core_config_.bootstrap, - ddl_skip_kv); - -#elif defined(DATA_STORE_TYPE_ROCKSDB) - bool is_single_node = - (network_config_.standby_ip_list.empty() && - network_config_.voter_ip_list.empty() && - network_config_.ip_list.find(',') == network_config_.ip_list.npos); - - EloqShare::RocksDBConfig rocksdb_config(config_reader, - core_config_.data_path); - - store_hd_ = std::make_unique( - rocksdb_config, - (core_config_.bootstrap || is_single_node), - core_config_.enable_cache_replacement); - -#elif ELOQDS - bool is_single_node = - (network_config_.standby_ip_list.empty() && - network_config_.voter_ip_list.empty() && - network_config_.ip_list.find(',') == network_config_.ip_list.npos); - - std::string eloq_dss_peer_node = - !CheckCommandLineFlagIsDefault("eloq_dss_peer_node") - ? FLAGS_eloq_dss_peer_node - : config_reader.GetString( - "store", "eloq_dss_peer_node", FLAGS_eloq_dss_peer_node); - - std::string eloq_dss_data_path = core_config_.data_path + "/eloq_dss"; - if (!std::filesystem::exists(eloq_dss_data_path)) - { - std::filesystem::create_directories(eloq_dss_data_path); - } - - std::string dss_config_file_path = !CheckCommandLineFlagIsDefault("eloq_dss_config_file_path") - ? FLAGS_eloq_dss_config_file_path - : config_reader.GetString( - "store", "eloq_dss_config_file_path", FLAGS_eloq_dss_config_file_path); - - if (dss_config_file_path.empty()) - { - dss_config_file_path = eloq_dss_data_path + "/dss_config.ini"; - } - - EloqDS::DataStoreServiceClusterManager ds_config; - if (std::filesystem::exists(dss_config_file_path)) - { - bool load_res = ds_config.Load(dss_config_file_path); - if (!load_res) - { - LOG(ERROR) << "Failed to load dss config file : " - << dss_config_file_path; - return false; - } - } - else - { - if (!eloq_dss_peer_node.empty()) - { - ds_config.SetThisNode(network_config_.local_ip, - network_config_.local_port + 7); - // Fetch ds topology from peer node - if (!EloqDS::DataStoreService::FetchConfigFromPeer( - eloq_dss_peer_node, ds_config)) - { - LOG(ERROR) << "Failed to fetch config from peer node: " - << eloq_dss_peer_node; - return false; - } - - // Save the fetched config to the local file - if (!ds_config.Save(dss_config_file_path)) - { - LOG(ERROR) << "Failed to save config to file: " - << dss_config_file_path; - return false; - } - } - else if (core_config_.bootstrap || is_single_node) - { - // Initialize the data store service config - ds_config.Initialize(network_config_.local_ip, - network_config_.local_port + 7); - if (!ds_config.Save(dss_config_file_path)) - { - LOG(ERROR) << "Failed to save config to file: " - << dss_config_file_path; - return false; - } - } - else - { - LOG(ERROR) << "Failed to load data store service config file: " - << dss_config_file_path; - return false; - } - } - -#if defined(DATA_STORE_TYPE_ELOQDSS_ROCKSDB_CLOUD_S3) || \ - defined(DATA_STORE_TYPE_ELOQDSS_ROCKSDB_CLOUD_GCS) - EloqDS::RocksDBConfig rocksdb_config(config_reader, eloq_dss_data_path); - EloqDS::RocksDBCloudConfig rocksdb_cloud_config(config_reader); - rocksdb_cloud_config.branch_name_ = FLAGS_eloq_dss_branch_name; - auto ds_factory = std::make_unique( - rocksdb_config, - rocksdb_cloud_config, - core_config_.enable_cache_replacement); - -#elif defined(DATA_STORE_TYPE_ELOQDSS_ROCKSDB) - EloqDS::RocksDBConfig rocksdb_config(config_reader, eloq_dss_data_path); - auto ds_factory = std::make_unique( - rocksdb_config, core_config_.enable_cache_replacement); - -#elif defined(DATA_STORE_TYPE_ELOQDSS_ELOQSTORE) - EloqDS::EloqStoreConfig eloq_store_config(config_reader, - eloq_dss_data_path); - auto ds_factory = std::make_unique( - std::move(eloq_store_config)); -#endif - - data_store_service_ = std::make_unique( - ds_config, - dss_config_file_path, - eloq_dss_data_path + "/DSMigrateLog", - std::move(ds_factory)); - // setup local data store service - bool ret = data_store_service_->StartService(core_config_.bootstrap || is_single_node); - if (!ret) - { - LOG(ERROR) << "Failed to start data store service"; - return false; - } - // setup data store service client - txservice::CatalogFactory *catalog_factory[NUM_EXTERNAL_ENGINES] = {nullptr, nullptr, nullptr}; - - for (size_t i = 0; i < NUM_EXTERNAL_ENGINES; i++) - { - catalog_factory[i] = engines_[i].catalog_factory; - } - - store_hd_ = std::make_unique( - catalog_factory, ds_config, data_store_service_.get()); - -#endif - - for (const auto &[table_name, _] : prebuilt_tables_) - { - store_hd_->AppendPreBuiltTable(table_name); - } - - if (!store_hd_->Connect()) - { - LOG(ERROR) << "!!!!!!!! Failed to connect to kvstore, startup is " - "terminated !!!!!!!!"; - return false; - } - return true; -} \ No newline at end of file diff --git a/data_substrate/core/src/tx_service_init.cpp b/data_substrate/core/src/tx_service_init.cpp deleted file mode 100644 index 6ed3f8963f0..00000000000 --- a/data_substrate/core/src/tx_service_init.cpp +++ /dev/null @@ -1,290 +0,0 @@ -#include -#include -#include -#include -#include - -#include "data_substrate.h" -#include "eloq_log_wrapper.h" -#include "sequences/sequences.h" -#include "tx_service.h" -DEFINE_int32(checkpointer_interval, 10, "Checkpointer interval in seconds"); -DEFINE_int32(node_memory_limit_mb, 8192, "Node memory limit in MB"); -DEFINE_int32(checkpointer_delay_seconds, 5, "Checkpointer delay in seconds"); -DEFINE_int32(collect_active_tx_ts_interval_seconds, - 2, - "Active transaction timestamp collection interval"); -DEFINE_bool(kickout_data_for_test, false, "Kickout data for test"); -DEFINE_bool(enable_key_cache, false, "Enable key cache"); -DEFINE_uint32(max_standby_lag, - 400000, - "txservice max msg lag between primary and standby"); -DEFINE_string(tx_service_data_path, "", "path for tx_service data"); -DEFINE_bool(fork_host_manager, true, "fork host manager process"); -DEFINE_string(hm_ip, "", "Host manager IP"); -DEFINE_int32(hm_port, 0, "Host manager port"); -DEFINE_string(hm_bin_path, - "", - "host manager binary path if forking host manager process from " - "main process"); -DEFINE_uint32(deadlock_check_interval_seconds, - 10, - "Deadlock check interval in seconds"); -DEFINE_bool(realtime_sampling, - true, - "Whether enable realtime sampling. If disable it, user may need " - "to execute " - "analyze command at some time. Different from Innodb, Eloq never " - "analyze table automatically."); -DEFINE_uint32(range_split_worker_num, 0, "Range split worker number"); -DEFINE_bool(auto_redirect, - false, - "If redirect remote object command to remote node internally"); - -#ifdef ELOQ_MODULE_ELOQKV -namespace EloqKV -{ -extern std::function - eloqkv_publish_func; -} -#endif - -bool DataSubstrate::InitializeTxService(const INIReader &config_reader) -{ - uint64_t checkpointer_interval = - !CheckCommandLineFlagIsDefault("checkpointer_interval") - ? FLAGS_checkpointer_interval - : config_reader.GetInteger("local", - "checkpointer_interval", - FLAGS_checkpointer_interval); - - uint64_t checkpointer_delay_seconds = - !CheckCommandLineFlagIsDefault("checkpointer_delay_seconds") - ? FLAGS_checkpointer_delay_seconds - : config_reader.GetInteger("local", - "checkpointer_delay_seconds", - FLAGS_checkpointer_delay_seconds); - - uint64_t collect_active_tx_ts_interval_seconds = - !CheckCommandLineFlagIsDefault("collect_active_tx_ts_interval_seconds") - ? FLAGS_collect_active_tx_ts_interval_seconds - : config_reader.GetInteger( - "local", - "collect_active_tx_ts_interval_seconds", - FLAGS_collect_active_tx_ts_interval_seconds); - - uint64_t max_standby_lag = - !CheckCommandLineFlagIsDefault("max_standby_lag") - ? FLAGS_max_standby_lag - : config_reader.GetInteger( - "local", "max_standby_lag", FLAGS_max_standby_lag); - - bool kickout_data_for_test = - !CheckCommandLineFlagIsDefault("kickout_data_for_test") - ? FLAGS_kickout_data_for_test - : config_reader.GetBoolean("local", - "kickout_data_for_test", - FLAGS_kickout_data_for_test); - - bool enable_key_cache = - !CheckCommandLineFlagIsDefault("enable_key_cache") - ? FLAGS_enable_key_cache - : config_reader.GetBoolean( - "local", "enable_key_cache", FLAGS_enable_key_cache); - - std::string tx_service_data_path = - !CheckCommandLineFlagIsDefault("tx_service_data_path") - ? FLAGS_tx_service_data_path - : config_reader.GetString( - "local", "tx_service_data_path", FLAGS_tx_service_data_path); - std::string tx_path("local://"); - if (tx_service_data_path.empty()) - { - tx_path.append(core_config_.data_path); - } - else - { - tx_path.append(tx_service_data_path); - } - - const char *field_mem = "node_memory_limit_mb"; - uint64_t node_memory_limit_mb = FLAGS_node_memory_limit_mb; - if (CheckCommandLineFlagIsDefault(field_mem)) - { - if (config_reader.HasValue("local", field_mem)) - { - node_memory_limit_mb = - config_reader.GetInteger("local", field_mem, 0); - assert(node_memory_limit_mb); - } - else - { - struct sysinfo meminfo; - if (sysinfo(&meminfo)) - { - LOG(ERROR) << "config is missing: " << field_mem; - } - uint32_t mem_mib = - ((uint64_t) meminfo.totalram * meminfo.mem_unit) / - (1024 * 1024); - node_memory_limit_mb = std::max(uint32_t(512), (mem_mib * 4) / 5); - LOG(INFO) << "config is automatically set: " << field_mem << "=" - << node_memory_limit_mb - << "(MiB), total memory=" << mem_mib; - } - } - FLAGS_node_memory_limit_mb = node_memory_limit_mb; - - uint64_t deadlock_check_interval_seconds = - !CheckCommandLineFlagIsDefault("deadlock_check_interval_seconds") - ? FLAGS_deadlock_check_interval_seconds - : config_reader.GetInteger("local", - "deadlock_check_interval_seconds", - FLAGS_deadlock_check_interval_seconds); - txservice::DeadLockCheck::SetTimeInterval(deadlock_check_interval_seconds); - - bool realtime_sampling = - !CheckCommandLineFlagIsDefault("realtime_sampling") - ? FLAGS_realtime_sampling - : config_reader.GetBoolean( - "local", "realtime_sampling", FLAGS_realtime_sampling); - - uint64_t range_split_worker_num = - !CheckCommandLineFlagIsDefault("range_split_worker_num") - ? FLAGS_range_split_worker_num - : config_reader.GetInteger("local", - "range_split_worker_num", - FLAGS_range_split_worker_num); - - bool auto_redirect = - !CheckCommandLineFlagIsDefault("auto_redirect") - ? FLAGS_auto_redirect - : config_reader.GetBoolean( - "local", "auto_redirect", FLAGS_auto_redirect); - - bool fork_host_manager = - !CheckCommandLineFlagIsDefault("fork_host_manager") - ? FLAGS_fork_host_manager - : config_reader.GetBoolean( - "local", "fork_host_manager", FLAGS_fork_host_manager); - - std::string hm_ip = !CheckCommandLineFlagIsDefault("hm_ip") - ? FLAGS_hm_ip - : config_reader.Get("local", "hm_ip", FLAGS_hm_ip); - - uint16_t hm_port = - !CheckCommandLineFlagIsDefault("hm_port") - ? FLAGS_hm_port - : config_reader.GetInteger("local", "hm_port", FLAGS_hm_port); - - std::string hm_bin_path = - !CheckCommandLineFlagIsDefault("hm_bin_path") - ? FLAGS_hm_bin_path - : config_reader.Get("local", "hm_bin_path", FLAGS_hm_bin_path); -#ifdef FORK_HM_PROCESS - if (hm_ip.empty()) - { - hm_ip = network_config_.local_ip; - } - if (hm_port == 0) - { - hm_port = network_config_.local_port + 4; - } - if (hm_bin_path.empty()) - { - char path_buf[PATH_MAX]; - ssize_t len = ::readlink("/proc/self/exe", path_buf, sizeof(path_buf)); - path_buf[len] = '\0'; - std::string s_path(path_buf); - std::string::size_type pos = s_path.find_last_of("/"); - std::string parent_path = s_path.substr(0, pos); - hm_bin_path = parent_path + "/host_manager"; - } -#endif - - std::map tx_service_conf{ - {"core_num", core_config_.core_num}, - {"checkpointer_interval", checkpointer_interval}, - {"node_memory_limit_mb", node_memory_limit_mb}, - {"checkpointer_delay_seconds", checkpointer_delay_seconds}, - {"collect_active_tx_ts_interval_seconds", - collect_active_tx_ts_interval_seconds}, - {"realtime_sampling", realtime_sampling ? 1 : 0}, - {"rep_group_cnt", network_config_.node_group_replica_num}, - {"range_split_worker_num", range_split_worker_num}, - {"enable_shard_heap_defragment", - core_config_.enable_heap_defragment ? 1 : 0}, - {"enable_key_cache", enable_key_cache}, - {"max_standby_lag", max_standby_lag}, - {"kickout_data_for_test", kickout_data_for_test ? 1 : 0}}; - - txservice::CatalogFactory *catalog_factory[NUM_EXTERNAL_ENGINES] = { - nullptr, nullptr, nullptr}; - - for (size_t i = 0; i < NUM_EXTERNAL_ENGINES; i++) - { - catalog_factory[i] = engines_[i].catalog_factory; - } - - auto log_agent = std::make_unique( - network_config_.node_group_replica_num); - - tx_service_ = std::make_unique( - catalog_factory, - system_handler_, - tx_service_conf, - network_config_.node_id, - network_config_.native_ng_id, - &network_config_.ng_configs, - network_config_.cluster_config_version, - core_config_.enable_data_store ? store_hd_.get() : nullptr, - log_agent.get(), - core_config_.enable_mvcc, // enable_mvcc - !core_config_.enable_wal, // skip_wal - !core_config_.enable_data_store, // skip_kv - core_config_.enable_cache_replacement, // enable_cache_replacement - auto_redirect, // auto_redirect - metrics_registry_.get(), // metrics_registry - tx_service_common_labels_, // common_labels - &prebuilt_tables_ -#ifdef ELOQ_MODULE_ELOQKV - , - EloqKV::eloqkv_publish_func, - external_metrics -#endif - ); - - if (core_config_.enable_data_store) - { - store_hd_->SetTxService(tx_service_.get()); - } - - if (tx_service_->Start(network_config_.node_id, - network_config_.native_ng_id, - &network_config_.ng_configs, - network_config_.cluster_config_version, - &log_service_config_.txlog_ips, - &log_service_config_.txlog_ports, - &hm_ip, - &hm_port, - &hm_bin_path, - tx_service_conf, - std::move(log_agent), - tx_path, - network_config_.cluster_config_file_path, - fork_host_manager) != 0) - { - LOG(ERROR) << "Failed to start tx service!!!!!"; - return false; - } - - txservice::Sequences::InitSequence(tx_service_.get(), store_hd_.get()); - - // tx_service is a distributed service, should wait for all the tx_service - // nodes to finish the log recovery process and setup the cc_stream_sender. - tx_service_->WaitClusterReady(); - // wait for the tx_service node to become the native group leader. - tx_service_->WaitNodeBecomeNativeGroupLeader(); - - return true; -} \ No newline at end of file diff --git a/data_substrate/eloq_metrics b/data_substrate/eloq_metrics deleted file mode 160000 index 9ff8eddfaf3..00000000000 --- a/data_substrate/eloq_metrics +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 9ff8eddfaf3c7980b8585de681a3802efe0cc1ba diff --git a/data_substrate/log_service b/data_substrate/log_service deleted file mode 160000 index 9caaf7cd9e7..00000000000 --- a/data_substrate/log_service +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 9caaf7cd9e78ee4a6e4eb6e6b35bff16f852e249 diff --git a/data_substrate/store_handler b/data_substrate/store_handler deleted file mode 160000 index 86f996199cd..00000000000 --- a/data_substrate/store_handler +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 86f996199cdbf7812e9f394ea25f9b1f7a1b2809 diff --git a/data_substrate/tx_service b/data_substrate/tx_service deleted file mode 160000 index 285d2ad6879..00000000000 --- a/data_substrate/tx_service +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 285d2ad68793b940705e75f3382e604ff747863a From 9a8370c39b9b09bc88c536679616c1f49f418d70 Mon Sep 17 00:00:00 2001 From: liunyl Date: Fri, 24 Oct 2025 09:50:40 +0000 Subject: [PATCH 09/10] update build and pr script --- concourse/scripts/pr.ent.bash | 6 ++--- concourse/scripts/tag.sh | 4 +-- sql/CMakeLists.txt | 3 ++- sql/mysqld.cc | 2 -- storage/eloq/eloq_db_dl.h | 22 --------------- storage/eloq/ha_eloq.cc | 51 ++++++++++++++++++----------------- 6 files changed, 34 insertions(+), 54 deletions(-) diff --git a/concourse/scripts/pr.ent.bash b/concourse/scripts/pr.ent.bash index 2106bae2a16..600043032f2 100755 --- a/concourse/scripts/pr.ent.bash +++ b/concourse/scripts/pr.ent.bash @@ -34,7 +34,7 @@ git submodule sync git submodule update --init --recursive pr_branch_name=$(cat .git/resource/metadata.json | jq -r '.[] | select(.name=="head_name") | .value') -cd /home/$current_user/workspace/eloqsql/storage/eloq +cd /home/$current_user/workspace/eloqsql/data_substrate ln -s $WORKSPACE/logservice_src eloq_log_service cd eloq_log_service @@ -183,7 +183,7 @@ echo "installing" cmake --install . --config Debug echo "building dss_server" -cd /home/$current_user/workspace/eloqsql/storage/eloq/store_handler/eloq_data_store_service +cd /home/$current_user/workspace/eloqsql/data_substrate/store_handler/eloq_data_store_service mkdir bld && cd bld cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DCMAKE_BUILD_TYPE=Debug -DWITH_DATA_STORE=ELOQDSS_ROCKSDB_CLOUD_S3 ../ cmake --build . --config Debug -j8 @@ -191,7 +191,7 @@ echo "installing dss_server" cp dss_server /home/$current_user/workspace/eloqsql/install/bin/ echo "building log_server" -cd /home/$current_user/workspace/eloqsql/storage/eloq/eloq_log_service +cd /home/$current_user/workspace/eloqsql/data_substrate/eloq_log_service mkdir bld && cd bld cmake -DCMAKE_BUILD_TYPE=Debug -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DWITH_LOG_STATE=ROCKSDB_CLOUD_S3 ../ cmake --build . --config Debug -j8 diff --git a/concourse/scripts/tag.sh b/concourse/scripts/tag.sh index 212e0c958b7..fff5024a6a7 100755 --- a/concourse/scripts/tag.sh +++ b/concourse/scripts/tag.sh @@ -14,8 +14,8 @@ fi cd $HOME ln -s ${WORKSPACE}/eloqsql_src eloqsql cd eloqsql -ln -s $WORKSPACE/logservice_src storage/eloq/eloq_log_service -pushd storage/eloq/tx_service +ln -s $WORKSPACE/logservice_src data_substrate/eloq_log_service +pushd data_substrate/tx_service ln -s $WORKSPACE/raft_host_manager_src raft_host_manager popd diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt index 70fcc9bed2d..662b6df8018 100644 --- a/sql/CMakeLists.txt +++ b/sql/CMakeLists.txt @@ -279,6 +279,7 @@ SET(SQL_DEP_LIBS ${SSL_LIBRARIES} ${LIBSYSTEMD} ${LIBSYSTEMD} + ${DATA_SUBSTRATE_LIB} ) IF(COROUTINE_ENABLED) @@ -287,7 +288,7 @@ IF(COROUTINE_ENABLED) ENDIF() IF(ELOQ_MODULE_ENABLED) - SET(SQL_DEP_LIBS ${SQL_DEP_LIBS} ${BRPC_LIB} ${DATA_SUBSTRATE_LIB}) + SET(SQL_DEP_LIBS ${SQL_DEP_LIBS} ${BRPC_LIB}) set(SQL_INCLUDE_DIR ${BRPC_INCLUDE_PATH}) ENDIF() diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 5e2b2f8f8b7..38b2a6e9ba6 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -7837,7 +7837,6 @@ static int mysql_init_variables(void) mysql_real_data_home_len= (uint)(strmake_buf(mysql_real_data_home, get_relative_path(MYSQL_DATADIR)) - mysql_real_data_home); - LOG(INFO) << "default mysql data home " << mysql_real_data_home; /* Replication parameters */ master_info_file= (char*) "master.info", relay_log_info_file= (char*) "relay-log.info"; @@ -8024,7 +8023,6 @@ mysqld_get_one_option(const struct my_option *opt, const char *argument, break; case 'h': strmake_buf(mysql_real_data_home, argument); - LOG(INFO) << "mysql data homne " << mysql_real_data_home; /* Correct pointer set by my_getopt (for embedded library) */ mysql_real_data_home_ptr= mysql_real_data_home; break; diff --git a/storage/eloq/eloq_db_dl.h b/storage/eloq/eloq_db_dl.h index 86387af1561..6bc5f025ced 100644 --- a/storage/eloq/eloq_db_dl.h +++ b/storage/eloq/eloq_db_dl.h @@ -53,32 +53,10 @@ static LEX_CSTRING ELOQ_ENGINE_NAME= {STRING_WITH_LEN("ELOQ")}; * These functions must be wrapped with extern "C", and function signature * must match with that defined by ELOQ_DEFINEx. */ -// #if defined(ELOQ_LINK_STATIC) // Statically linked: declare the symbol and let the linker resolve it to the // implementation provided in the static library (see eloq_db.cpp extern "C" block). #define MONOCALL_DEFINEx(x, name, ...) \ extern "C" int name(__SC_DECL##x(__VA_ARGS__)); -// #else -// #define MONOCALL_DEFINEx(x, name, ...) \ -// static inline int name(__SC_DECL##x(__VA_ARGS__)) \ -// { \ -// using T= decltype(&name); \ -// \ -// int ret= 0; \ -// \ -// static_assert(std::is_same::value); \ -// static THD *t= __ARG1(__VA_ARGS__); \ -// \ -// static plugin_ref pi= ha_resolve_by_name(t, &ELOQ_ENGINE_NAME, false); \ -// \ -// static void *handle= plugin_handle(pi); \ -// static T fp= reinterpret_cast(dlsym(handle, __FUNCTION__)); \ -// ret= fp(__SC_CAST##x(__VA_ARGS__)); \ -// \ -// return ret; \ -// } -// #endif - #define MONOCALL_DEFINE1(name, ...) MONOCALL_DEFINEx(1, name, __VA_ARGS__) #define MONOCALL_DEFINE2(name, ...) MONOCALL_DEFINEx(2, name, __VA_ARGS__) #define MONOCALL_DEFINE3(name, ...) MONOCALL_DEFINEx(3, name, __VA_ARGS__) diff --git a/storage/eloq/ha_eloq.cc b/storage/eloq/ha_eloq.cc index b2423362ae8..93df9a1008f 100644 --- a/storage/eloq/ha_eloq.cc +++ b/storage/eloq/ha_eloq.cc @@ -190,13 +190,14 @@ #include #include // Define synchronization variables -namespace mysqld_converged_sync { - std::mutex init_mutex; - std::condition_variable mysqld_basic_init_done_cv; - std::condition_variable data_substrate_init_done_cv; - bool mysqld_basic_init_done = false; - bool data_substrate_init_done = false; -} +namespace mysqld_converged_sync +{ +std::mutex init_mutex; +std::condition_variable mysqld_basic_init_done_cv; +std::condition_variable data_substrate_init_done_cv; +bool mysqld_basic_init_done= false; +bool data_substrate_init_done= false; +} // namespace mysqld_converged_sync #endif #define DEFAULT_SCAN_TUPLE_SIZE 128 @@ -210,8 +211,7 @@ namespace mysqld_converged_sync { using namespace MyEloq; using namespace txservice; -DEFINE_string(config, "", - "Path to data substrate configuration file."); +DEFINE_string(config, "", "Path to data substrate configuration file."); MariaCatalogFactory maria_catalog_factory; txservice::CatalogFactory *eloqsql_catalog_factory= &maria_catalog_factory; @@ -421,13 +421,13 @@ static MYSQL_SYSVAR_INT(signal_monitor, eloq_signal_monitor, nullptr, nullptr, 0, 0, INT_MAX, 0); static struct st_mysql_sys_var *eloq_system_variables[]= { - MYSQL_SYSVAR(cc_protocol), MYSQL_SYSVAR(enum_var), - MYSQL_SYSVAR(ulong_var), MYSQL_SYSVAR(int_var), - MYSQL_SYSVAR(double_var), MYSQL_SYSVAR(double_thdvar), - MYSQL_SYSVAR(varopt_default), MYSQL_SYSVAR(insert_semantic), - MYSQL_SYSVAR(auto_increment), MYSQL_SYSVAR(invalidate_cache_once), - MYSQL_SYSVAR(random_scan_sort), MYSQL_SYSVAR(report_debug_info), - MYSQL_SYSVAR(signal_monitor), NULL}; + MYSQL_SYSVAR(cc_protocol), MYSQL_SYSVAR(enum_var), + MYSQL_SYSVAR(ulong_var), MYSQL_SYSVAR(int_var), + MYSQL_SYSVAR(double_var), MYSQL_SYSVAR(double_thdvar), + MYSQL_SYSVAR(varopt_default), MYSQL_SYSVAR(insert_semantic), + MYSQL_SYSVAR(auto_increment), MYSQL_SYSVAR(invalidate_cache_once), + MYSQL_SYSVAR(random_scan_sort), MYSQL_SYSVAR(report_debug_info), + MYSQL_SYSVAR(signal_monitor), NULL}; /** Structure for CREATE TABLE options (table options). @@ -1457,16 +1457,16 @@ static int eloq_init_func(void *p) // 2. Wait for data substrate to be initialized by converged main { std::unique_lock lock(mysqld_converged_sync::init_mutex); - mysqld_converged_sync::mysqld_basic_init_done = true; + mysqld_converged_sync::mysqld_basic_init_done= true; mysqld_converged_sync::mysqld_basic_init_done_cv.notify_one(); - - LOG(INFO) << "MySQL basic initialization complete, waiting for data substrate..."; - + + LOG(INFO) << "MySQL basic initialization complete, waiting for data " + "substrate..."; + // Wait for data substrate initialization to complete - mysqld_converged_sync::data_substrate_init_done_cv.wait(lock, [] { - return mysqld_converged_sync::data_substrate_init_done; - }); - + mysqld_converged_sync::data_substrate_init_done_cv.wait( + lock, [] { return mysqld_converged_sync::data_substrate_init_done; }); + LOG(INFO) << "Data substrate initialized, MySQL continuing..."; } #else @@ -1506,6 +1506,9 @@ static int eloq_init_func(void *p) storage_hd= DataSubstrate::GetGlobal()->GetStoreHandler(); node_id= DataSubstrate::GetGlobal()->GetNetworkConfig().node_id; +#ifdef EXT_TX_PROC_ENABLED + get_tx_service_functors= tx_service->GetTxProcFunctors(); +#endif sql_print_information("MariaDB data home: %s", mysql_real_data_home_ptr); // initialize monogrpah specific error message. From f5fb5aa8c2f061889c41db1bf67b5d3bb79d13d7 Mon Sep 17 00:00:00 2001 From: liunyl Date: Tue, 4 Nov 2025 07:37:11 +0000 Subject: [PATCH 10/10] fix mtr compatible issue --- data_substrate | 2 +- mysql-test/include/eloq_kv_dss.cnf | 15 ------ mysql-test/include/eloq_kv_keyspace.cnf | 4 -- mysql-test/include/eloq_kv_storage.cnf | 2 - scripts/mysql_install_db.sh | 27 ++-------- sql/main.cc | 9 ++++ sql/mysqld.cc | 14 ++--- storage/eloq/eloq_db.cpp | 6 --- storage/eloq/eloq_system_handler.h | 9 +++- storage/eloq/eloqsql_catalog_factory.cpp | 4 +- storage/eloq/eloqsql_catalog_factory.h | 5 +- storage/eloq/ha_eloq.cc | 17 +++++-- .../mysql-test/mono_basic/data_substrate.cnf | 6 +++ .../include/have_range_partition.inc | 21 -------- storage/eloq/mysql-test/mono_basic/my.cnf | 15 ++---- .../mysql-test/mono_basic/t/alter_table.test | 1 - .../mono_basic/t/alter_table_recovery.test | 1 - .../mono_basic/t/create_drop_index.test | 1 - .../mono_basic/t/data_store_error.test | 1 - .../mono_basic/t/deadlock_handling.cnf | 14 ++--- .../mono_basic/t/flush_undo_while_ckpt.cnf | 15 ++---- .../mono_basic/t/flush_undo_while_ckpt_ds.cnf | 8 +++ .../t/isolation_level_for_locking.cnf | 12 ++--- .../mono_basic/t/isolation_level_for_mvcc.cnf | 14 ++--- .../mono_basic/t/isolation_level_for_occ.cnf | 18 +++---- .../mysql-test/mono_basic/t/key_cache.cnf | 14 ++--- .../mysql-test/mono_basic/t/key_cache_ds.cnf | 7 +++ .../mono_basic/t/mvcc_archives_recycle.cnf | 12 ++--- .../eloq/mysql-test/mono_basic/t/mvcc_ds.cnf | 6 +++ ...vcc_read_after_flushing_deleted_record.cnf | 12 ++--- ...nique_sk_after_flushing_deleted_record.cnf | 12 ++--- .../mysql-test/mono_basic/t/mvcc_scan.cnf | 15 ++---- ...vcc_scan_after_flushing_deleted_record.cnf | 12 ++--- .../mysql-test/mono_basic/t/mvcc_scan_sk.cnf | 12 ++--- .../mono_basic/t/mvcc_scan_sk_rc_isolvl.cnf | 4 +- .../mono_basic/t/mvcc_unique_sk_lookup.cnf | 12 ++--- .../mono_basic/t/poor_performance.cnf | 16 ++---- .../mono_basic/t/poor_performance.test | 1 - .../mono_basic/t/poor_performance_ds.cnf | 6 +++ .../mono_basic/t/range_split_keycache.cnf | 18 ++----- .../mono_basic/t/range_split_keycache.test | 1 - .../t/select_for_update_and_share.cnf | 12 ++--- .../t/select_for_update_and_share_occ.cnf | 26 ---------- .../mono_basic/t/single_dead_lock_check.cnf | 15 ++---- .../t/single_dead_lock_check_ds.cnf | 6 +++ .../mysql-test/mono_basic/t/single_node.cnf | 27 ---------- .../mysql-test/mono_main/data_substrate.cnf | 6 +++ storage/eloq/mysql-test/mono_main/my.cnf | 17 ++----- .../r/system_mysql_db_error_log.result | 8 +-- .../eloq/mysql-test/mono_main/t/commit.opt | 2 +- .../mono_main/t/connect-abstract.cnf | 17 +++---- .../eloq/mysql-test/mono_main/t/connect.cnf | 17 +++---- .../eloq/mysql-test/mono_main/t/connect2.cnf | 13 ++--- .../eloq/mysql-test/mono_main/t/connect2.opt | 1 + .../mono_main/t/consistent_snapshot.opt | 2 +- .../mono_main/t/pool_of_threads.cnf | 13 ++--- .../mono_main/t/pool_of_threads.opt | 1 + .../mysql-test/mono_main/t/sp_notembedded.opt | 2 +- .../t/subselect_exists2in_costmat.cnf | 13 ++--- .../mono_main/t/subselect_mat_cost.cnf | 13 ++--- .../t/system_mysql_db_error_log.test | 8 +-- .../mysql-test/mono_multi/data_substrate1.cnf | 13 +++++ .../mysql-test/mono_multi/data_substrate2.cnf | 13 +++++ .../mysql-test/mono_multi/data_substrate3.cnf | 12 +++++ storage/eloq/mysql-test/mono_multi/my.cnf | 21 ++------ .../t/alter_schema_term_changed.test | 1 - .../mono_multi/t/check_blocked_request.cnf | 46 ----------------- .../mono_multi/t/check_blocked_request.opt | 1 + .../mono_multi/t/create_drop_index.test | 1 - .../mono_multi/t/fault_create_drop_index.test | 1 - .../mono_multi/t/improve_ckpt_flush_undo.cnf | 46 ----------------- .../mono_multi/t/improve_ckpt_flush_undo.opt | 1 + .../t/improve_ckpt_flush_undo_sk1.cnf | 46 ----------------- .../t/improve_ckpt_flush_undo_sk1.opt | 1 + .../t/improve_ckpt_flush_undo_sk2.cnf | 46 ----------------- .../t/improve_ckpt_flush_undo_sk2.opt | 1 + .../mono_multi/t/multi_dead_lock_check.cnf | 48 ----------------- .../mono_multi/t/multi_dead_lock_check.opt | 1 + .../mysql-test/mono_multi/t/range_split.test | 1 - .../mono_multi/t/range_split_keycache.cnf | 51 ------------------- .../mono_multi/t/range_split_keycache.opt | 1 + .../mono_multi/t/range_split_keycache.test | 1 - .../mono_multi/t/range_split_recovery.cnf | 49 ------------------ .../mono_multi/t/range_split_recovery.opt | 1 + .../mono_multi/t/range_split_recovery.test | 1 - .../t/range_split_write_forward.test | 1 - .../mysql-test/mono_multi/t/tx_timeout.cnf | 45 ---------------- .../mysql-test/mono_multi/t/tx_timeout.opt | 1 + 88 files changed, 248 insertions(+), 823 deletions(-) delete mode 100644 mysql-test/include/eloq_kv_dss.cnf delete mode 100644 mysql-test/include/eloq_kv_storage.cnf create mode 100644 storage/eloq/mysql-test/mono_basic/data_substrate.cnf delete mode 100644 storage/eloq/mysql-test/mono_basic/include/have_range_partition.inc create mode 100644 storage/eloq/mysql-test/mono_basic/t/flush_undo_while_ckpt_ds.cnf create mode 100644 storage/eloq/mysql-test/mono_basic/t/key_cache_ds.cnf create mode 100644 storage/eloq/mysql-test/mono_basic/t/mvcc_ds.cnf create mode 100644 storage/eloq/mysql-test/mono_basic/t/poor_performance_ds.cnf delete mode 100644 storage/eloq/mysql-test/mono_basic/t/select_for_update_and_share_occ.cnf create mode 100644 storage/eloq/mysql-test/mono_basic/t/single_dead_lock_check_ds.cnf delete mode 100644 storage/eloq/mysql-test/mono_basic/t/single_node.cnf create mode 100644 storage/eloq/mysql-test/mono_main/data_substrate.cnf create mode 100644 storage/eloq/mysql-test/mono_main/t/connect2.opt create mode 100644 storage/eloq/mysql-test/mono_main/t/pool_of_threads.opt create mode 100644 storage/eloq/mysql-test/mono_multi/data_substrate1.cnf create mode 100644 storage/eloq/mysql-test/mono_multi/data_substrate2.cnf create mode 100644 storage/eloq/mysql-test/mono_multi/data_substrate3.cnf delete mode 100644 storage/eloq/mysql-test/mono_multi/t/check_blocked_request.cnf create mode 100644 storage/eloq/mysql-test/mono_multi/t/check_blocked_request.opt delete mode 100644 storage/eloq/mysql-test/mono_multi/t/improve_ckpt_flush_undo.cnf create mode 100644 storage/eloq/mysql-test/mono_multi/t/improve_ckpt_flush_undo.opt delete mode 100644 storage/eloq/mysql-test/mono_multi/t/improve_ckpt_flush_undo_sk1.cnf create mode 100644 storage/eloq/mysql-test/mono_multi/t/improve_ckpt_flush_undo_sk1.opt delete mode 100644 storage/eloq/mysql-test/mono_multi/t/improve_ckpt_flush_undo_sk2.cnf create mode 100644 storage/eloq/mysql-test/mono_multi/t/improve_ckpt_flush_undo_sk2.opt delete mode 100644 storage/eloq/mysql-test/mono_multi/t/multi_dead_lock_check.cnf create mode 100644 storage/eloq/mysql-test/mono_multi/t/multi_dead_lock_check.opt delete mode 100644 storage/eloq/mysql-test/mono_multi/t/range_split_keycache.cnf create mode 100644 storage/eloq/mysql-test/mono_multi/t/range_split_keycache.opt delete mode 100644 storage/eloq/mysql-test/mono_multi/t/range_split_recovery.cnf create mode 100644 storage/eloq/mysql-test/mono_multi/t/range_split_recovery.opt delete mode 100644 storage/eloq/mysql-test/mono_multi/t/tx_timeout.cnf create mode 100644 storage/eloq/mysql-test/mono_multi/t/tx_timeout.opt diff --git a/data_substrate b/data_substrate index 09b423ebb72..99aea9346ca 160000 --- a/data_substrate +++ b/data_substrate @@ -1 +1 @@ -Subproject commit 09b423ebb72f83d9013116bf7b7b591a7105692c +Subproject commit 99aea9346ca97550e57a1d4c9f2cceff5a57cb06 diff --git a/mysql-test/include/eloq_kv_dss.cnf b/mysql-test/include/eloq_kv_dss.cnf deleted file mode 100644 index 11a267751b5..00000000000 --- a/mysql-test/include/eloq_kv_dss.cnf +++ /dev/null @@ -1,15 +0,0 @@ -[mysqld] -eloq_aws_access_key_id=XXXXXXXXXXXXXXX -eloq_aws_secret_key=XXXXXXXXXXXXXXXXXXXXXXXXX -eloq_txlog_rocksdb_cloud_endpoint_url=http://127.0.0.1:9900 -eloq_txlog_rocksdb_cloud_bucket_name = eloqsql-mtr-test -eloq_txlog_rocksdb_cloud_bucket_prefix = txlog- -eloq_txlog_rocksdb_cloud_region = ap-northeast-1 -eloq_dss_rocksdb_cloud_endpoint_url=http://127.0.0.1:9900 -eloq_dss_rocksdb_cloud_bucket_name = eloqsql-mtr-test -eloq_dss_rocksdb_cloud_bucket_prefix = dss- -eloq_dss_rocksdb_cloud_region = ap-northeast-1 -eloq_dss_config_file_path = -eloq_dss_peer_node = -eloq_dss_rocksdb_cloud_sst_file_cache_size = 20GB -eloq_dss_rocksdb_cloud_purger_periodicity_secs = 30 diff --git a/mysql-test/include/eloq_kv_keyspace.cnf b/mysql-test/include/eloq_kv_keyspace.cnf index 0942ada113c..e670b33f60a 100644 --- a/mysql-test/include/eloq_kv_keyspace.cnf +++ b/mysql-test/include/eloq_kv_keyspace.cnf @@ -2,7 +2,3 @@ [mysqld] eloq_random_scan_sort=on -eloq_keyspace_name=mono_test -eloq_dynamodb_endpoint=http://127.0.0.1:9100 -eloq_bigtable_project_id=XXXXXXXX -eloq_bigtable_instance_id=XXXXXXXX diff --git a/mysql-test/include/eloq_kv_storage.cnf b/mysql-test/include/eloq_kv_storage.cnf deleted file mode 100644 index ffcaf59ee52..00000000000 --- a/mysql-test/include/eloq_kv_storage.cnf +++ /dev/null @@ -1,2 +0,0 @@ -[mysqld] -eloq_kv_storage=eloqds diff --git a/scripts/mysql_install_db.sh b/scripts/mysql_install_db.sh index 33028fded8a..7c39d2b2122 100644 --- a/scripts/mysql_install_db.sh +++ b/scripts/mysql_install_db.sh @@ -540,28 +540,11 @@ fi mysqld_bootstrap="${MYSQLD_BOOTSTRAP-$mysqld}" mysqld_install_cmd_line() { - # Create temporary config file for bootstrap - local temp_config=$(mktemp) - - cat > "$temp_config" << EOF -[mysqld] -basedir=$basedir -datadir=$ldata -plugin-dir=$plugindir -log-warnings=0 -enforce-storage-engine= -max-allowed-packet=8M -net-buffer-length=16K -EOF - - # Call mysqld with config file and bootstrap flag - "$mysqld_bootstrap" --eloqsql_config="$temp_config" --bootstrap $args - local exit_code=$? - - # Clean up temp file - rm -f "$temp_config" - - return $exit_code + "$mysqld_bootstrap" $defaults $defaults_group_suffix "$mysqld_opt" --bootstrap $silent_startup\ + "--basedir=$basedir" "--datadir=$ldata" --log-warnings=0 --enforce-storage-engine="" \ + "--plugin-dir=${plugindir}" \ + $args --max_allowed_packet=8M \ + --net_buffer_length=16K } # Use $auth_root_socket_user if explicitly specified. diff --git a/sql/main.cc b/sql/main.cc index ffd28c94014..84d3dd75658 100644 --- a/sql/main.cc +++ b/sql/main.cc @@ -43,7 +43,16 @@ int main(int argc, char **argv) "Data substrate flags can be passed on command line.\n" "Use --help to see all available flags."); +#ifdef MYSQLD_LIBRARY_MODE + // When compiled as library mode, only accept flags that are defined in the data substrate gflags::ParseCommandLineFlags(&argc, &argv, true); +#else + // When compiled as standalone mode, allow unrecognized flags to be passed through to MySQL + gflags::AllowCommandLineReparsing(); + // Allow all MySQL-specific flags to pass through without error + gflags::SetCommandLineOption("undefok", "*"); + gflags::ParseCommandLineNonHelpFlags(&argc, &argv, false); +#endif #ifdef WITH_GLOG InitGoogleLogging(argv); diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 38b2a6e9ba6..2a3a07b77a1 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -5263,7 +5263,7 @@ static int init_server_components() We need to eat any 'loose' arguments first before we conclude that there are unprocessed options. */ - my_getopt_skip_unknown= 0; + my_getopt_skip_unknown= TRUE; #ifdef WITH_WSREP if (wsrep_recovery) my_getopt_skip_unknown= TRUE; @@ -5281,12 +5281,12 @@ static int init_server_components() if (!wsrep_recovery) { #endif - if (remaining_argc > 1) - { - fprintf(stderr, "%s: Too many arguments (first extra is '%s').\n", - my_progname, remaining_argv[1]); - unireg_abort(1); - } + // if (remaining_argc > 1) + // { + // fprintf(stderr, "%s: Too many arguments (first extra is '%s').\n", + // my_progname, remaining_argv[1]); + // unireg_abort(1); + // } #ifdef WITH_WSREP } #endif diff --git a/storage/eloq/eloq_db.cpp b/storage/eloq/eloq_db.cpp index da13ed801ab..43c693c6994 100644 --- a/storage/eloq/eloq_db.cpp +++ b/storage/eloq/eloq_db.cpp @@ -587,26 +587,22 @@ static int load_view_fetch_func() std::string schema_image; bool ok= storage_hd->FetchTable(VIEW_TABLE_NAME, schema_image, exists, version_ts); - LOG(INFO) << "load_view_fetch_func " << ok << " exists: " << exists << " version_ts: " << version_ts; if (ok) { if (exists) { eloq_fetch_view_impl= eloq_fetch_view_normal; - LOG(INFO) << "eloq_fetch_view_impl= eloq_fetch_view_normal"; ret= 0; } else { eloq_fetch_view_impl= eloq_fetch_view_bootstrap; - LOG(INFO) << "eloq_fetch_view_impl= eloq_fetch_view_bootstrap"; ret= HA_ERR_KEY_NOT_FOUND; } } else { eloq_fetch_view_impl= eloq_fetch_view_error; - LOG(INFO) << "eloq_fetch_view_impl= eloq_fetch_view_error"; ret= HA_ERR_INTERNAL_ERROR; } return ret; @@ -616,7 +612,6 @@ static int eloq_fetch_view_bootstrap(THD *thd, LEX_CSTRING db, LEX_CSTRING view, LEX_CSTRING &frm_binary) { DBUG_ENTER_FUNC(); - LOG(INFO) << "eloq_fetch_view_bootstrap " << db.str << " " << view.str << ", opt bootstrap: " << opt_bootstrap; int err= load_view_fetch_func(); if (err == 0) { @@ -636,7 +631,6 @@ static int eloq_fetch_view_normal(THD *thd, LEX_CSTRING db, LEX_CSTRING view, LEX_CSTRING &frm_binary) { DBUG_ENTER_FUNC(); - LOG(INFO) << "eloq_fetch_view_normal " << db.str << " " << view.str; int ret= 0; diff --git a/storage/eloq/eloq_system_handler.h b/storage/eloq/eloq_system_handler.h index 0f2fbfb1c35..bec9300d613 100644 --- a/storage/eloq/eloq_system_handler.h +++ b/storage/eloq/eloq_system_handler.h @@ -44,7 +44,14 @@ class MariaSystemHandler : public txservice::SystemHandler private: MariaSystemHandler(); - ~MariaSystemHandler()= default; + ~MariaSystemHandler() + { + Shutdown(); + if (thd_.joinable()) + { + thd_.join(); + } + } void SubmitWork(std::packaged_task work); diff --git a/storage/eloq/eloqsql_catalog_factory.cpp b/storage/eloq/eloqsql_catalog_factory.cpp index 2dce4fe528e..5e35882638a 100644 --- a/storage/eloq/eloqsql_catalog_factory.cpp +++ b/storage/eloq/eloqsql_catalog_factory.cpp @@ -435,7 +435,9 @@ MysqlTableSchema::MysqlTableSchema(const txservice::TableName &table_name, } size_t offset= 0; - kv_info_= storage_hd->DeserializeKVCatalogInfo(kv_info, offset); + kv_info_= DataSubstrate::GetGlobal() + ->GetStoreHandler() + ->DeserializeKVCatalogInfo(kv_info, offset); record_schema_= EloqRecordSchema(&mysql_table_share_); } diff --git a/storage/eloq/eloqsql_catalog_factory.h b/storage/eloq/eloqsql_catalog_factory.h index 6d71258e029..e1ba9cee23a 100644 --- a/storage/eloq/eloqsql_catalog_factory.h +++ b/storage/eloq/eloqsql_catalog_factory.h @@ -27,6 +27,7 @@ #include "eloqsql_key.h" #include "eloqsql_schema.h" #include "ha_eloq_macro.h" +#include "data_substrate.h" #include "tx_service/include/table_statistics.h" #include "tx_service/include/type.h" // TableName @@ -116,7 +117,9 @@ struct MysqlTableSchema : public txservice::TableSchema void SetKVCatalogInfo(const std::string &kv_info) override { size_t offset= 0; - kv_info_= storage_hd->DeserializeKVCatalogInfo(kv_info, offset); + kv_info_= DataSubstrate::GetGlobal() + ->GetStoreHandler() + ->DeserializeKVCatalogInfo(kv_info, offset); } txservice::KVCatalogInfo *GetKVCatalogInfo() const override diff --git a/storage/eloq/ha_eloq.cc b/storage/eloq/ha_eloq.cc index 93df9a1008f..8e2722438f5 100644 --- a/storage/eloq/ha_eloq.cc +++ b/storage/eloq/ha_eloq.cc @@ -211,12 +211,11 @@ bool data_substrate_init_done= false; using namespace MyEloq; using namespace txservice; -DEFINE_string(config, "", "Path to data substrate configuration file."); +DECLARE_string(eloq_data_path); MariaCatalogFactory maria_catalog_factory; txservice::CatalogFactory *eloqsql_catalog_factory= &maria_catalog_factory; -txservice::SystemHandler *eloqsql_system_handler= - &MyEloq::MariaSystemHandler::Instance(); +txservice::SystemHandler *eloqsql_system_handler= nullptr; extern my_bool opt_bootstrap; // Defined in Mariadb context. extern std::function terminate_hook; @@ -238,6 +237,7 @@ static handler *eloq_create_handler(handlerton *hton, TABLE_SHARE *table, handlerton *eloq_hton; +static char *eloq_config= nullptr; static char *eloq_insert_semantic= nullptr; static char *eloq_auto_increment= nullptr; static char *eloq_invalidate_cache_once= nullptr; @@ -280,6 +280,10 @@ static MYSQL_SYSVAR_ENUM(enum_var, // name 0, // def &enum_var_typelib); // typelib + +static MYSQL_SYSVAR_STR(config, eloq_config, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_MEMALLOC, + "Path to data substrate configuration file.", nullptr, nullptr, + ""); static MYSQL_SYSVAR_STR(insert_semantic, eloq_insert_semantic, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_MEMALLOC, "Insert semantic: insert or upsert", nullptr, nullptr, @@ -427,7 +431,7 @@ static struct st_mysql_sys_var *eloq_system_variables[]= { MYSQL_SYSVAR(varopt_default), MYSQL_SYSVAR(insert_semantic), MYSQL_SYSVAR(auto_increment), MYSQL_SYSVAR(invalidate_cache_once), MYSQL_SYSVAR(random_scan_sort), MYSQL_SYSVAR(report_debug_info), - MYSQL_SYSVAR(signal_monitor), NULL}; + MYSQL_SYSVAR(signal_monitor), MYSQL_SYSVAR(config), NULL}; /** Structure for CREATE TABLE options (table options). @@ -1451,6 +1455,7 @@ static int eloq_init_func(void *p) MY_MUTEX_INIT_FAST); mysql_mutex_init(mono_mem_cmp_space_mutex_key, &mono_mem_cmp_space_mutex, MY_MUTEX_INIT_FAST); + eloqsql_system_handler= &MyEloq::MariaSystemHandler::Instance(); #ifdef MYSQLD_LIBRARY_MODE // In library mode (converged binary), use synchronization: // 1. Signal that MySQL basic initialization is done @@ -1470,11 +1475,13 @@ static int eloq_init_func(void *p) LOG(INFO) << "Data substrate initialized, MySQL continuing..."; } #else + // Set the data substrate data path to mysql home directory in standalone mode. + FLAGS_eloq_data_path= mysql_real_data_home_ptr; // In standalone mode, initialize data substrate here // Wait for mysqld to initialize before initializing data substrate // as data substrate may need to access some mysqld variables during replay. LOG(INFO) << "Standalone mode: Initializing data substrate..."; - DataSubstrate::InitializeGlobal(FLAGS_config); + DataSubstrate::InitializeGlobal(eloq_config); #endif eloq_hton= (handlerton *) p; diff --git a/storage/eloq/mysql-test/mono_basic/data_substrate.cnf b/storage/eloq/mysql-test/mono_basic/data_substrate.cnf new file mode 100644 index 00000000000..cc45f001c37 --- /dev/null +++ b/storage/eloq/mysql-test/mono_basic/data_substrate.cnf @@ -0,0 +1,6 @@ +[local] +core_number=2 +enable_mvcc=false +deadlock_check_interval_seconds=10 +txlog_rocksdb_scan_threads=3 +hm_bin_path=../../../../data_substrate/host_manager \ No newline at end of file diff --git a/storage/eloq/mysql-test/mono_basic/include/have_range_partition.inc b/storage/eloq/mysql-test/mono_basic/include/have_range_partition.inc deleted file mode 100644 index 70578b199f3..00000000000 --- a/storage/eloq/mysql-test/mono_basic/include/have_range_partition.inc +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright (C) 2010 Sun Microsystems, Inc. -# Use is subject to license terms -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA - -if (!`SELECT count(*) FROM information_schema.GLOBAL_VARIABLES - WHERE VARIABLE_NAME='eloq_partition_type' - AND VARIABLE_VALUE='Range'`){ - skip Need range partition enabled; -} diff --git a/storage/eloq/mysql-test/mono_basic/my.cnf b/storage/eloq/mysql-test/mono_basic/my.cnf index d7eca0350bf..937f86997d9 100644 --- a/storage/eloq/mysql-test/mono_basic/my.cnf +++ b/storage/eloq/mysql-test/mono_basic/my.cnf @@ -2,33 +2,26 @@ # Use default setting for mysqld processes !include include/default_mysqld.cnf !include include/default_client.cnf -!include include/eloq_kv_storage.cnf + !include include/eloq_kv_keyspace.cnf -!include include/eloq_kv_dss.cnf + [mysqld] plugin-maturity=experimental default-storage-engine=eloq -plugin_load_add=ha_eloq thread_handling=pool-of-threads thread_pool_size=2 thread_pool_oversubscribe=10 thread_pool_dedicated_listener=1 table_open_cache_instances=20 -eloq skip-log-bin -eloq_core_num=2 eloq_cc_protocol=OCC -eloq_enable_mvcc=off -eloq_logserver_rocksdb_scan_thread_num=3 use_stat_tables=complementary -eloq_hm_bin_path = @ENV.MYSQLTEST_VARDIR/../../storage/eloq/host_manager +eloq_config = @ENV.MYSQLTEST_VARDIR/../../../storage/eloq/mysql-test/mono_basic/data_substrate.cnf [mysqld.1] -eloq_local_ip=127.0.0.1:8000 -eloq_ip_list=127.0.0.1:8000 -eloq_metrics_port=18081 + [ENV] MONO1_PORT= @mysqld.1.port diff --git a/storage/eloq/mysql-test/mono_basic/t/alter_table.test b/storage/eloq/mysql-test/mono_basic/t/alter_table.test index 24f596fccc2..4f21f1af795 100644 --- a/storage/eloq/mysql-test/mono_basic/t/alter_table.test +++ b/storage/eloq/mysql-test/mono_basic/t/alter_table.test @@ -1,5 +1,4 @@ --source include/have_eloq.inc ---source include/have_range_partition.inc --disable_warnings DROP DATABASE IF EXISTS mono; diff --git a/storage/eloq/mysql-test/mono_basic/t/alter_table_recovery.test b/storage/eloq/mysql-test/mono_basic/t/alter_table_recovery.test index fe16341481c..66484aa94e0 100644 --- a/storage/eloq/mysql-test/mono_basic/t/alter_table_recovery.test +++ b/storage/eloq/mysql-test/mono_basic/t/alter_table_recovery.test @@ -1,7 +1,6 @@ # --source include/have_eloq.inc --source include/long_test.inc ---source include/have_range_partition.inc --disable_warnings DROP DATABASE IF EXISTS mono_test; diff --git a/storage/eloq/mysql-test/mono_basic/t/create_drop_index.test b/storage/eloq/mysql-test/mono_basic/t/create_drop_index.test index 6067b75c4f8..5a3f8717dce 100644 --- a/storage/eloq/mysql-test/mono_basic/t/create_drop_index.test +++ b/storage/eloq/mysql-test/mono_basic/t/create_drop_index.test @@ -1,6 +1,5 @@ --source include/have_eloq.inc --source include/long_test.inc ---source include/have_range_partition.inc set @@default_storage_engine= eloq; # # test of left outer join (from join_outer.test) diff --git a/storage/eloq/mysql-test/mono_basic/t/data_store_error.test b/storage/eloq/mysql-test/mono_basic/t/data_store_error.test index 78f013dec4a..4ebf3cfcf60 100644 --- a/storage/eloq/mysql-test/mono_basic/t/data_store_error.test +++ b/storage/eloq/mysql-test/mono_basic/t/data_store_error.test @@ -1,5 +1,4 @@ --source include/have_eloq.inc ---source include/have_range_partition.inc set @@default_storage_engine= eloq; select @@default_storage_engine; diff --git a/storage/eloq/mysql-test/mono_basic/t/deadlock_handling.cnf b/storage/eloq/mysql-test/mono_basic/t/deadlock_handling.cnf index 8ed2f8801ce..844e5ffa0a0 100644 --- a/storage/eloq/mysql-test/mono_basic/t/deadlock_handling.cnf +++ b/storage/eloq/mysql-test/mono_basic/t/deadlock_handling.cnf @@ -2,26 +2,18 @@ # Use default setting for mysqld processes !include include/default_mysqld.cnf !include include/default_client.cnf -!include include/eloq_kv_storage.cnf + !include include/eloq_kv_keyspace.cnf -!include include/eloq_kv_dss.cnf + [mysqld] plugin-maturity=experimental -default-storage-engine=eloq -plugin_load_add=ha_eloq -eloq skip-log-bin -eloq_core_num=1 eloq_cc_protocol=LOCKING -eloq_enable_mvcc=off transaction-isolation=READ-COMMITTED -eloq_deadlock_interval_sec=10 -eloq_hm_bin_path = @ENV.MYSQLTEST_VARDIR/../../storage/eloq/host_manager +eloq_config = @ENV.MYSQLTEST_VARDIR/../../../storage/eloq/mysql-test/mono_basic/data_substrate.cnf [mysqld.1] -eloq_local_ip=127.0.0.1:8000 -eloq_ip_list=127.0.0.1:8000 [ENV] MONO1_PORT= @mysqld.1.port diff --git a/storage/eloq/mysql-test/mono_basic/t/flush_undo_while_ckpt.cnf b/storage/eloq/mysql-test/mono_basic/t/flush_undo_while_ckpt.cnf index b73f81de73c..a27a72e0b21 100644 --- a/storage/eloq/mysql-test/mono_basic/t/flush_undo_while_ckpt.cnf +++ b/storage/eloq/mysql-test/mono_basic/t/flush_undo_while_ckpt.cnf @@ -2,26 +2,17 @@ # Use default setting for mysqld processes !include include/default_mysqld.cnf !include include/default_client.cnf -!include include/eloq_kv_storage.cnf + !include include/eloq_kv_keyspace.cnf -!include include/eloq_kv_dss.cnf + [mysqld] plugin-maturity=experimental -default-storage-engine=eloq -plugin_load_add=ha_eloq -eloq skip-log-bin -eloq_core_num=1 -eloq_enable_mvcc=on eloq_cc_protocol=OccRead -eloq_checkpointer_interval_sec=10 -eloq_checkpointer_delay_sec=0 -eloq_hm_bin_path = @ENV.MYSQLTEST_VARDIR/../../storage/eloq/host_manager +eloq_config = @ENV.MYSQLTEST_VARDIR/../../../storage/eloq/mysql-test/mono_basic/t/flush_undo_while_ckpt_ds.cnf [mysqld.1] -eloq_local_ip=127.0.0.1:8000 -eloq_ip_list=127.0.0.1:8000 [ENV] MONO1_PORT= @mysqld.1.port diff --git a/storage/eloq/mysql-test/mono_basic/t/flush_undo_while_ckpt_ds.cnf b/storage/eloq/mysql-test/mono_basic/t/flush_undo_while_ckpt_ds.cnf new file mode 100644 index 00000000000..e5568e1c458 --- /dev/null +++ b/storage/eloq/mysql-test/mono_basic/t/flush_undo_while_ckpt_ds.cnf @@ -0,0 +1,8 @@ +[local] +core_number=1 +enable_mvcc=true +deadlock_check_interval_seconds=10 +checkpointer_interval=10 +checkpointer_delay_seconds=0 +txlog_rocksdb_scan_threads=3 +hm_bin_path=../../../../data_substrate/host_manager \ No newline at end of file diff --git a/storage/eloq/mysql-test/mono_basic/t/isolation_level_for_locking.cnf b/storage/eloq/mysql-test/mono_basic/t/isolation_level_for_locking.cnf index 69b3389dfa7..d3799ed598d 100644 --- a/storage/eloq/mysql-test/mono_basic/t/isolation_level_for_locking.cnf +++ b/storage/eloq/mysql-test/mono_basic/t/isolation_level_for_locking.cnf @@ -2,23 +2,17 @@ # Use default setting for mysqld processes !include include/default_mysqld.cnf !include include/default_client.cnf -!include include/eloq_kv_storage.cnf + !include include/eloq_kv_keyspace.cnf -!include include/eloq_kv_dss.cnf + [mysqld] plugin-maturity=experimental -default-storage-engine=eloq -plugin_load_add=ha_eloq -eloq skip-log-bin -eloq_core_num=1 eloq_cc_protocol=Locking -eloq_hm_bin_path = @ENV.MYSQLTEST_VARDIR/../../storage/eloq/host_manager +eloq_config = @ENV.MYSQLTEST_VARDIR/../../../storage/eloq/mysql-test/mono_basic/data_substrate.cnf [mysqld.1] -eloq_local_ip=127.0.0.1:8000 -eloq_ip_list=127.0.0.1:8000 [ENV] MONO1_PORT= @mysqld.1.port diff --git a/storage/eloq/mysql-test/mono_basic/t/isolation_level_for_mvcc.cnf b/storage/eloq/mysql-test/mono_basic/t/isolation_level_for_mvcc.cnf index 537115923f2..d6f6216ff43 100644 --- a/storage/eloq/mysql-test/mono_basic/t/isolation_level_for_mvcc.cnf +++ b/storage/eloq/mysql-test/mono_basic/t/isolation_level_for_mvcc.cnf @@ -2,24 +2,16 @@ # Use default setting for mysqld processes !include include/default_mysqld.cnf !include include/default_client.cnf -!include include/eloq_kv_storage.cnf + !include include/eloq_kv_keyspace.cnf -!include include/eloq_kv_dss.cnf + [mysqld] -plugin-maturity=experimental -default-storage-engine=eloq -plugin_load_add=ha_eloq -eloq skip-log-bin -eloq_core_num=1 -eloq_enable_mvcc=on eloq_cc_protocol=OCC -eloq_hm_bin_path = @ENV.MYSQLTEST_VARDIR/../../storage/eloq/host_manager +eloq_config = @ENV.MYSQLTEST_VARDIR/../../../storage/eloq/mysql-test/mono_basic/t/mvcc_ds.cnf [mysqld.1] -eloq_local_ip=127.0.0.1:8000 -eloq_ip_list=127.0.0.1:8000 [ENV] MONO1_PORT= @mysqld.1.port diff --git a/storage/eloq/mysql-test/mono_basic/t/isolation_level_for_occ.cnf b/storage/eloq/mysql-test/mono_basic/t/isolation_level_for_occ.cnf index 8e8bd7d19cc..5b63170fd4b 100644 --- a/storage/eloq/mysql-test/mono_basic/t/isolation_level_for_occ.cnf +++ b/storage/eloq/mysql-test/mono_basic/t/isolation_level_for_occ.cnf @@ -2,29 +2,23 @@ # Use default setting for mysqld processes !include include/default_mysqld.cnf !include include/default_client.cnf -!include include/eloq_kv_storage.cnf + !include include/eloq_kv_keyspace.cnf -!include include/eloq_kv_dss.cnf + [mysqld] -plugin-maturity=experimental -default-storage-engine=eloq -plugin_load_add=ha_eloq thread_handling=pool-of-threads -thread_pool_size=1 +default-storage-engine=eloq +thread_pool_size=2 thread_pool_oversubscribe=10 thread_pool_dedicated_listener=1 table_open_cache_instances=20 -eloq skip-log-bin -eloq_core_num=1 eloq_cc_protocol=OCC -eloq_enable_mvcc=off -eloq_hm_bin_path = @ENV.MYSQLTEST_VARDIR/../../storage/eloq/host_manager +eloq_config = @ENV.MYSQLTEST_VARDIR/../../../storage/eloq/mysql-test/mono_basic/data_substrate.cnf + [mysqld.1] -eloq_local_ip=127.0.0.1:8000 -eloq_ip_list=127.0.0.1:8000 [ENV] MONO1_PORT= @mysqld.1.port diff --git a/storage/eloq/mysql-test/mono_basic/t/key_cache.cnf b/storage/eloq/mysql-test/mono_basic/t/key_cache.cnf index 71f159b5bb7..52f8491005c 100644 --- a/storage/eloq/mysql-test/mono_basic/t/key_cache.cnf +++ b/storage/eloq/mysql-test/mono_basic/t/key_cache.cnf @@ -2,33 +2,25 @@ # Use default setting for mysqld processes !include include/default_mysqld.cnf !include include/default_client.cnf -!include include/eloq_kv_storage.cnf + !include include/eloq_kv_keyspace.cnf -!include include/eloq_kv_dss.cnf + [mysqld] plugin-maturity=experimental default-storage-engine=eloq -plugin_load_add=ha_eloq thread_handling=pool-of-threads thread_pool_size=2 thread_pool_oversubscribe=10 thread_pool_dedicated_listener=1 table_open_cache_instances=20 -eloq skip-log-bin -eloq_core_num=2 eloq_cc_protocol=OCC -eloq_enable_mvcc=off use_stat_tables=complementary -eloq_hm_bin_path = @ENV.MYSQLTEST_VARDIR/../../storage/eloq/host_manager -eloq_use_key_cache=true +eloq_config = @ENV.MYSQLTEST_VARDIR/../../../storage/eloq/mysql-test/mono_basic/t/key_cache_ds.cnf [mysqld.1] -eloq_local_ip=127.0.0.1:8000 -eloq_ip_list=127.0.0.1:8000 -eloq_metrics_port=18081 [ENV] MONO1_PORT= @mysqld.1.port diff --git a/storage/eloq/mysql-test/mono_basic/t/key_cache_ds.cnf b/storage/eloq/mysql-test/mono_basic/t/key_cache_ds.cnf new file mode 100644 index 00000000000..1d9c2d83659 --- /dev/null +++ b/storage/eloq/mysql-test/mono_basic/t/key_cache_ds.cnf @@ -0,0 +1,7 @@ +[local] +core_number=2 +enable_mvcc=false +deadlock_check_interval_seconds=10 +txlog_rocksdb_scan_threads=3 +hm_bin_path=../../../../data_substrate/host_manager +enable_key_cache=true \ No newline at end of file diff --git a/storage/eloq/mysql-test/mono_basic/t/mvcc_archives_recycle.cnf b/storage/eloq/mysql-test/mono_basic/t/mvcc_archives_recycle.cnf index d819778524a..d27f7a4182f 100644 --- a/storage/eloq/mysql-test/mono_basic/t/mvcc_archives_recycle.cnf +++ b/storage/eloq/mysql-test/mono_basic/t/mvcc_archives_recycle.cnf @@ -2,25 +2,19 @@ # Use default setting for mysqld processes !include include/default_mysqld.cnf !include include/default_client.cnf -!include include/eloq_kv_storage.cnf + !include include/eloq_kv_keyspace.cnf -!include include/eloq_kv_dss.cnf + [mysqld] plugin-maturity=experimental default-storage-engine=eloq -plugin_load_add=ha_eloq -eloq skip-log-bin -eloq_core_num=1 -eloq_enable_mvcc=on eloq_cc_protocol=OccRead -eloq_hm_bin_path = @ENV.MYSQLTEST_VARDIR/../../storage/eloq/host_manager +eloq_config = @ENV.MYSQLTEST_VARDIR/../../../storage/eloq/mysql-test/mono_basic/t/mvcc_ds.cnf [mysqld.1] -eloq_local_ip=127.0.0.1:8000 -eloq_ip_list=127.0.0.1:8000 [ENV] MONO1_PORT= @mysqld.1.port diff --git a/storage/eloq/mysql-test/mono_basic/t/mvcc_ds.cnf b/storage/eloq/mysql-test/mono_basic/t/mvcc_ds.cnf new file mode 100644 index 00000000000..da84dcc21e4 --- /dev/null +++ b/storage/eloq/mysql-test/mono_basic/t/mvcc_ds.cnf @@ -0,0 +1,6 @@ +[local] +core_number=2 +enable_mvcc=true +deadlock_check_interval_seconds=10 +txlog_rocksdb_scan_threads=3 +hm_bin_path=../../../../data_substrate/host_manager \ No newline at end of file diff --git a/storage/eloq/mysql-test/mono_basic/t/mvcc_read_after_flushing_deleted_record.cnf b/storage/eloq/mysql-test/mono_basic/t/mvcc_read_after_flushing_deleted_record.cnf index 667dd493c3e..2170ed61589 100644 --- a/storage/eloq/mysql-test/mono_basic/t/mvcc_read_after_flushing_deleted_record.cnf +++ b/storage/eloq/mysql-test/mono_basic/t/mvcc_read_after_flushing_deleted_record.cnf @@ -2,24 +2,18 @@ # Use default setting for mysqld processes !include include/default_mysqld.cnf !include include/default_client.cnf -!include include/eloq_kv_storage.cnf + !include include/eloq_kv_keyspace.cnf -!include include/eloq_kv_dss.cnf + [mysqld] plugin-maturity=experimental default-storage-engine=eloq -plugin_load_add=ha_eloq -eloq skip-log-bin -eloq_core_num=1 -eloq_enable_mvcc=on eloq_cc_protocol=OccRead -eloq_hm_bin_path = @ENV.MYSQLTEST_VARDIR/../../storage/eloq/host_manager +eloq_config = @ENV.MYSQLTEST_VARDIR/../../../storage/eloq/mysql-test/mono_basic/t/mvcc_ds.cnf [mysqld.1] -eloq_local_ip=127.0.0.1:8000 -eloq_ip_list=127.0.0.1:8000 [ENV] MONO1_PORT= @mysqld.1.port diff --git a/storage/eloq/mysql-test/mono_basic/t/mvcc_read_unique_sk_after_flushing_deleted_record.cnf b/storage/eloq/mysql-test/mono_basic/t/mvcc_read_unique_sk_after_flushing_deleted_record.cnf index 667dd493c3e..2170ed61589 100644 --- a/storage/eloq/mysql-test/mono_basic/t/mvcc_read_unique_sk_after_flushing_deleted_record.cnf +++ b/storage/eloq/mysql-test/mono_basic/t/mvcc_read_unique_sk_after_flushing_deleted_record.cnf @@ -2,24 +2,18 @@ # Use default setting for mysqld processes !include include/default_mysqld.cnf !include include/default_client.cnf -!include include/eloq_kv_storage.cnf + !include include/eloq_kv_keyspace.cnf -!include include/eloq_kv_dss.cnf + [mysqld] plugin-maturity=experimental default-storage-engine=eloq -plugin_load_add=ha_eloq -eloq skip-log-bin -eloq_core_num=1 -eloq_enable_mvcc=on eloq_cc_protocol=OccRead -eloq_hm_bin_path = @ENV.MYSQLTEST_VARDIR/../../storage/eloq/host_manager +eloq_config = @ENV.MYSQLTEST_VARDIR/../../../storage/eloq/mysql-test/mono_basic/t/mvcc_ds.cnf [mysqld.1] -eloq_local_ip=127.0.0.1:8000 -eloq_ip_list=127.0.0.1:8000 [ENV] MONO1_PORT= @mysqld.1.port diff --git a/storage/eloq/mysql-test/mono_basic/t/mvcc_scan.cnf b/storage/eloq/mysql-test/mono_basic/t/mvcc_scan.cnf index c19b418c579..73af9cb7957 100644 --- a/storage/eloq/mysql-test/mono_basic/t/mvcc_scan.cnf +++ b/storage/eloq/mysql-test/mono_basic/t/mvcc_scan.cnf @@ -3,23 +3,16 @@ !include include/default_mysqld.cnf !include include/default_client.cnf !include include/eloq_kv_keyspace.cnf -!include include/eloq_kv_storage.cnf -!include include/eloq_kv_dss.cnf + + [mysqld] -plugin-maturity=experimental -default-storage-engine=eloq -plugin_load_add=ha_eloq -eloq skip-log-bin -eloq_core_num=1 -eloq_enable_mvcc=on +default-storage-engine=eloq eloq_cc_protocol=OccRead -eloq_hm_bin_path = @ENV.MYSQLTEST_VARDIR/../../storage/eloq/host_manager +eloq_config = @ENV.MYSQLTEST_VARDIR/../../../storage/eloq/mysql-test/mono_basic/t/mvcc_ds.cnf [mysqld.1] -eloq_local_ip=127.0.0.1:8000 -eloq_ip_list=127.0.0.1:8000 [ENV] MONO1_PORT= @mysqld.1.port diff --git a/storage/eloq/mysql-test/mono_basic/t/mvcc_scan_after_flushing_deleted_record.cnf b/storage/eloq/mysql-test/mono_basic/t/mvcc_scan_after_flushing_deleted_record.cnf index 786da545843..16ee9705eba 100644 --- a/storage/eloq/mysql-test/mono_basic/t/mvcc_scan_after_flushing_deleted_record.cnf +++ b/storage/eloq/mysql-test/mono_basic/t/mvcc_scan_after_flushing_deleted_record.cnf @@ -2,23 +2,17 @@ # Use default setting for mysqld processes !include include/default_mysqld.cnf !include include/default_client.cnf -!include include/eloq_kv_storage.cnf + !include include/eloq_kv_keyspace.cnf -!include include/eloq_kv_dss.cnf + [mysqld] plugin-maturity=experimental default-storage-engine=eloq -plugin_load_add=ha_eloq -eloq skip-log-bin -eloq_core_num=1 -eloq_enable_mvcc=on -eloq_hm_bin_path = @ENV.MYSQLTEST_VARDIR/../../storage/eloq/host_manager +eloq_config = @ENV.MYSQLTEST_VARDIR/../../../storage/eloq/mysql-test/mono_basic/t/mvcc_ds.cnf [mysqld.1] -eloq_local_ip=127.0.0.1:8000 -eloq_ip_list=127.0.0.1:8000 [ENV] MONO1_PORT= @mysqld.1.port diff --git a/storage/eloq/mysql-test/mono_basic/t/mvcc_scan_sk.cnf b/storage/eloq/mysql-test/mono_basic/t/mvcc_scan_sk.cnf index 667dd493c3e..2170ed61589 100644 --- a/storage/eloq/mysql-test/mono_basic/t/mvcc_scan_sk.cnf +++ b/storage/eloq/mysql-test/mono_basic/t/mvcc_scan_sk.cnf @@ -2,24 +2,18 @@ # Use default setting for mysqld processes !include include/default_mysqld.cnf !include include/default_client.cnf -!include include/eloq_kv_storage.cnf + !include include/eloq_kv_keyspace.cnf -!include include/eloq_kv_dss.cnf + [mysqld] plugin-maturity=experimental default-storage-engine=eloq -plugin_load_add=ha_eloq -eloq skip-log-bin -eloq_core_num=1 -eloq_enable_mvcc=on eloq_cc_protocol=OccRead -eloq_hm_bin_path = @ENV.MYSQLTEST_VARDIR/../../storage/eloq/host_manager +eloq_config = @ENV.MYSQLTEST_VARDIR/../../../storage/eloq/mysql-test/mono_basic/t/mvcc_ds.cnf [mysqld.1] -eloq_local_ip=127.0.0.1:8000 -eloq_ip_list=127.0.0.1:8000 [ENV] MONO1_PORT= @mysqld.1.port diff --git a/storage/eloq/mysql-test/mono_basic/t/mvcc_scan_sk_rc_isolvl.cnf b/storage/eloq/mysql-test/mono_basic/t/mvcc_scan_sk_rc_isolvl.cnf index 667dd493c3e..3963a874155 100644 --- a/storage/eloq/mysql-test/mono_basic/t/mvcc_scan_sk_rc_isolvl.cnf +++ b/storage/eloq/mysql-test/mono_basic/t/mvcc_scan_sk_rc_isolvl.cnf @@ -2,9 +2,9 @@ # Use default setting for mysqld processes !include include/default_mysqld.cnf !include include/default_client.cnf -!include include/eloq_kv_storage.cnf + !include include/eloq_kv_keyspace.cnf -!include include/eloq_kv_dss.cnf + [mysqld] plugin-maturity=experimental diff --git a/storage/eloq/mysql-test/mono_basic/t/mvcc_unique_sk_lookup.cnf b/storage/eloq/mysql-test/mono_basic/t/mvcc_unique_sk_lookup.cnf index 667dd493c3e..2170ed61589 100644 --- a/storage/eloq/mysql-test/mono_basic/t/mvcc_unique_sk_lookup.cnf +++ b/storage/eloq/mysql-test/mono_basic/t/mvcc_unique_sk_lookup.cnf @@ -2,24 +2,18 @@ # Use default setting for mysqld processes !include include/default_mysqld.cnf !include include/default_client.cnf -!include include/eloq_kv_storage.cnf + !include include/eloq_kv_keyspace.cnf -!include include/eloq_kv_dss.cnf + [mysqld] plugin-maturity=experimental default-storage-engine=eloq -plugin_load_add=ha_eloq -eloq skip-log-bin -eloq_core_num=1 -eloq_enable_mvcc=on eloq_cc_protocol=OccRead -eloq_hm_bin_path = @ENV.MYSQLTEST_VARDIR/../../storage/eloq/host_manager +eloq_config = @ENV.MYSQLTEST_VARDIR/../../../storage/eloq/mysql-test/mono_basic/t/mvcc_ds.cnf [mysqld.1] -eloq_local_ip=127.0.0.1:8000 -eloq_ip_list=127.0.0.1:8000 [ENV] MONO1_PORT= @mysqld.1.port diff --git a/storage/eloq/mysql-test/mono_basic/t/poor_performance.cnf b/storage/eloq/mysql-test/mono_basic/t/poor_performance.cnf index a13387805fd..f51332ae63b 100644 --- a/storage/eloq/mysql-test/mono_basic/t/poor_performance.cnf +++ b/storage/eloq/mysql-test/mono_basic/t/poor_performance.cnf @@ -1,33 +1,23 @@ # Use default setting for mysqld processes !include include/default_mysqld.cnf !include include/default_client.cnf -!include include/eloq_kv_storage.cnf + !include include/eloq_kv_keyspace.cnf -!include include/eloq_kv_dss.cnf + [mysqld] default-storage-engine=eloq -plugin-maturity=experimental -default-storage-engine=eloq -plugin_load_add=ha_eloq thread_handling=pool-of-threads thread_pool_size=1 thread_pool_oversubscribe=10 thread_pool_dedicated_listener=1 table_open_cache_instances=20 -eloq skip-log-bin -eloq_core_num=1 -eloq_checkpointer_interval_sec=86400 eloq_cc_protocol=OCC -eloq_enable_mvcc=off use_stat_tables=complementary -eloq_hm_bin_path = @ENV.MYSQLTEST_VARDIR/../../storage/eloq/host_manager +eloq_config = @ENV.MYSQLTEST_VARDIR/../../../storage/eloq/mysql-test/mono_basic/t/poor_performance_ds.cnf [mysqld.1] -eloq_local_ip=127.0.0.1:8000 -eloq_ip_list=127.0.0.1:8000 -eloq_metrics_port=18081 [ENV] MONO1_PORT= @mysqld.1.port diff --git a/storage/eloq/mysql-test/mono_basic/t/poor_performance.test b/storage/eloq/mysql-test/mono_basic/t/poor_performance.test index e3d3fa72746..a34aa9dfe45 100644 --- a/storage/eloq/mysql-test/mono_basic/t/poor_performance.test +++ b/storage/eloq/mysql-test/mono_basic/t/poor_performance.test @@ -1,5 +1,4 @@ --source include/have_eloq.inc ---source include/have_range_partition.inc --source include/long_test.inc # diff --git a/storage/eloq/mysql-test/mono_basic/t/poor_performance_ds.cnf b/storage/eloq/mysql-test/mono_basic/t/poor_performance_ds.cnf new file mode 100644 index 00000000000..3219b96593a --- /dev/null +++ b/storage/eloq/mysql-test/mono_basic/t/poor_performance_ds.cnf @@ -0,0 +1,6 @@ +[local] +core_number=1 +enable_mvcc=false +txlog_rocksdb_scan_threads=3 +checkpointer_interval=86400 +hm_bin_path=../../../../data_substrate/host_manager \ No newline at end of file diff --git a/storage/eloq/mysql-test/mono_basic/t/range_split_keycache.cnf b/storage/eloq/mysql-test/mono_basic/t/range_split_keycache.cnf index eb02b44fef9..255de4dc419 100644 --- a/storage/eloq/mysql-test/mono_basic/t/range_split_keycache.cnf +++ b/storage/eloq/mysql-test/mono_basic/t/range_split_keycache.cnf @@ -2,34 +2,22 @@ # Use default setting for mysqld processes !include include/default_mysqld.cnf !include include/default_client.cnf -!include include/eloq_kv_storage.cnf + !include include/eloq_kv_keyspace.cnf -!include include/eloq_kv_dss.cnf + [mysqld] -plugin-maturity=experimental -default-storage-engine=eloq -plugin_load_add=ha_eloq thread_handling=pool-of-threads thread_pool_size=2 thread_pool_oversubscribe=10 thread_pool_dedicated_listener=1 table_open_cache_instances=20 -eloq skip-log-bin -eloq_core_num=2 -eloq_logserver_rocksdb_scan_thread_num=3 eloq_cc_protocol=OCC -eloq_enable_mvcc=off -use_stat_tables=complementary -eloq_hm_bin_path = @ENV.MYSQLTEST_VARDIR/../../storage/eloq/host_manager -eloq_use_key_cache=true +eloq_config = @ENV.MYSQLTEST_VARDIR/../../../storage/eloq/mysql-test/mono_basic/t/key_cache_ds.cnf [mysqld.1] -eloq_local_ip=127.0.0.1:8000 -eloq_ip_list=127.0.0.1:8000 -eloq_metrics_port=18081 [ENV] MONO1_PORT= @mysqld.1.port diff --git a/storage/eloq/mysql-test/mono_basic/t/range_split_keycache.test b/storage/eloq/mysql-test/mono_basic/t/range_split_keycache.test index 164f64f069c..8c597030a0f 100644 --- a/storage/eloq/mysql-test/mono_basic/t/range_split_keycache.test +++ b/storage/eloq/mysql-test/mono_basic/t/range_split_keycache.test @@ -1,5 +1,4 @@ --source include/have_eloq.inc ---source include/have_range_partition.inc --sleep 2 #Here is bug, If not sleep, the mysqld will not return result. --disable_warnings diff --git a/storage/eloq/mysql-test/mono_basic/t/select_for_update_and_share.cnf b/storage/eloq/mysql-test/mono_basic/t/select_for_update_and_share.cnf index 7d76f9ddef7..ceac07fb6ce 100644 --- a/storage/eloq/mysql-test/mono_basic/t/select_for_update_and_share.cnf +++ b/storage/eloq/mysql-test/mono_basic/t/select_for_update_and_share.cnf @@ -2,24 +2,18 @@ # Use default setting for mysqld processes !include include/default_mysqld.cnf !include include/default_client.cnf -!include include/eloq_kv_storage.cnf + !include include/eloq_kv_keyspace.cnf -!include include/eloq_kv_dss.cnf + [mysqld] plugin-maturity=experimental default-storage-engine=eloq -plugin_load_add=ha_eloq -eloq skip-log-bin -eloq_core_num=1 eloq_cc_protocol=OccRead -eloq_enable_mvcc=off -eloq_hm_bin_path = @ENV.MYSQLTEST_VARDIR/../../storage/eloq/host_manager +eloq_config = @ENV.MYSQLTEST_VARDIR/../../../storage/eloq/mysql-test/mono_basic/data_substrate.cnf [mysqld.1] -eloq_local_ip=127.0.0.1:8000 -eloq_ip_list=127.0.0.1:8000 [ENV] MONO1_PORT= @mysqld.1.port diff --git a/storage/eloq/mysql-test/mono_basic/t/select_for_update_and_share_occ.cnf b/storage/eloq/mysql-test/mono_basic/t/select_for_update_and_share_occ.cnf deleted file mode 100644 index a6fc6821781..00000000000 --- a/storage/eloq/mysql-test/mono_basic/t/select_for_update_and_share_occ.cnf +++ /dev/null @@ -1,26 +0,0 @@ - -# Use default setting for mysqld processes -!include include/default_mysqld.cnf -!include include/default_client.cnf -!include include/eloq_kv_storage.cnf -!include include/eloq_kv_keyspace.cnf -!include include/eloq_kv_dss.cnf - -[mysqld] -plugin-maturity=experimental -default-storage-engine=eloq -plugin_load_add=ha_eloq -eloq -skip-log-bin -eloq_core_num=1 -eloq_cc_protocol=OCC -eloq_enable_mvcc=off -eloq_hm_bin_path = @ENV.MYSQLTEST_VARDIR/../../storage/eloq/host_manager - -[mysqld.1] -eloq_local_ip=127.0.0.1:8000 -eloq_ip_list=127.0.0.1:8000 - -[ENV] -MONO1_PORT= @mysqld.1.port -MONO1_SOCK= @mysqld.1.socket diff --git a/storage/eloq/mysql-test/mono_basic/t/single_dead_lock_check.cnf b/storage/eloq/mysql-test/mono_basic/t/single_dead_lock_check.cnf index 1623f6e96ef..598bf638064 100644 --- a/storage/eloq/mysql-test/mono_basic/t/single_dead_lock_check.cnf +++ b/storage/eloq/mysql-test/mono_basic/t/single_dead_lock_check.cnf @@ -2,30 +2,21 @@ # Use default setting for mysqld processes !include include/default_mysqld.cnf !include include/default_client.cnf -!include include/eloq_kv_storage.cnf + !include include/eloq_kv_keyspace.cnf -!include include/eloq_kv_dss.cnf + [mysqld] -default-storage-engine=eloq -plugin-maturity=experimental thread_handling=pool-of-threads thread_pool_size=1 thread_pool_oversubscribe=1 thread_pool_dedicated_listener=1 table_open_cache_instances=20 -plugin_load_add=ha_eloq -eloq skip-log-bin -eloq_core_num=1 -eloq_cc_protocol=Occ -eloq_enable_mvcc=off eloq_cc_protocol=Locking -eloq_hm_bin_path = @ENV.MYSQLTEST_VARDIR/../../storage/eloq/host_manager +eloq_config = @ENV.MYSQLTEST_VARDIR/../../../storage/eloq/mysql-test/mono_basic/t/single_dead_lock_check_ds.cnf [mysqld.1] -eloq_local_ip=127.0.0.1:8000 -eloq_ip_list=127.0.0.1:8000 [ENV] MONO1_PORT= @mysqld.1.port diff --git a/storage/eloq/mysql-test/mono_basic/t/single_dead_lock_check_ds.cnf b/storage/eloq/mysql-test/mono_basic/t/single_dead_lock_check_ds.cnf new file mode 100644 index 00000000000..96212ae39d0 --- /dev/null +++ b/storage/eloq/mysql-test/mono_basic/t/single_dead_lock_check_ds.cnf @@ -0,0 +1,6 @@ +[local] +core_number=1 +enable_mvcc=false +deadlock_check_interval_seconds=10 +checkpointer_interval=10 +hm_bin_path=../../../../data_substrate/host_manager \ No newline at end of file diff --git a/storage/eloq/mysql-test/mono_basic/t/single_node.cnf b/storage/eloq/mysql-test/mono_basic/t/single_node.cnf deleted file mode 100644 index 026539218eb..00000000000 --- a/storage/eloq/mysql-test/mono_basic/t/single_node.cnf +++ /dev/null @@ -1,27 +0,0 @@ - -# Use default setting for mysqld processes -!include include/default_mysqld.cnf -!include include/default_client.cnf -!include include/eloq_kv_storage.cnf -!include include/eloq_kv_keyspace.cnf -!include include/eloq_kv_dss.cnf - -[mysqld] -default-storage-engine=eloq -plugin-maturity=experimental -default-storage-engine=eloq -plugin_load_add=ha_eloq -eloq -skip-log-bin -eloq_core_num=1 -eloq_cc_protocol=Occ -eloq_enable_mvcc=off -eloq_hm_bin_path = @ENV.MYSQLTEST_VARDIR/../../storage/eloq/host_manager - -[mysqld.1] -eloq_local_ip=127.0.0.1:8000 -eloq_ip_list=127.0.0.1:8000 - -[ENV] -MONO1_PORT= @mysqld.1.port -MONO1_SOCK= @mysqld.1.socket diff --git a/storage/eloq/mysql-test/mono_main/data_substrate.cnf b/storage/eloq/mysql-test/mono_main/data_substrate.cnf new file mode 100644 index 00000000000..bfa5825fdd4 --- /dev/null +++ b/storage/eloq/mysql-test/mono_main/data_substrate.cnf @@ -0,0 +1,6 @@ +[local] +core_number=2 +enable_mvcc=false +checkpointer_interval=86400 +txlog_rocksdb_scan_threads=3 +hm_bin_path=../../../../data_substrate/host_manager \ No newline at end of file diff --git a/storage/eloq/mysql-test/mono_main/my.cnf b/storage/eloq/mysql-test/mono_main/my.cnf index 8ec799bb850..0e79f14a136 100644 --- a/storage/eloq/mysql-test/mono_main/my.cnf +++ b/storage/eloq/mysql-test/mono_main/my.cnf @@ -2,37 +2,26 @@ # Use default setting for mysqld processes !include include/default_mysqld.cnf !include include/default_client.cnf -!include include/eloq_kv_storage.cnf + !include include/eloq_kv_keyspace.cnf -!include include/eloq_kv_dss.cnf + [mysqld] -plugin-maturity=experimental -plugin_load_add=ha_eloq thread_handling=pool-of-threads thread_pool_size=2 thread_pool_oversubscribe=10 thread_pool_dedicated_listener=1 table_open_cache_instances=20 -eloq -eloq_core_num=2 -eloq_logserver_rocksdb_scan_thread_num=3 default-storage-engine=eloq default-tmp-storage-engine=eloq collation-server=latin1_bin -eloq_checkpointer_interval_sec=86400 # NOTE: the value of this variable should be consistent with the value in bootstrap.test -eloq_ddl_skip_kv=on eloq_cc_protocol=OCC -eloq_enable_mvcc=off use_stat_tables=complementary -eloq_hm_bin_path = @ENV.MYSQLTEST_VARDIR/../../storage/eloq/host_manager +eloq_config = @ENV.MYSQLTEST_VARDIR/../../../storage/eloq/mysql-test/mono_main/data_substrate.cnf [mysqld.1] -eloq_local_ip=127.0.0.1:8000 -eloq_ip_list=127.0.0.1:8000 -eloq_metrics_port=18081 [ENV] MONO1_PORT= @mysqld.1.port diff --git a/storage/eloq/mysql-test/mono_main/r/system_mysql_db_error_log.result b/storage/eloq/mysql-test/mono_main/r/system_mysql_db_error_log.result index 5af3eda0700..94d81b99d61 100644 --- a/storage/eloq/mysql-test/mono_main/r/system_mysql_db_error_log.result +++ b/storage/eloq/mysql-test/mono_main/r/system_mysql_db_error_log.result @@ -112,7 +112,7 @@ SHOW GRANTS FOR good_version_id_100500@localhost; Grants for good_version_id_100500@localhost GRANT SUPER, SET USER, FEDERATED ADMIN, CONNECTION ADMIN, READ_ONLY ADMIN, REPLICATION SLAVE ADMIN, BINLOG ADMIN, BINLOG REPLAY ON *.* TO `good_version_id_100500`@`localhost` DROP USER good_version_id_100500@localhost; -FOUND 1 /Warning.*'user' entry 'bad_access1@localhost' has a wrong 'access' value.*version_id=/ in system_mysql_db_error_log.err -FOUND 1 /Warning.*'user' entry 'bad_version_id_1000000@localhost' has a wrong 'version_id' value 1000000/ in system_mysql_db_error_log.err -FOUND 1 /Warning.*'user' entry 'bad_version_id_minus_3@localhost' has a wrong 'version_id' value -3/ in system_mysql_db_error_log.err -FOUND 1 /Warning.*'user' entry 'bad_version_id_100300@localhost' has a wrong 'version_id' value 100300/ in system_mysql_db_error_log.err +FOUND 1 /WARNING.*'user' entry 'bad_access1@localhost' has a wrong 'access' value.*version_id=/ in system_mysql_db_error_log.err +FOUND 1 /WARNING.*'user' entry 'bad_version_id_1000000@localhost' has a wrong 'version_id' value 1000000/ in system_mysql_db_error_log.err +FOUND 1 /WARNING.*'user' entry 'bad_version_id_minus_3@localhost' has a wrong 'version_id' value -3/ in system_mysql_db_error_log.err +FOUND 1 /WARNING.*'user' entry 'bad_version_id_100300@localhost' has a wrong 'version_id' value 100300/ in system_mysql_db_error_log.err diff --git a/storage/eloq/mysql-test/mono_main/t/commit.opt b/storage/eloq/mysql-test/mono_main/t/commit.opt index c8924faf76c..66a6c43ba79 100644 --- a/storage/eloq/mysql-test/mono_main/t/commit.opt +++ b/storage/eloq/mysql-test/mono_main/t/commit.opt @@ -1,2 +1,2 @@ ---eloq_enable_mvcc=on +--enable_mvcc=true --eloq_cc_protocol=OccRead diff --git a/storage/eloq/mysql-test/mono_main/t/connect-abstract.cnf b/storage/eloq/mysql-test/mono_main/t/connect-abstract.cnf index e50915df444..df9f65a19de 100644 --- a/storage/eloq/mysql-test/mono_main/t/connect-abstract.cnf +++ b/storage/eloq/mysql-test/mono_main/t/connect-abstract.cnf @@ -1,25 +1,22 @@ !include include/default_my.cnf -!include include/eloq_kv_storage.cnf + !include include/eloq_kv_keyspace.cnf -!include include/eloq_kv_dss.cnf + [mysqld] plugin-maturity=experimental -plugin_load_add=ha_eloq -eloq skip-log-bin -eloq_core_num=1 default-storage-engine=eloq default-tmp-storage-engine=eloq +thread_handling=pool-of-threads +thread_pool_size=2 +thread_pool_oversubscribe=10 +thread_pool_dedicated_listener=1 collation-server=latin1_bin -eloq_checkpointer_interval_sec=86400 -eloq_ddl_skip_kv=on -eloq_hm_bin_path = @ENV.MYSQLTEST_VARDIR/../../storage/eloq/host_manager +eloq_config = @ENV.MYSQLTEST_VARDIR/../../../storage/eloq/mysql-test/mono_main/data_substrate.cnf [mysqld.1] -eloq_local_ip=127.0.0.1:8000 -eloq_ip_list=127.0.0.1:8000 # Using @OPT.port here for uniqueness [ENV] diff --git a/storage/eloq/mysql-test/mono_main/t/connect.cnf b/storage/eloq/mysql-test/mono_main/t/connect.cnf index 2eaa8c0fe9a..ac82c3859e8 100644 --- a/storage/eloq/mysql-test/mono_main/t/connect.cnf +++ b/storage/eloq/mysql-test/mono_main/t/connect.cnf @@ -1,26 +1,23 @@ !include include/default_my.cnf -!include include/eloq_kv_storage.cnf + !include include/eloq_kv_keyspace.cnf -!include include/eloq_kv_dss.cnf + [mysqld] plugin-maturity=experimental -plugin_load_add=ha_eloq -eloq skip-log-bin -eloq_core_num=1 default-storage-engine=eloq default-tmp-storage-engine=eloq +thread_handling=pool-of-threads +thread_pool_size=2 +thread_pool_oversubscribe=10 +thread_pool_dedicated_listener=1 collation-server=latin1_bin -eloq_checkpointer_interval_sec=86400 -eloq_ddl_skip_kv=on -eloq_hm_bin_path = @ENV.MYSQLTEST_VARDIR/../../storage/eloq/host_manager +eloq_config = @ENV.MYSQLTEST_VARDIR/../../../storage/eloq/mysql-test/mono_main/data_substrate.cnf [mysqld.1] extra-port= @ENV.MASTER_EXTRA_PORT extra-max-connections=1 -eloq_local_ip=127.0.0.1:8000 -eloq_ip_list=127.0.0.1:8000 [ENV] MASTER_EXTRA_PORT= @OPT.port diff --git a/storage/eloq/mysql-test/mono_main/t/connect2.cnf b/storage/eloq/mysql-test/mono_main/t/connect2.cnf index e0231a32836..74f50f9d4df 100644 --- a/storage/eloq/mysql-test/mono_main/t/connect2.cnf +++ b/storage/eloq/mysql-test/mono_main/t/connect2.cnf @@ -1,20 +1,15 @@ !include include/default_my.cnf -!include include/eloq_kv_storage.cnf + !include include/eloq_kv_keyspace.cnf -!include include/eloq_kv_dss.cnf + [mysqld] plugin-maturity=experimental -plugin_load_add=ha_eloq -eloq skip-log-bin -eloq_core_num=1 default-storage-engine=eloq default-tmp-storage-engine=eloq collation-server=latin1_bin -eloq_checkpointer_interval_sec=86400 -eloq_ddl_skip_kv=on -eloq_hm_bin_path = @ENV.MYSQLTEST_VARDIR/../../storage/eloq/host_manager +eloq_config = @ENV.MYSQLTEST_VARDIR/../../../storage/eloq/mysql-test/mono_main/data_substrate.cnf [mysqld.1] extra-port= @ENV.MASTER_EXTRA_PORT @@ -22,8 +17,6 @@ extra-max-connections=2 thread_handling=pool-of-threads thread_pool_size=1 thread_pool_dedicated_listener=1 -eloq_local_ip=127.0.0.1:8000 -eloq_ip_list=127.0.0.1:8000 [ENV] MASTER_EXTRA_PORT= @OPT.port diff --git a/storage/eloq/mysql-test/mono_main/t/connect2.opt b/storage/eloq/mysql-test/mono_main/t/connect2.opt new file mode 100644 index 00000000000..23109bca354 --- /dev/null +++ b/storage/eloq/mysql-test/mono_main/t/connect2.opt @@ -0,0 +1 @@ +--core_number=1 \ No newline at end of file diff --git a/storage/eloq/mysql-test/mono_main/t/consistent_snapshot.opt b/storage/eloq/mysql-test/mono_main/t/consistent_snapshot.opt index cb3674d1d02..f10baca9e49 100644 --- a/storage/eloq/mysql-test/mono_main/t/consistent_snapshot.opt +++ b/storage/eloq/mysql-test/mono_main/t/consistent_snapshot.opt @@ -1,2 +1,2 @@ ---eloq_enable_mvcc=on +--enable_mvcc=true --eloq_cc_protocol=OCC \ No newline at end of file diff --git a/storage/eloq/mysql-test/mono_main/t/pool_of_threads.cnf b/storage/eloq/mysql-test/mono_main/t/pool_of_threads.cnf index 69393f6870a..2dd914b587e 100644 --- a/storage/eloq/mysql-test/mono_main/t/pool_of_threads.cnf +++ b/storage/eloq/mysql-test/mono_main/t/pool_of_threads.cnf @@ -1,20 +1,15 @@ !include include/default_my.cnf -!include include/eloq_kv_storage.cnf + !include include/eloq_kv_keyspace.cnf -!include include/eloq_kv_dss.cnf + [mysqld] plugin-maturity=experimental -plugin_load_add=ha_eloq -eloq skip-log-bin -eloq_core_num=1 default-storage-engine=eloq default-tmp-storage-engine=eloq collation-server=latin1_bin -eloq_checkpointer_interval_sec=86400 -eloq_ddl_skip_kv=on -eloq_hm_bin_path = @ENV.MYSQLTEST_VARDIR/../../storage/eloq/host_manager +eloq_config = @ENV.MYSQLTEST_VARDIR/../../../storage/eloq/mysql-test/mono_main/data_substrate.cnf [mysqld.1] loose-thread-handling= pool-of-threads @@ -22,8 +17,6 @@ loose-thread_pool_size= 2 loose-thread_pool_max_threads= 2 extra-port= @ENV.MASTER_EXTRA_PORT extra-max-connections=1 -eloq_local_ip=127.0.0.1:8000 -eloq_ip_list=127.0.0.1:8000 [ENV] MASTER_EXTRA_PORT= @OPT.port diff --git a/storage/eloq/mysql-test/mono_main/t/pool_of_threads.opt b/storage/eloq/mysql-test/mono_main/t/pool_of_threads.opt new file mode 100644 index 00000000000..23109bca354 --- /dev/null +++ b/storage/eloq/mysql-test/mono_main/t/pool_of_threads.opt @@ -0,0 +1 @@ +--core_number=1 \ No newline at end of file diff --git a/storage/eloq/mysql-test/mono_main/t/sp_notembedded.opt b/storage/eloq/mysql-test/mono_main/t/sp_notembedded.opt index c8924faf76c..66a6c43ba79 100644 --- a/storage/eloq/mysql-test/mono_main/t/sp_notembedded.opt +++ b/storage/eloq/mysql-test/mono_main/t/sp_notembedded.opt @@ -1,2 +1,2 @@ ---eloq_enable_mvcc=on +--enable_mvcc=true --eloq_cc_protocol=OccRead diff --git a/storage/eloq/mysql-test/mono_main/t/subselect_exists2in_costmat.cnf b/storage/eloq/mysql-test/mono_main/t/subselect_exists2in_costmat.cnf index 7dc932b8130..0b4e7d6904c 100644 --- a/storage/eloq/mysql-test/mono_main/t/subselect_exists2in_costmat.cnf +++ b/storage/eloq/mysql-test/mono_main/t/subselect_exists2in_costmat.cnf @@ -2,28 +2,21 @@ # Use default setting for mysqld processes !include include/default_mysqld.cnf !include include/default_client.cnf -!include include/eloq_kv_storage.cnf + !include include/eloq_kv_keyspace.cnf -!include include/eloq_kv_dss.cnf + [mysqld] plugin-maturity=experimental -plugin_load_add=ha_eloq -eloq skip-log-bin -eloq_core_num=1 default-storage-engine=eloq default-tmp-storage-engine=eloq collation-server=latin1_bin eloq_cc_protocol=OCC -eloq_enable_mvcc=off use_stat_tables=complementary -eloq_hm_bin_path = @ENV.MYSQLTEST_VARDIR/../../storage/eloq/host_manager +eloq_config = @ENV.MYSQLTEST_VARDIR/../../../storage/eloq/mysql-test/mono_main/data_substrate.cnf [mysqld.1] -eloq_local_ip=127.0.0.1:8000 -eloq_ip_list=127.0.0.1:8000 -eloq_metrics_port=18081 [ENV] MONO1_PORT= @mysqld.1.port diff --git a/storage/eloq/mysql-test/mono_main/t/subselect_mat_cost.cnf b/storage/eloq/mysql-test/mono_main/t/subselect_mat_cost.cnf index 7dc932b8130..0b4e7d6904c 100644 --- a/storage/eloq/mysql-test/mono_main/t/subselect_mat_cost.cnf +++ b/storage/eloq/mysql-test/mono_main/t/subselect_mat_cost.cnf @@ -2,28 +2,21 @@ # Use default setting for mysqld processes !include include/default_mysqld.cnf !include include/default_client.cnf -!include include/eloq_kv_storage.cnf + !include include/eloq_kv_keyspace.cnf -!include include/eloq_kv_dss.cnf + [mysqld] plugin-maturity=experimental -plugin_load_add=ha_eloq -eloq skip-log-bin -eloq_core_num=1 default-storage-engine=eloq default-tmp-storage-engine=eloq collation-server=latin1_bin eloq_cc_protocol=OCC -eloq_enable_mvcc=off use_stat_tables=complementary -eloq_hm_bin_path = @ENV.MYSQLTEST_VARDIR/../../storage/eloq/host_manager +eloq_config = @ENV.MYSQLTEST_VARDIR/../../../storage/eloq/mysql-test/mono_main/data_substrate.cnf [mysqld.1] -eloq_local_ip=127.0.0.1:8000 -eloq_ip_list=127.0.0.1:8000 -eloq_metrics_port=18081 [ENV] MONO1_PORT= @mysqld.1.port diff --git a/storage/eloq/mysql-test/mono_main/t/system_mysql_db_error_log.test b/storage/eloq/mysql-test/mono_main/t/system_mysql_db_error_log.test index 17c04a9cc14..3404431a99b 100644 --- a/storage/eloq/mysql-test/mono_main/t/system_mysql_db_error_log.test +++ b/storage/eloq/mysql-test/mono_main/t/system_mysql_db_error_log.test @@ -104,14 +104,14 @@ DROP USER good_version_id_100500@localhost; --let SEARCH_FILE=$MYSQLTEST_VARDIR/tmp/system_mysql_db_error_log.err ---let SEARCH_PATTERN= Warning.*'user' entry 'bad_access1@localhost' has a wrong 'access' value.*version_id= +--let SEARCH_PATTERN= WARNING.*'user' entry 'bad_access1@localhost' has a wrong 'access' value.*version_id= --source include/search_pattern_in_file.inc ---let SEARCH_PATTERN= Warning.*'user' entry 'bad_version_id_1000000@localhost' has a wrong 'version_id' value 1000000 +--let SEARCH_PATTERN= WARNING.*'user' entry 'bad_version_id_1000000@localhost' has a wrong 'version_id' value 1000000 --source include/search_pattern_in_file.inc ---let SEARCH_PATTERN= Warning.*'user' entry 'bad_version_id_minus_3@localhost' has a wrong 'version_id' value -3 +--let SEARCH_PATTERN= WARNING.*'user' entry 'bad_version_id_minus_3@localhost' has a wrong 'version_id' value -3 --source include/search_pattern_in_file.inc ---let SEARCH_PATTERN= Warning.*'user' entry 'bad_version_id_100300@localhost' has a wrong 'version_id' value 100300 +--let SEARCH_PATTERN= WARNING.*'user' entry 'bad_version_id_100300@localhost' has a wrong 'version_id' value 100300 --source include/search_pattern_in_file.inc diff --git a/storage/eloq/mysql-test/mono_multi/data_substrate1.cnf b/storage/eloq/mysql-test/mono_multi/data_substrate1.cnf new file mode 100644 index 00000000000..616153b6952 --- /dev/null +++ b/storage/eloq/mysql-test/mono_multi/data_substrate1.cnf @@ -0,0 +1,13 @@ +[local] +core_number=2 +enable_mvcc=false +txlog_rocksdb_scan_threads=3 +hm_bin_path=../../../../data_substrate/host_manager +tx_ip=127.0.0.1 +tx_port=8000 + +[cluster] +ip_port_list=127.0.0.1:8000,127.0.0.1:8010,127.0.0.1:8020 + +[store] +eloq_dss_peer_node=127.0.0.1:9100 \ No newline at end of file diff --git a/storage/eloq/mysql-test/mono_multi/data_substrate2.cnf b/storage/eloq/mysql-test/mono_multi/data_substrate2.cnf new file mode 100644 index 00000000000..b11d749afd4 --- /dev/null +++ b/storage/eloq/mysql-test/mono_multi/data_substrate2.cnf @@ -0,0 +1,13 @@ +[local] +core_number=2 +enable_mvcc=false +txlog_rocksdb_scan_threads=3 +hm_bin_path=../../../../data_substrate/host_manager +tx_ip=127.0.0.1 +tx_port=8010 + +[cluster] +ip_port_list=127.0.0.1:8000,127.0.0.1:8010,127.0.0.1:8020 + +[store] +eloq_dss_peer_node=127.0.0.1:9100 \ No newline at end of file diff --git a/storage/eloq/mysql-test/mono_multi/data_substrate3.cnf b/storage/eloq/mysql-test/mono_multi/data_substrate3.cnf new file mode 100644 index 00000000000..98d27990923 --- /dev/null +++ b/storage/eloq/mysql-test/mono_multi/data_substrate3.cnf @@ -0,0 +1,12 @@ +[local] +core_number=2 +enable_mvcc=false +txlog_rocksdb_scan_threads=3 +hm_bin_path=../../../../data_substrate/host_manager +tx_ip=127.0.0.1 +tx_port=8020 + +[cluster] +ip_port_list=127.0.0.1:8000,127.0.0.1:8010,127.0.0.1:8020 +[store] +eloq_dss_peer_node=127.0.0.1:9100 \ No newline at end of file diff --git a/storage/eloq/mysql-test/mono_multi/my.cnf b/storage/eloq/mysql-test/mono_multi/my.cnf index f658cab8fee..f9954465798 100644 --- a/storage/eloq/mysql-test/mono_multi/my.cnf +++ b/storage/eloq/mysql-test/mono_multi/my.cnf @@ -2,42 +2,31 @@ # Use default setting for mysqld processes !include include/default_mysqld.cnf !include include/default_client.cnf -!include include/eloq_kv_storage.cnf + !include include/eloq_kv_keyspace.cnf -!include include/eloq_kv_dss.cnf + [mysqld] performance_schema=OFF default_storage_engine=eloq plugin-maturity=experimental -plugin_load_add=ha_eloq skip-log-bin thread_handling=pool-of-threads thread_pool_size=2 thread_pool_dedicated_listener=1 table_open_cache_instances=20 -eloq_core_num=2 -eloq_logserver_rocksdb_scan_thread_num=3 eloq_cc_protocol=OCC -eloq_enable_mvcc=off use_stat_tables=complementary -eloq_hm_bin_path = @ENV.MYSQLTEST_VARDIR/../../storage/eloq/host_manager [mysqld.1] -eloq_local_ip=127.0.0.1:8000 -eloq_ip_list=127.0.0.1:8000,127.0.0.1:8010,127.0.0.1:8020 -eloq_metrics_port=18081 +eloq_config = @ENV.MYSQLTEST_VARDIR/../../../storage/eloq/mysql-test/mono_multi/data_substrate1.cnf [mysqld.2] -eloq_local_ip=127.0.0.1:8010 -eloq_ip_list=127.0.0.1:8000,127.0.0.1:8010,127.0.0.1:8020 -eloq_metrics_port=18082 +eloq_config = @ENV.MYSQLTEST_VARDIR/../../../storage/eloq/mysql-test/mono_multi/data_substrate2.cnf [mysqld.3] -eloq_local_ip=127.0.0.1:8020 -eloq_ip_list=127.0.0.1:8000,127.0.0.1:8010,127.0.0.1:8020 -eloq_metrics_port=18083 +eloq_config = @ENV.MYSQLTEST_VARDIR/../../../storage/eloq/mysql-test/mono_multi/data_substrate3.cnf [ENV] MONO_PORT_1= @mysqld.1.port diff --git a/storage/eloq/mysql-test/mono_multi/t/alter_schema_term_changed.test b/storage/eloq/mysql-test/mono_multi/t/alter_schema_term_changed.test index f708566d939..62f843fee93 100644 --- a/storage/eloq/mysql-test/mono_multi/t/alter_schema_term_changed.test +++ b/storage/eloq/mysql-test/mono_multi/t/alter_schema_term_changed.test @@ -1,5 +1,4 @@ --source include/have_eloq.inc ---source include/have_range_partition.inc set @@default_storage_engine=eloq; --let $mono_server_count=3 diff --git a/storage/eloq/mysql-test/mono_multi/t/check_blocked_request.cnf b/storage/eloq/mysql-test/mono_multi/t/check_blocked_request.cnf deleted file mode 100644 index 6757dd6a869..00000000000 --- a/storage/eloq/mysql-test/mono_multi/t/check_blocked_request.cnf +++ /dev/null @@ -1,46 +0,0 @@ - -# Use default setting for mysqld processes -!include include/default_mysqld.cnf -!include include/default_client.cnf -!include include/eloq_kv_storage.cnf -!include include/eloq_kv_keyspace.cnf -!include include/eloq_kv_dss.cnf - -[mysqld] -default-storage-engine=eloq -plugin-maturity=experimental -default-storage-engine=eloq -plugin_load_add=ha_eloq -eloq -skip-log-bin -thread_handling=pool-of-threads -thread_pool_size=1 -thread_pool_dedicated_listener=1 -table_open_cache_instances=20 -eloq_core_num=1 -eloq_cc_protocol=Occ -eloq_enable_mvcc=off -eloq_cc_protocol=Locking -eloq_hm_bin_path = @ENV.MYSQLTEST_VARDIR/../../storage/eloq/host_manager - -[mysqld.1] -eloq_local_ip=127.0.0.1:8000 -eloq_ip_list=127.0.0.1:8000,127.0.0.1:8010,127.0.0.1:8020 - -[mysqld.2] -eloq_local_ip=127.0.0.1:8010 -eloq_ip_list=127.0.0.1:8000,127.0.0.1:8010,127.0.0.1:8020 - -[mysqld.3] -eloq_local_ip=127.0.0.1:8020 -eloq_ip_list=127.0.0.1:8000,127.0.0.1:8010,127.0.0.1:8020 - -[ENV] -MONO_PORT_1= @mysqld.1.port -MONO_SOCK_1= @mysqld.1.socket - -MONO_PORT_2= @mysqld.2.port -MONO_SOCK_2= @mysqld.2.socket - -MONO_PORT_3= @mysqld.3.port -MONO_SOCK_3= @mysqld.3.socket diff --git a/storage/eloq/mysql-test/mono_multi/t/check_blocked_request.opt b/storage/eloq/mysql-test/mono_multi/t/check_blocked_request.opt new file mode 100644 index 00000000000..c86fab7c7cb --- /dev/null +++ b/storage/eloq/mysql-test/mono_multi/t/check_blocked_request.opt @@ -0,0 +1 @@ +--thread_pool_size=1 --core_number=1 --eloq_cc_protocol=Locking \ No newline at end of file diff --git a/storage/eloq/mysql-test/mono_multi/t/create_drop_index.test b/storage/eloq/mysql-test/mono_multi/t/create_drop_index.test index 3d045b2f2a5..39e51798983 100644 --- a/storage/eloq/mysql-test/mono_multi/t/create_drop_index.test +++ b/storage/eloq/mysql-test/mono_multi/t/create_drop_index.test @@ -1,5 +1,4 @@ --source include/have_eloq.inc ---source include/have_range_partition.inc --let $mono_server_count= 3 --source include/mono_init.inc diff --git a/storage/eloq/mysql-test/mono_multi/t/fault_create_drop_index.test b/storage/eloq/mysql-test/mono_multi/t/fault_create_drop_index.test index 3c3c658b119..154de1bf67a 100644 --- a/storage/eloq/mysql-test/mono_multi/t/fault_create_drop_index.test +++ b/storage/eloq/mysql-test/mono_multi/t/fault_create_drop_index.test @@ -1,6 +1,5 @@ --source include/have_eloq.inc --source include/long_test.inc ---source include/have_range_partition.inc # Skip for dynamodb temporary. if (!`SELECT count(*) FROM information_schema.GLOBAL_VARIABLES WHERE VARIABLE_NAME='eloq_kv_storage' diff --git a/storage/eloq/mysql-test/mono_multi/t/improve_ckpt_flush_undo.cnf b/storage/eloq/mysql-test/mono_multi/t/improve_ckpt_flush_undo.cnf deleted file mode 100644 index 819b0699c59..00000000000 --- a/storage/eloq/mysql-test/mono_multi/t/improve_ckpt_flush_undo.cnf +++ /dev/null @@ -1,46 +0,0 @@ - -# Use default setting for mysqld processes -!include include/default_mysqld.cnf -!include include/default_client.cnf -!include include/eloq_kv_storage.cnf -!include include/eloq_kv_keyspace.cnf -!include include/eloq_kv_dss.cnf - -[mysqld] -default_storage_engine=eloq -plugin-maturity=experimental -plugin_load_add=ha_eloq -eloq -skip-log-bin -thread_handling=pool-of-threads -thread_pool_size=1 -thread_pool_dedicated_listener=1 -table_open_cache_instances=20 -eloq_core_num=1 -eloq_cc_protocol=OCC -eloq_enable_mvcc=on -eloq_checkpointer_delay_sec=0 -eloq_hm_bin_path = @ENV.MYSQLTEST_VARDIR/../../storage/eloq/host_manager - -[mysqld.1] -eloq_local_ip=127.0.0.1:8000 -eloq_ip_list=127.0.0.1:8000,127.0.0.1:8010,127.0.0.1:8020 - -[mysqld.2] -eloq_local_ip=127.0.0.1:8010 -eloq_ip_list=127.0.0.1:8000,127.0.0.1:8010,127.0.0.1:8020 - -[mysqld.3] -eloq_local_ip=127.0.0.1:8020 -eloq_ip_list=127.0.0.1:8000,127.0.0.1:8010,127.0.0.1:8020 - -[ENV] -MONO_PORT_1= @mysqld.1.port -MONO_SOCK_1= @mysqld.1.socket - -MONO_PORT_2= @mysqld.2.port -MONO_SOCK_2= @mysqld.2.socket - -MONO_PORT_3= @mysqld.3.port -MONO_SOCK_3= @mysqld.3.socket - diff --git a/storage/eloq/mysql-test/mono_multi/t/improve_ckpt_flush_undo.opt b/storage/eloq/mysql-test/mono_multi/t/improve_ckpt_flush_undo.opt new file mode 100644 index 00000000000..712bf69a313 --- /dev/null +++ b/storage/eloq/mysql-test/mono_multi/t/improve_ckpt_flush_undo.opt @@ -0,0 +1 @@ +--enable_mvcc=true --thread_pool_size=1 --core_number=1 --checkpointer_delay_seconds=0 \ No newline at end of file diff --git a/storage/eloq/mysql-test/mono_multi/t/improve_ckpt_flush_undo_sk1.cnf b/storage/eloq/mysql-test/mono_multi/t/improve_ckpt_flush_undo_sk1.cnf deleted file mode 100644 index 819b0699c59..00000000000 --- a/storage/eloq/mysql-test/mono_multi/t/improve_ckpt_flush_undo_sk1.cnf +++ /dev/null @@ -1,46 +0,0 @@ - -# Use default setting for mysqld processes -!include include/default_mysqld.cnf -!include include/default_client.cnf -!include include/eloq_kv_storage.cnf -!include include/eloq_kv_keyspace.cnf -!include include/eloq_kv_dss.cnf - -[mysqld] -default_storage_engine=eloq -plugin-maturity=experimental -plugin_load_add=ha_eloq -eloq -skip-log-bin -thread_handling=pool-of-threads -thread_pool_size=1 -thread_pool_dedicated_listener=1 -table_open_cache_instances=20 -eloq_core_num=1 -eloq_cc_protocol=OCC -eloq_enable_mvcc=on -eloq_checkpointer_delay_sec=0 -eloq_hm_bin_path = @ENV.MYSQLTEST_VARDIR/../../storage/eloq/host_manager - -[mysqld.1] -eloq_local_ip=127.0.0.1:8000 -eloq_ip_list=127.0.0.1:8000,127.0.0.1:8010,127.0.0.1:8020 - -[mysqld.2] -eloq_local_ip=127.0.0.1:8010 -eloq_ip_list=127.0.0.1:8000,127.0.0.1:8010,127.0.0.1:8020 - -[mysqld.3] -eloq_local_ip=127.0.0.1:8020 -eloq_ip_list=127.0.0.1:8000,127.0.0.1:8010,127.0.0.1:8020 - -[ENV] -MONO_PORT_1= @mysqld.1.port -MONO_SOCK_1= @mysqld.1.socket - -MONO_PORT_2= @mysqld.2.port -MONO_SOCK_2= @mysqld.2.socket - -MONO_PORT_3= @mysqld.3.port -MONO_SOCK_3= @mysqld.3.socket - diff --git a/storage/eloq/mysql-test/mono_multi/t/improve_ckpt_flush_undo_sk1.opt b/storage/eloq/mysql-test/mono_multi/t/improve_ckpt_flush_undo_sk1.opt new file mode 100644 index 00000000000..9cb02da304e --- /dev/null +++ b/storage/eloq/mysql-test/mono_multi/t/improve_ckpt_flush_undo_sk1.opt @@ -0,0 +1 @@ +--thread_pool_size=1 --core_number=1 --enable_mvcc=true --checkpointer_delay_seconds=0 \ No newline at end of file diff --git a/storage/eloq/mysql-test/mono_multi/t/improve_ckpt_flush_undo_sk2.cnf b/storage/eloq/mysql-test/mono_multi/t/improve_ckpt_flush_undo_sk2.cnf deleted file mode 100644 index 819b0699c59..00000000000 --- a/storage/eloq/mysql-test/mono_multi/t/improve_ckpt_flush_undo_sk2.cnf +++ /dev/null @@ -1,46 +0,0 @@ - -# Use default setting for mysqld processes -!include include/default_mysqld.cnf -!include include/default_client.cnf -!include include/eloq_kv_storage.cnf -!include include/eloq_kv_keyspace.cnf -!include include/eloq_kv_dss.cnf - -[mysqld] -default_storage_engine=eloq -plugin-maturity=experimental -plugin_load_add=ha_eloq -eloq -skip-log-bin -thread_handling=pool-of-threads -thread_pool_size=1 -thread_pool_dedicated_listener=1 -table_open_cache_instances=20 -eloq_core_num=1 -eloq_cc_protocol=OCC -eloq_enable_mvcc=on -eloq_checkpointer_delay_sec=0 -eloq_hm_bin_path = @ENV.MYSQLTEST_VARDIR/../../storage/eloq/host_manager - -[mysqld.1] -eloq_local_ip=127.0.0.1:8000 -eloq_ip_list=127.0.0.1:8000,127.0.0.1:8010,127.0.0.1:8020 - -[mysqld.2] -eloq_local_ip=127.0.0.1:8010 -eloq_ip_list=127.0.0.1:8000,127.0.0.1:8010,127.0.0.1:8020 - -[mysqld.3] -eloq_local_ip=127.0.0.1:8020 -eloq_ip_list=127.0.0.1:8000,127.0.0.1:8010,127.0.0.1:8020 - -[ENV] -MONO_PORT_1= @mysqld.1.port -MONO_SOCK_1= @mysqld.1.socket - -MONO_PORT_2= @mysqld.2.port -MONO_SOCK_2= @mysqld.2.socket - -MONO_PORT_3= @mysqld.3.port -MONO_SOCK_3= @mysqld.3.socket - diff --git a/storage/eloq/mysql-test/mono_multi/t/improve_ckpt_flush_undo_sk2.opt b/storage/eloq/mysql-test/mono_multi/t/improve_ckpt_flush_undo_sk2.opt new file mode 100644 index 00000000000..9cb02da304e --- /dev/null +++ b/storage/eloq/mysql-test/mono_multi/t/improve_ckpt_flush_undo_sk2.opt @@ -0,0 +1 @@ +--thread_pool_size=1 --core_number=1 --enable_mvcc=true --checkpointer_delay_seconds=0 \ No newline at end of file diff --git a/storage/eloq/mysql-test/mono_multi/t/multi_dead_lock_check.cnf b/storage/eloq/mysql-test/mono_multi/t/multi_dead_lock_check.cnf deleted file mode 100644 index fdb49b8dfa3..00000000000 --- a/storage/eloq/mysql-test/mono_multi/t/multi_dead_lock_check.cnf +++ /dev/null @@ -1,48 +0,0 @@ - -# Use default setting for mysqld processes -!include include/default_mysqld.cnf -!include include/default_client.cnf -!include include/eloq_kv_storage.cnf -!include include/eloq_kv_keyspace.cnf -!include include/eloq_kv_dss.cnf - -[mysqld] -default-storage-engine=eloq -plugin-maturity=experimental -default-storage-engine=eloq -plugin_load_add=ha_eloq -eloq -skip-log-bin -thread_handling=pool-of-threads -thread_pool_size=1 -thread_pool_dedicated_listener=1 -table_open_cache_instances=20 -eloq_core_num=1 -eloq_enable_mvcc=off -eloq_cc_protocol=Locking -eloq_hm_bin_path = @ENV.MYSQLTEST_VARDIR/../../storage/eloq/host_manager -# Trigger a deadlock check every 15 seconds to ensure detection occurs between ReRunOp retries during transaction timeouts. -# This guarantees the deadlock is identified before ReRunOp exhausts its 3 retry attempts and forces an error. -eloq_deadlock_interval_sec=15 - -[mysqld.1] -eloq_local_ip=127.0.0.1:8000 -eloq_ip_list=127.0.0.1:8000,127.0.0.1:8010,127.0.0.1:8020 - -[mysqld.2] -eloq_local_ip=127.0.0.1:8010 -eloq_ip_list=127.0.0.1:8000,127.0.0.1:8010,127.0.0.1:8020 - -[mysqld.3] -eloq_local_ip=127.0.0.1:8020 -eloq_ip_list=127.0.0.1:8000,127.0.0.1:8010,127.0.0.1:8020 - -[ENV] -MONO_PORT_1= @mysqld.1.port -MONO_SOCK_1= @mysqld.1.socket - -MONO_PORT_2= @mysqld.2.port -MONO_SOCK_2= @mysqld.2.socket - -MONO_PORT_3= @mysqld.3.port -MONO_SOCK_3= @mysqld.3.socket diff --git a/storage/eloq/mysql-test/mono_multi/t/multi_dead_lock_check.opt b/storage/eloq/mysql-test/mono_multi/t/multi_dead_lock_check.opt new file mode 100644 index 00000000000..98bcad26206 --- /dev/null +++ b/storage/eloq/mysql-test/mono_multi/t/multi_dead_lock_check.opt @@ -0,0 +1 @@ +--thread_pool_size=1 --core_number=1 --eloq_cc_protocol=Locking --deadlock_check_interval_seconds=15 \ No newline at end of file diff --git a/storage/eloq/mysql-test/mono_multi/t/range_split.test b/storage/eloq/mysql-test/mono_multi/t/range_split.test index 4234963b9d9..32e64ba6001 100644 --- a/storage/eloq/mysql-test/mono_multi/t/range_split.test +++ b/storage/eloq/mysql-test/mono_multi/t/range_split.test @@ -1,5 +1,4 @@ --source include/have_eloq.inc ---source include/have_range_partition.inc --sleep 2 #Here is bug, If not sleep, the mysqld will not return result. --let $mono_server_count= 3 diff --git a/storage/eloq/mysql-test/mono_multi/t/range_split_keycache.cnf b/storage/eloq/mysql-test/mono_multi/t/range_split_keycache.cnf deleted file mode 100644 index 308e12c1b68..00000000000 --- a/storage/eloq/mysql-test/mono_multi/t/range_split_keycache.cnf +++ /dev/null @@ -1,51 +0,0 @@ - -# Use default setting for mysqld processes -!include include/default_mysqld.cnf -!include include/default_client.cnf -!include include/eloq_kv_storage.cnf -!include include/eloq_kv_keyspace.cnf -!include include/eloq_kv_dss.cnf - -[mysqld] -performance_schema=OFF -default_storage_engine=eloq -plugin-maturity=experimental -plugin_load_add=ha_eloq -skip-log-bin -thread_handling=pool-of-threads -thread_pool_size=2 -thread_pool_dedicated_listener=1 -table_open_cache_instances=20 - -eloq_core_num=2 -eloq_logserver_rocksdb_scan_thread_num=3 -eloq_cc_protocol=OCC -eloq_enable_mvcc=off -eloq_use_key_cache=true -use_stat_tables=complementary -eloq_hm_bin_path = @ENV.MYSQLTEST_VARDIR/../../storage/eloq/host_manager - -[mysqld.1] -eloq_local_ip=127.0.0.1:8000 -eloq_ip_list=127.0.0.1:8000,127.0.0.1:8010,127.0.0.1:8020 -eloq_metrics_port=18081 - -[mysqld.2] -eloq_local_ip=127.0.0.1:8010 -eloq_ip_list=127.0.0.1:8000,127.0.0.1:8010,127.0.0.1:8020 -eloq_metrics_port=18082 - -[mysqld.3] -eloq_local_ip=127.0.0.1:8020 -eloq_ip_list=127.0.0.1:8000,127.0.0.1:8010,127.0.0.1:8020 -eloq_metrics_port=18083 - -[ENV] -MONO_PORT_1= @mysqld.1.port -MONO_SOCK_1= @mysqld.1.socket - -MONO_PORT_2= @mysqld.2.port -MONO_SOCK_2= @mysqld.2.socket - -MONO_PORT_3= @mysqld.3.port -MONO_SOCK_3= @mysqld.3.socket diff --git a/storage/eloq/mysql-test/mono_multi/t/range_split_keycache.opt b/storage/eloq/mysql-test/mono_multi/t/range_split_keycache.opt new file mode 100644 index 00000000000..a5bcb4384f6 --- /dev/null +++ b/storage/eloq/mysql-test/mono_multi/t/range_split_keycache.opt @@ -0,0 +1 @@ +--enable_key_cache=true \ No newline at end of file diff --git a/storage/eloq/mysql-test/mono_multi/t/range_split_keycache.test b/storage/eloq/mysql-test/mono_multi/t/range_split_keycache.test index 4234963b9d9..32e64ba6001 100644 --- a/storage/eloq/mysql-test/mono_multi/t/range_split_keycache.test +++ b/storage/eloq/mysql-test/mono_multi/t/range_split_keycache.test @@ -1,5 +1,4 @@ --source include/have_eloq.inc ---source include/have_range_partition.inc --sleep 2 #Here is bug, If not sleep, the mysqld will not return result. --let $mono_server_count= 3 diff --git a/storage/eloq/mysql-test/mono_multi/t/range_split_recovery.cnf b/storage/eloq/mysql-test/mono_multi/t/range_split_recovery.cnf deleted file mode 100644 index 40b42867fcf..00000000000 --- a/storage/eloq/mysql-test/mono_multi/t/range_split_recovery.cnf +++ /dev/null @@ -1,49 +0,0 @@ - -# Use default setting for mysqld processes -!include include/default_mysqld.cnf -!include include/default_client.cnf -!include include/eloq_kv_storage.cnf -!include include/eloq_kv_keyspace.cnf -!include include/eloq_kv_dss.cnf - -[mysqld] -default_storage_engine=eloq -plugin-maturity=experimental -plugin_load_add=ha_eloq -eloq -skip-log-bin -thread_handling=pool-of-threads -thread_pool_size=1 -thread_pool_dedicated_listener=1 -table_open_cache_instances=20 -eloq_core_num=1 -eloq_cc_protocol=OCC -eloq_enable_mvcc=off -use_stat_tables=complementary -eloq_checkpointer_interval_sec=86400 -eloq_hm_bin_path = @ENV.MYSQLTEST_VARDIR/../../storage/eloq/host_manager - -[mysqld.1] -eloq_local_ip=127.0.0.1:8000 -eloq_ip_list=127.0.0.1:8000,127.0.0.1:8010,127.0.0.1:8020 -eloq_metrics_port=18081 - -[mysqld.2] -eloq_local_ip=127.0.0.1:8010 -eloq_ip_list=127.0.0.1:8000,127.0.0.1:8010,127.0.0.1:8020 -eloq_metrics_port=18082 - -[mysqld.3] -eloq_local_ip=127.0.0.1:8020 -eloq_ip_list=127.0.0.1:8000,127.0.0.1:8010,127.0.0.1:8020 -eloq_metrics_port=18083 - -[ENV] -MONO_PORT_1= @mysqld.1.port -MONO_SOCK_1= @mysqld.1.socket - -MONO_PORT_2= @mysqld.2.port -MONO_SOCK_2= @mysqld.2.socket - -MONO_PORT_3= @mysqld.3.port -MONO_SOCK_3= @mysqld.3.socket diff --git a/storage/eloq/mysql-test/mono_multi/t/range_split_recovery.opt b/storage/eloq/mysql-test/mono_multi/t/range_split_recovery.opt new file mode 100644 index 00000000000..b6c8b13ccdb --- /dev/null +++ b/storage/eloq/mysql-test/mono_multi/t/range_split_recovery.opt @@ -0,0 +1 @@ +--checkpointer_interval=86400 \ No newline at end of file diff --git a/storage/eloq/mysql-test/mono_multi/t/range_split_recovery.test b/storage/eloq/mysql-test/mono_multi/t/range_split_recovery.test index 9042e57464d..a5423466710 100644 --- a/storage/eloq/mysql-test/mono_multi/t/range_split_recovery.test +++ b/storage/eloq/mysql-test/mono_multi/t/range_split_recovery.test @@ -1,5 +1,4 @@ --source include/have_eloq.inc ---source include/have_range_partition.inc --sleep 2 #Here is bug, If not sleep, the mysqld will not return result. --let $mono_server_count= 3 diff --git a/storage/eloq/mysql-test/mono_multi/t/range_split_write_forward.test b/storage/eloq/mysql-test/mono_multi/t/range_split_write_forward.test index 70c2a340a3f..3b0b3e75ff9 100644 --- a/storage/eloq/mysql-test/mono_multi/t/range_split_write_forward.test +++ b/storage/eloq/mysql-test/mono_multi/t/range_split_write_forward.test @@ -1,5 +1,4 @@ --source include/have_eloq.inc ---source include/have_range_partition.inc --sleep 2 #Here is bug, If not sleep, the mysqld will not return result. --let $mono_server_count= 3 diff --git a/storage/eloq/mysql-test/mono_multi/t/tx_timeout.cnf b/storage/eloq/mysql-test/mono_multi/t/tx_timeout.cnf deleted file mode 100644 index a2530b9a8e4..00000000000 --- a/storage/eloq/mysql-test/mono_multi/t/tx_timeout.cnf +++ /dev/null @@ -1,45 +0,0 @@ - -# Use default setting for mysqld processes -!include include/default_mysqld.cnf -!include include/default_client.cnf -!include include/eloq_kv_storage.cnf -!include include/eloq_kv_keyspace.cnf -!include include/eloq_kv_dss.cnf - -[mysqld] -default_storage_engine=eloq -plugin-maturity=experimental -plugin_load_add=ha_eloq -eloq -skip-log-bin -thread_handling=pool-of-threads -thread_pool_size=1 -thread_pool_dedicated_listener=1 -table_open_cache_instances=20 -eloq_core_num=1 -eloq_cc_protocol=OCC -eloq_enable_mvcc=off -eloq_hm_bin_path = @ENV.MYSQLTEST_VARDIR/../../storage/eloq/host_manager - -[mysqld.1] -eloq_local_ip=127.0.0.1:8000 -eloq_ip_list=127.0.0.1:8000,127.0.0.1:8010,127.0.0.1:8020 - -[mysqld.2] -eloq_local_ip=127.0.0.1:8010 -eloq_ip_list=127.0.0.1:8000,127.0.0.1:8010,127.0.0.1:8020 - -[mysqld.3] -eloq_local_ip=127.0.0.1:8020 -eloq_ip_list=127.0.0.1:8000,127.0.0.1:8010,127.0.0.1:8020 - -[ENV] -MONO_PORT_1= @mysqld.1.port -MONO_SOCK_1= @mysqld.1.socket - -MONO_PORT_2= @mysqld.2.port -MONO_SOCK_2= @mysqld.2.socket - -MONO_PORT_3= @mysqld.3.port -MONO_SOCK_3= @mysqld.3.socket - diff --git a/storage/eloq/mysql-test/mono_multi/t/tx_timeout.opt b/storage/eloq/mysql-test/mono_multi/t/tx_timeout.opt new file mode 100644 index 00000000000..ab1db8c83b8 --- /dev/null +++ b/storage/eloq/mysql-test/mono_multi/t/tx_timeout.opt @@ -0,0 +1 @@ +--thread_pool_size=1 --core_number=1 \ No newline at end of file