diff --git a/concourse/scripts/build_tarball.bash b/concourse/scripts/build_tarball.bash index fe814441b84..11a4ab104a7 100755 --- a/concourse/scripts/build_tarball.bash +++ b/concourse/scripts/build_tarball.bash @@ -99,8 +99,11 @@ DATA_STORE_ID=$(echo ${DATA_STORE_TYPE} | tr '[:upper:]' '[:lower:]') # Normalize behavior for supported DATA_STORE_TYPE values if [ "${DATA_STORE_TYPE}" = "ELOQDSS_ROCKSDB_CLOUD_S3" ]; then - CMAKE_ARGS="${CMAKE_ARGS} -DUSE_ROCKSDB_LOG_STATE=ON -DWITH_ROCKSDB_CLOUD=S3 -DWITH_CLOUD_AZ_INFO=ON" + CMAKE_ARGS="${CMAKE_ARGS} -DWITH_LOG_STATE=ROCKSDB_CLOUD_S3 -DWITH_CLOUD_AZ_INFO=ON" DATA_STORE_ID="rocks_s3" +elif [ "${DATA_STORE_TYPE}" = "ELOQDSS_ROCKSDB_CLOUD_GCS" ]; then + CMAKE_ARGS="${CMAKE_ARGS} -DWITH_LOG_STATE=ROCKSDB_CLOUD_GCS" + DATA_STORE_ID="rocks_gcs" elif [ "${DATA_STORE_TYPE}" = "ELOQDSS_ROCKSDB" ]; then DATA_STORE_ID="eloqdss_rocksdb" elif [ "${DATA_STORE_TYPE}" = "ELOQDSS_ELOQSTORE" ]; then @@ -220,10 +223,10 @@ fi # Build and install log_server (launch_sv) cd ${ELOQSQL_SRC}/storage/eloq/eloq_log_service mkdir bld && cd bld -cmake -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DUSE_ROCKSDB_LOG_STATE=ON ${CMAKE_ARGS} ../ +cmake -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DCMAKE_EXPORT_COMPILE_COMMANDS=ON ${CMAKE_ARGS} ../ cmake --build . --config ${BUILD_TYPE} -j4 -copy_libraries launch_sv ${DEST_DIR}/lib -mv launch_sv ${DEST_DIR}/bin/ +copy_libraries launch_sv ${DEST_DIR}/lib/ +mv launch_sv ${DEST_DIR}/bin/launch_sv cd ${HOME} tar -czvf eloqsql.tar.gz -C ${HOME} EloqSQL @@ -261,9 +264,11 @@ build_upload_log_srv() { cd ${log_sv_src} mkdir -p LogService/bin mkdir build && cd build - local cmake_args="-DCMAKE_BUILD_TYPE=$BUILD_TYPE -DWITH_ASAN=$ASAN -DDISABLE_CODE_LINE_IN_LOG=ON -DUSE_ROCKSDB_LOG_STATE=ON" + local cmake_args="-DCMAKE_BUILD_TYPE=$BUILD_TYPE -DWITH_ASAN=$ASAN -DDISABLE_CODE_LINE_IN_LOG=ON" if [ "$ds_type" = "ELOQDSS_ROCKSDB_CLOUD_S3" ]; then - cmake_args="$cmake_args -DWITH_ROCKSDB_CLOUD=S3 -DWITH_CLOUD_AZ_INFO=ON" + cmake_args="$cmake_args -DWITH_LOG_STATE=ROCKSDB_CLOUD_S3 -DWITH_CLOUD_AZ_INFO=ON" + elif [ "$ds_type" = "ELOQDSS_ROCKSDB_CLOUD_GCS" ]; then + cmake_args="$cmake_args -DWITH_LOG_STATE=ROCKSDB_CLOUD_GCS" fi cmake .. $cmake_args # build and copy log_server diff --git a/concourse/scripts/build_tarball_open.bash b/concourse/scripts/build_tarball_open.bash index 7535644eb14..718ef1da262 100755 --- a/concourse/scripts/build_tarball_open.bash +++ b/concourse/scripts/build_tarball_open.bash @@ -133,7 +133,7 @@ popd pushd "${ELOQSQL_SRC}/storage/eloq/log_service" rm -rf bld mkdir bld && cd bld -cmake -DCMAKE_BUILD_TYPE="${BUILD_TYPE}" -DUSE_ROCKSDB_LOG_STATE=ON ../ +cmake -DCMAKE_BUILD_TYPE="${BUILD_TYPE}" ../ cmake --build . --config "${BUILD_TYPE}" -j"${NCORE}" copy_libraries launch_sv "${DEST_DIR}/lib" mv launch_sv "${DEST_DIR}/bin/" diff --git a/concourse/scripts/main.bash b/concourse/scripts/main.bash index 6d84c847591..10d842bdafc 100755 --- a/concourse/scripts/main.bash +++ b/concourse/scripts/main.bash @@ -139,7 +139,6 @@ if [ ! -f "Makefile" ]; then -DMARIA_WITH_GLOG=ON \ -DSTATISTICS=ON \ -DWITH_DATA_STORE=ELOQDSS_ROCKSDB_CLOUD_S3 \ - -DUSE_ROCKSDB_LOG_STATE=ON \ ../ fi @@ -162,7 +161,7 @@ cp dss_server /home/$current_user/workspace/eloqsql/install/bin/ echo "building log_server" cd /home/$current_user/workspace/eloqsql/storage/eloq/log_service mkdir bld && cd bld -cmake -DCMAKE_BUILD_TYPE=Debug -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DUSE_ROCKSDB_LOG_STATE=ON ../ +cmake -DCMAKE_BUILD_TYPE=Debug -DCMAKE_EXPORT_COMPILE_COMMANDS=ON ../ cmake --build . --config Debug -j8 echo "installing launch_sv" cp launch_sv /home/$current_user/workspace/eloqsql/install/bin/ diff --git a/concourse/scripts/main.ent.bash b/concourse/scripts/main.ent.bash index 42927a73e9e..96d4d41883b 100755 --- a/concourse/scripts/main.ent.bash +++ b/concourse/scripts/main.ent.bash @@ -147,10 +147,9 @@ if [ ! -f "Makefile" ]; then -DMARIA_WITH_GLOG=ON \ -DSTATISTICS=ON \ -DWITH_DATA_STORE=ELOQDSS_ROCKSDB_CLOUD_S3 \ - -DUSE_ROCKSDB_LOG_STATE=ON \ - -DWITH_ROCKSDB_CLOUD=S3 \ -DOPEN_LOG_SERVICE=OFF \ -DFORK_HM_PROCESS=ON \ + -DWITH_LOG_STATE=ROCKSDB_CLOUD_S3 \ ../ fi @@ -173,7 +172,7 @@ cp dss_server /home/$current_user/workspace/eloqsql/install/bin/ echo "building log_server" cd /home/$current_user/workspace/eloqsql/storage/eloq/log_service mkdir bld && cd bld -cmake -DCMAKE_BUILD_TYPE=Debug -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DUSE_ROCKSDB_LOG_STATE=ON ../ +cmake -DCMAKE_BUILD_TYPE=Debug -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DWITH_LOG_STATE=ROCKSDB_CLOUD_S3 ../ cmake --build . --config Debug -j8 echo "installing launch_sv" cp launch_sv /home/$current_user/workspace/eloqsql/install/bin/ diff --git a/concourse/scripts/pr.ent.bash b/concourse/scripts/pr.ent.bash index a7719c65cb6..2106bae2a16 100755 --- a/concourse/scripts/pr.ent.bash +++ b/concourse/scripts/pr.ent.bash @@ -168,10 +168,9 @@ if [ ! -f "Makefile" ]; then -DMARIA_WITH_GLOG=ON \ -DSTATISTICS=ON \ -DWITH_DATA_STORE=ELOQDSS_ROCKSDB_CLOUD_S3 \ - -DUSE_ROCKSDB_LOG_STATE=ON \ - -DWITH_ROCKSDB_CLOUD=S3 \ -DOPEN_LOG_SERVICE=OFF \ -DFORK_HM_PROCESS=ON \ + -DWITH_LOG_STATE=ROCKSDB_CLOUD_S3 \ ../ fi @@ -194,7 +193,7 @@ cp dss_server /home/$current_user/workspace/eloqsql/install/bin/ echo "building log_server" cd /home/$current_user/workspace/eloqsql/storage/eloq/eloq_log_service mkdir bld && cd bld -cmake -DCMAKE_BUILD_TYPE=Debug -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DUSE_ROCKSDB_LOG_STATE=ON -DWITH_ROCKSDB_CLOUD=S3 ../ +cmake -DCMAKE_BUILD_TYPE=Debug -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DWITH_LOG_STATE=ROCKSDB_CLOUD_S3 ../ cmake --build . --config Debug -j8 echo "installing launch_sv" cp launch_sv /home/$current_user/workspace/eloqsql/install/bin/ diff --git a/storage/eloq/CMakeLists.txt b/storage/eloq/CMakeLists.txt index ee03a309894..4b3643a3258 100644 --- a/storage/eloq/CMakeLists.txt +++ b/storage/eloq/CMakeLists.txt @@ -32,7 +32,6 @@ elseif(WITH_DATA_STORE STREQUAL "BIGTABLE") elseif(WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB_CLOUD_S3") set(KV_STORAGE_VAL 3 CACHE STRING "eloq_ds" FORCE) add_compile_definitions(DATA_STORE_TYPE_ELOQDSS_ROCKSDB_CLOUD_S3) - add_compile_definitions(ROCKSDB_CLOUD_FS_TYPE=1) elseif(WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB") set(KV_STORAGE_VAL 3 CACHE STRING "eloq_ds" FORCE) add_compile_definitions(DATA_STORE_TYPE_ELOQDSS_ROCKSDB) diff --git a/storage/eloq/build_eloq_log_service.cmake b/storage/eloq/build_eloq_log_service.cmake index 89d0ce2388a..9aa66165f5e 100644 --- a/storage/eloq/build_eloq_log_service.cmake +++ b/storage/eloq/build_eloq_log_service.cmake @@ -4,11 +4,24 @@ SET(TX_LOG_PROTOS_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/tx_service/tx-log-proto set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error") option(BRPC_WITH_GLOG "With glog" ON) -option(USE_ROCKSDB_LOG_STATE "Whether use rocksdb log state or in-memory log state" ON) -option(WITH_ROCKSDB_CLOUD "RocksDB Cloud storage backend, S3 or GCS") -set_property(CACHE WITH_ROCKSDB_CLOUD PROPERTY STRINGS "S3" "GCS") -message(NOTICE "With RocksDB Cloud: ${WITH_ROCKSDB_CLOUD}") +set(WITH_LOG_STATE "ROCKSDB" CACHE STRING "The log state implementation") +set_property(CACHE WITH_LOG_STATE PROPERTY STRINGS "MEMORY" "ROCKSDB" "ROCKSDB_CLOUD_S3" "ROCKSDB_CLOUD_GCS") +message(NOTICE "WITH_LOG_STATE: ${WITH_LOG_STATE}") + +# Add compile flags for LOG STATE TYPE +if(WITH_LOG_STATE STREQUAL "MEMORY") + add_compile_definitions(LOG_STATE_TYPE_MEM) +elseif(WITH_LOG_STATE STREQUAL "ROCKSDB") + add_compile_definitions(LOG_STATE_TYPE_RKDB) +elseif(WITH_LOG_STATE STREQUAL "ROCKSDB_CLOUD_S3") + add_compile_definitions(LOG_STATE_TYPE_RKDB_S3) +elseif(WITH_LOG_STATE STREQUAL "ROCKSDB_CLOUD_GCS") + add_compile_definitions(LOG_STATE_TYPE_RKDB_GCS) +else() + message(FATAL_ERROR "Unknown WITH_LOG_STATE: ${WITH_LOG_STATE}") +endif() + find_path(BRPC_INCLUDE_PATH NAMES brpc/stream.h) find_library(BRPC_LIB NAMES brpc) @@ -100,9 +113,11 @@ endif() include_directories(${LEVELDB_INCLUDE_PATH}) set(LOG_SHIPPING_THREADS_NUM 1) -if (USE_ROCKSDB_LOG_STATE) - if (WITH_ROCKSDB_CLOUD MATCHES "S3|GCS") - if (WITH_ROCKSDB_CLOUD STREQUAL "S3") + +# Add RocksDB include and libs for LOG_STATE +if (WITH_LOG_STATE MATCHES "ROCKSDB|ROCKSDB_CLOUD_S3|ROCKSDB_CLOUD_GCS") + if (WITH_LOG_STATE MATCHES "ROCKSDB_CLOUD_S3|ROCKSDB_CLOUD_GCS") + if (WITH_LOG_STATE STREQUAL "ROCKSDB_CLOUD_S3") find_path(AWS_CORE_INCLUDE_PATH aws/core/Aws.h) if((NOT AWS_CORE_INCLUDE_PATH)) message(FATAL_ERROR "Fail to find aws/core include path") @@ -163,8 +178,7 @@ if (USE_ROCKSDB_LOG_STATE) find_library(ROCKSDB_CLOUD_LIB NAMES rocksdb-cloud-aws) add_compile_definitions(USE_AWS) - add_compile_definitions(WITH_ROCKSDB_CLOUD=1) - elseif (WITH_ROCKSDB_CLOUD STREQUAL "GCS") + elseif (WITH_LOG_STATE STREQUAL "ROCKSDB_CLOUD_GCS") find_path(GCP_CS_INCLUDE_PATH google/cloud/storage/client.h) if((NOT GCP_CS_INCLUDE_PATH)) message(FATAL_ERROR "Fail to find google/cloud/storage include path") @@ -189,7 +203,6 @@ if (USE_ROCKSDB_LOG_STATE) find_library(ROCKSDB_CLOUD_LIB NAMES rocksdb-cloud-gcp) add_compile_definitions(USE_GCP) - add_compile_definitions(WITH_ROCKSDB_CLOUD=2) endif () find_path(ROCKSDB_CLOUD_INCLUDE_PATH NAMES rocksdb/db.h PATH_SUFFIXES "rocksdb_cloud_header") @@ -225,8 +238,6 @@ if (USE_ROCKSDB_LOG_STATE) ${ROCKSDB_LIB} ) - # add preprocessor definition USE_ROCKSDB_LOG_STATE - add_compile_definitions(USE_ROCKSDB_LOG_STATE) # one shipping thread is enough for rocksdb version log state set(LOG_SHIPPING_THREADS_NUM 1) endif () diff --git a/storage/eloq/build_log_service.cmake b/storage/eloq/build_log_service.cmake index 1a63bea62e1..86a86c4f740 100644 --- a/storage/eloq/build_log_service.cmake +++ b/storage/eloq/build_log_service.cmake @@ -4,11 +4,6 @@ SET(TX_LOG_PROTOS_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/tx_service/tx-log-proto set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error") option(BRPC_WITH_GLOG "With glog" ON) -option(USE_ROCKSDB_LOG_STATE "Whether use rocksdb log state or in-memory log state" ON) - -option(WITH_ROCKSDB_CLOUD "RocksDB Cloud storage backend, S3 or GCS") -set_property(CACHE WITH_ROCKSDB_CLOUD PROPERTY STRINGS "S3" "GCS") -message(NOTICE "With RocksDB Cloud: ${WITH_ROCKSDB_CLOUD}") find_path(BRPC_INCLUDE_PATH NAMES brpc/stream.h) find_library(BRPC_LIB NAMES brpc) @@ -99,140 +94,34 @@ if ((NOT LEVELDB_INCLUDE_PATH) OR (NOT LEVELDB_LIB)) endif() include_directories(${LEVELDB_INCLUDE_PATH}) -set(LOG_SHIPPING_THREADS_NUM 1) -if (USE_ROCKSDB_LOG_STATE) - if (WITH_ROCKSDB_CLOUD MATCHES "S3|GCS") - if (WITH_ROCKSDB_CLOUD STREQUAL "S3") - find_path(AWS_CORE_INCLUDE_PATH aws/core/Aws.h) - if((NOT AWS_CORE_INCLUDE_PATH)) - message(FATAL_ERROR "Fail to find aws/core include path") - endif() - message(STATUS "aws/core include path: ${AWS_CORE_INCLUDE_PATH}") - - find_library(AWS_CORE_LIB aws-cpp-sdk-core) - if((NOT AWS_CORE_LIB )) - message(FATAL_ERROR "Fail to find aws-cpp-sdk-core lib") - endif() - message(STATUS "aws-cpp-sdk-core library: ${AWS_CORE_LIB}") - - find_path(AWS_KINESIS_INCLUDE_PATH aws/kinesis/KinesisClient.h) - if((NOT AWS_KINESIS_INCLUDE_PATH)) - message(FATAL_ERROR "Fail to find aws/kinesis include path") - endif() - message(STATUS "aws/kinesis include path: ${AWS_KINESIS_INCLUDE_PATH}") - - find_library(AWS_KINESIS_LIB aws-cpp-sdk-kinesis) - if((NOT AWS_KINESIS_LIB)) - message(FATAL_ERROR "Fail to find aws-cpp-sdk-kinesis lib") - endif() - message(STATUS "aws-cpp-sdk-kinesis library: ${AWS_KINESIS_LIB}") - - - find_path(AWS_KINESIS_INCLUDE_PATH aws/kinesis/KinesisClient.h) - if((NOT AWS_KINESIS_INCLUDE_PATH)) - message(FATAL_ERROR "Fail to find aws/kinesis include path") - endif() - message(STATUS "aws/kinesis include path: ${AWS_KINESIS_INCLUDE_PATH}") - - find_library(AWS_KINESIS_LIB aws-cpp-sdk-kinesis) - if((NOT AWS_KINESIS_LIB)) - message(FATAL_ERROR "Fail to find aws-cpp-sdk-kinesis lib") - endif() - message(STATUS "aws-cpp-sdk-kinesis library: ${AWS_KINESIS_LIB}") - - find_path(AWS_S3_INCLUDE_PATH aws/s3/S3Client.h) - if((NOT AWS_S3_INCLUDE_PATH)) - message(FATAL_ERROR "Fail to find aws/s3 include path") - endif() - message(STATUS "aws/s3 include path: ${AWS_S3_INCLUDE_PATH}") - - find_library(AWS_S3_LIB aws-cpp-sdk-s3) - if((NOT AWS_S3_LIB )) - message(FATAL_ERROR "Fail to find aws-cpp-sdk-s3 lib") - endif() - message(STATUS "aws-cpp-sdk-s3 library: ${AWS_S3_LIB}") - - set(ROCKSDB_INCLUDE_PATH ${ROCKSDB_INCLUDE_PATH} ${AWS_CORE_INCLUDE_PATH}) - set(ROCKSDB_INCLUDE_PATH ${ROCKSDB_INCLUDE_PATH} ${AWS_KINESIS_INCLUDE_PATH}) - set(ROCKSDB_INCLUDE_PATH ${ROCKSDB_INCLUDE_PATH} ${AWS_S3_INCLUDE_PATH}) - - set(ROCKSDB_LIB ${ROCKSDB_LIB} ${AWS_CORE_LIB}) - set(ROCKSDB_LIB ${ROCKSDB_LIB} ${AWS_KINESIS_LIB}) - set(ROCKSDB_LIB ${ROCKSDB_LIB} ${AWS_S3_LIB}) - - find_library(ROCKSDB_CLOUD_LIB NAMES rocksdb-cloud-aws) - - add_compile_definitions(USE_AWS) - add_compile_definitions(WITH_ROCKSDB_CLOUD=1) - elseif (WITH_ROCKSDB_CLOUD STREQUAL "GCS") - find_path(GCP_CS_INCLUDE_PATH google/cloud/storage/client.h) - if((NOT GCP_CS_INCLUDE_PATH)) - message(FATAL_ERROR "Fail to find google/cloud/storage include path") - endif() - message(STATUS "google/cloud/storage include path: ${GCP_CS_INCLUDE_PATH}") - - find_library(GCP_COMMON_LIB google_cloud_cpp_common) - if((NOT GCP_COMMON_LIB)) - message(FATAL_ERROR "Fail to find google_cloud_cpp_common lib") - endif() - message(STATUS "google_cloud_cpp_common library: ${GCP_COMMON_LIB}") - - find_library(GCP_CS_LIB google_cloud_cpp_storage) - if((NOT GCP_CS_LIB)) - message(FATAL_ERROR "Fail to find google_cloud_cpp_storage lib") - endif() - message(STATUS "google_cloud_cpp_storage library: ${GCP_CS_LIB}") - - set(ROCKSDB_LIB ${ROCKSDB_LIB} ${GCP_COMMON_LIB}) - set(ROCKSDB_LIB ${ROCKSDB_LIB} ${GCP_CS_LIB}) - - find_library(ROCKSDB_CLOUD_LIB NAMES rocksdb-cloud-gcp) - - add_compile_definitions(USE_GCP) - add_compile_definitions(WITH_ROCKSDB_CLOUD=2) - endif () - - find_path(ROCKSDB_CLOUD_INCLUDE_PATH NAMES rocksdb/db.h PATH_SUFFIXES "rocksdb_cloud_header") - if (NOT ROCKSDB_CLOUD_INCLUDE_PATH) - message(FATAL_ERROR "Fail to find RocksDB Cloud include path") - endif () - message(STATUS "ROCKSDB_CLOUD_INCLUDE_PATH: ${ROCKSDB_CLOUD_INCLUDE_PATH}") - set(ROCKSDB_INCLUDE_PATH ${ROCKSDB_INCLUDE_PATH} ${ROCKSDB_CLOUD_INCLUDE_PATH}) +find_path(ROCKSDB_INCLUDE_PATH NAMES rocksdb/db.h) - if (NOT ROCKSDB_CLOUD_LIB) - message(FATAL_ERROR "Fail to find RocksDB Cloud lib path") - endif () - message(STATUS "ROCKSDB_CLOUD_LIB: ${ROCKSDB_CLOUD_LIB}") - set(ROCKSDB_LIB ${ROCKSDB_LIB} ${ROCKSDB_CLOUD_LIB}) - else () - find_path(ROCKSDB_INCLUDE_PATH NAMES rocksdb/db.h) - if (NOT ROCKSDB_INCLUDE_PATH) - message(FATAL_ERROR "Fail to find RocksDB include path") - endif () - message(STATUS "ROCKSDB_INCLUDE_PATH: ${ROCKSDB_INCLUDE_PATH}") +if (NOT ROCKSDB_INCLUDE_PATH) +message(FATAL_ERROR "Fail to find RocksDB include path") +endif () +message(STATUS "ROCKSDB_INCLUDE_PATH: ${ROCKSDB_INCLUDE_PATH}") - find_library(ROCKSDB_LIB NAMES rocksdb) - if (NOT ROCKSDB_LIB) - message(FATAL_ERROR "Fail to find RocksDB lib path") - endif () - message(STATUS "ROCKSDB_LIB: ${ROCKSDB_LIB}") - endif () +find_library(ROCKSDB_LIB NAMES rocksdb) +if (NOT ROCKSDB_LIB) + message(FATAL_ERROR "Fail to find RocksDB lib path") +endif () +message(STATUS "ROCKSDB_LIB: ${ROCKSDB_LIB}") - include_directories(${ROCKSDB_INCLUDE_PATH}) +include_directories(${ROCKSDB_INCLUDE_PATH}) - set(LOG_LIB - ${LOG_LIB} - ${ROCKSDB_LIB} - ) +set(LOG_LIB + ${LOG_LIB} + ${ROCKSDB_LIB} + ) - # add preprocessor definition USE_ROCKSDB_LOG_STATE - add_compile_definitions(USE_ROCKSDB_LOG_STATE) - # one shipping thread is enough for rocksdb version log state - set(LOG_SHIPPING_THREADS_NUM 1) -endif () +# one shipping thread is enough for rocksdb version log state +set(LOG_SHIPPING_THREADS_NUM 1) add_compile_definitions(LOG_SHIPPING_THREADS_NUM=${LOG_SHIPPING_THREADS_NUM}) +# Add for compatiable reason +add_compile_definitions(LOG_STATE_TYPE_RKDB) + set(LOG_INCLUDE_DIR ${LOG_SOURCE_DIR}/include ${TX_LOG_PROTOS_SOURCE_DIR} @@ -257,7 +146,6 @@ find_package(Protobuf REQUIRED) ADD_CONVENIENCE_LIBRARY(logservice STATIC ${LOG_SOURCE_DIR}/src/log_server.cpp ${LOG_SOURCE_DIR}/src/log_state_rocksdb_impl.cpp - ${LOG_SOURCE_DIR}/src/log_state_rocksdb_cloud_impl.cpp ${LOG_SOURCE_DIR}/src/fault_inject.cpp ${LOG_SOURCE_DIR}/src/INIReader.cpp ${LOG_SOURCE_DIR}/src/ini.c diff --git a/storage/eloq/ha_eloq.cc b/storage/eloq/ha_eloq.cc index 0f0a7ea59ed..b1040385960 100644 --- a/storage/eloq/ha_eloq.cc +++ b/storage/eloq/ha_eloq.cc @@ -165,6 +165,11 @@ #define ELOQDS 1 #endif +#if (defined(DATA_STORE_TYPE_ELOQDSS_ROCKSDB_CLOUD_S3) || \ + defined(DATA_STORE_TYPE_ELOQDSS_ROCKSDB_CLOUD_GCS)) +#define ELOQDS_RKDB_CLOUD 1 +#endif + #if defined(DATA_STORE_TYPE_DYNAMODB) #include "store_handler/dynamo_handler.h" #elif defined(DATA_STORE_TYPE_BIGTABLE) @@ -174,9 +179,7 @@ #include "store_handler/eloq_data_store_service/data_store_service.h" #include "store_handler/eloq_data_store_service/data_store_service_config.h" #include "store_handler/data_store_service_client.h" -#if (defined(ROCKSDB_CLOUD_FS_TYPE) && \ - (ROCKSDB_CLOUD_FS_TYPE == ROCKSDB_CLOUD_FS_TYPE_S3 || \ - ROCKSDB_CLOUD_FS_TYPE == ROCKSDB_CLOUD_FS_TYPE_GCS)) +#if ELOQDS_RKDB_CLOUD #include "store_handler/eloq_data_store_service/rocksdb_cloud_data_store_factory.h" #include "store_handler/eloq_data_store_service/rocksdb_config.h" #elif defined(DATA_STORE_TYPE_ELOQDSS_ROCKSDB) @@ -198,19 +201,42 @@ #include "tx_service/include/tx_service_metrics.h" #include "tx_service/include/tx_request.h" #include "tx_service/include/util.h" + +#if (WITH_LOG_SERVICE) #include "log_server.h" #include "log_service_metrics.h" #include "log_utils.h" +#endif + +// Log state type +#if !defined(LOG_STATE_TYPE_RKDB_CLOUD) + +// Only if LOG_STATE_TYPE_RKDB_CLOUD undefined +#if ((defined(LOG_STATE_TYPE_RKDB_S3) || defined(LOG_STATE_TYPE_RKDB_GCS)) && \ + !defined(LOG_STATE_TYPE_RKDB)) +#define LOG_STATE_TYPE_RKDB_CLOUD 1 +#endif + +#endif -#if defined(USE_ROCKSDB_LOG_STATE) && defined(WITH_ROCKSDB_CLOUD) +#if !defined(LOG_STATE_TYPE_RKDB_ALL) + +// Only if LOG_STATE_TYPE_RKDB_ALL undefined +#if (defined(LOG_STATE_TYPE_RKDB_S3) || defined(LOG_STATE_TYPE_RKDB_GCS) || \ + defined(LOG_STATE_TYPE_RKDB)) +#define LOG_STATE_TYPE_RKDB_ALL 1 +#endif + +#endif + +#if defined(LOG_STATE_TYPE_RKDB_CLOUD) #include "rocksdb_cloud_config.h" #endif + // Don't put this include after sql_class.h include, it will cause compile // error -#if (defined(DATA_STORE_TYPE_DYNAMODB) || \ - (defined(USE_ROCKSDB_LOG_STATE) && \ - (WITH_ROCKSDB_CLOUD == CS_TYPE_S3)) || \ +#if (defined(DATA_STORE_TYPE_DYNAMODB) || defined(LOG_STATE_TYPE_RKDB_S3) || \ defined(DATA_STORE_TYPE_ELOQDSS_ROCKSDB_CLOUD_S3)) #include #endif @@ -2264,8 +2290,7 @@ static void PrintEloqConfig() std::cout << std::endl; } -#if defined(DATA_STORE_TYPE_DYNAMODB) || \ - (defined(USE_ROCKSDB_LOG_STATE) && (WITH_ROCKSDB_CLOUD == CS_TYPE_S3)) || \ +#if defined(DATA_STORE_TYPE_DYNAMODB) || defined(LOG_STATE_TYPE_RKDB_S3) || \ defined(DATA_STORE_TYPE_ELOQDSS_ROCKSDB_CLOUD_S3) Aws::SDKOptions aws_options; @@ -2297,8 +2322,7 @@ static int eloq_init_abort() txlog_server= nullptr; // stop and release txlog service } -#if defined(DATA_STORE_TYPE_DYNAMODB) || \ - (defined(USE_ROCKSDB_LOG_STATE) && (WITH_ROCKSDB_CLOUD == CS_TYPE_S3)) || \ +#if defined(DATA_STORE_TYPE_DYNAMODB) || defined(LOG_STATE_TYPE_RKDB_S3) || \ defined(DATA_STORE_TYPE_ELOQDSS_ROCKSDB_CLOUD_S3) aws_deinit(); #endif @@ -2404,8 +2428,7 @@ static int eloq_init_func(void *p) sql_print_information("Eloq initializing."); -#if defined(DATA_STORE_TYPE_DYNAMODB) || \ - (defined(USE_ROCKSDB_LOG_STATE) && (WITH_ROCKSDB_CLOUD == CS_TYPE_S3)) || \ +#if defined(DATA_STORE_TYPE_DYNAMODB) || defined(LOG_STATE_TYPE_RKDB_S3) || \ defined(DATA_STORE_TYPE_ELOQDSS_ROCKSDB_CLOUD_S3) if (aws_init()) { @@ -2832,6 +2855,8 @@ static int eloq_init_func(void *p) std::to_string(eloq_logserver_rocksdb_scan_thread_num).c_str()); std::vector txlog_ips; std::vector txlog_ports; + + // If eloq_txlog_service_list is empty, it means bounded txlog service if (std::strlen(eloq_txlog_service_list) == 0) { sql_print_information("Stand-alone txlog service is not provided, start " @@ -2877,15 +2902,15 @@ static int eloq_init_func(void *p) eloq_notify_checkpointer_threshold_size); } -#ifdef USE_ROCKSDB_LOG_STATE +#if defined(LOG_STATE_TYPE_RKDB_ALL) size_t rocksdb_target_file_size_base_val= txlog::parse_size(eloq_txlog_rocksdb_target_file_size_base); -#ifdef WITH_ROCKSDB_CLOUD +#if defined(LOG_STATE_TYPE_RKDB_CLOUD) txlog::RocksDBCloudConfig rocksdb_cloud_config; -#if WITH_ROCKSDB_CLOUD == CS_TYPE_S3 +#if defined(LOG_STATE_TYPE_RKDB_S3) rocksdb_cloud_config.aws_access_key_id_= eloq_aws_access_key_id; rocksdb_cloud_config.aws_secret_key_= eloq_aws_secret_key; -#endif /* WITH_ROCKSDB_CLOUD == CS_TYPE_S3 */ +#endif /* LOG_STATE_TYPE_RKDB_S3 */ rocksdb_cloud_config.bucket_name_= eloq_txlog_rocksdb_cloud_bucket_name; rocksdb_cloud_config.bucket_prefix_= eloq_txlog_rocksdb_cloud_bucket_prefix; @@ -2903,14 +2928,6 @@ static int eloq_init_func(void *p) if (opt_bootstrap) { -#if defined(OPEN_LOG_SERVICE) - txlog_server= std::make_unique<::txlog::LogServer>( - node_id, log_server_port, txlog_path, 1, rocksdb_cloud_config, - eloq_txlog_rocksdb_cloud_in_mem_log_size_high_watermark, - eloq_txlog_rocksdb_max_write_buffer_number, - eloq_txlog_rocksdb_max_background_jobs, - rocksdb_target_file_size_base_val); -#else txlog_server= std::make_unique<::txlog::LogServer>( node_id, log_server_port, txlog_ips, txlog_ports, txlog_path, 0, eloq_txlog_group_replica_num, txlog_rocksdb_path, @@ -2919,18 +2936,9 @@ static int eloq_init_func(void *p) eloq_txlog_rocksdb_max_write_buffer_number, eloq_txlog_rocksdb_max_background_jobs, rocksdb_target_file_size_base_val, eloq_logserver_snapshot_interval); -#endif } else { -#if defined(OPEN_LOG_SERVICE) - txlog_server= std::make_unique<::txlog::LogServer>( - node_id, log_server_port, txlog_path, 1, rocksdb_cloud_config, - eloq_txlog_rocksdb_cloud_in_mem_log_size_high_watermark, - eloq_txlog_rocksdb_max_write_buffer_number, - eloq_txlog_rocksdb_max_background_jobs, - rocksdb_target_file_size_base_val); -#else txlog_server= std::make_unique<::txlog::LogServer>( node_id, log_server_port, txlog_ips, txlog_ports, txlog_path, 0, eloq_txlog_group_replica_num, txlog_rocksdb_path, @@ -2942,9 +2950,9 @@ static int eloq_init_func(void *p) enable_txlog_request_checkpoint, eloq_check_replay_log_size_interval_sec, notify_checkpointer_threshold_size); -#endif } -#else /* WITH_ROCKSDB_CLOUD */ +#else /* LOG_STATE_TYPE_RKDB_CLOUD */ + // LOG_STATE_TYPE_RKDB size_t rocksdb_sst_files_size_limit_val= txlog::parse_size(eloq_txlog_rocksdb_sst_files_size_limit); @@ -2992,8 +3000,8 @@ static int eloq_init_func(void *p) #endif } -#endif /* WITH_ROCKSDB_CLOUD */ -#else /* USE_ROCKSDB_LOG_STATE */ +#endif /* LOG_STATE_TYPE_RKDB_CLOUD */ +#else /* LOG_STATE_TYPE_MEM */ #if defined(OPEN_LOG_SERVICE) txlog_server= std::make_unique<::txlog::LogServer>( node_id, log_server_port, txlog_path, 1); @@ -3002,7 +3010,7 @@ static int eloq_init_func(void *p) node_id, log_server_port, txlog_ips, txlog_ports, txlog_path, 0, eloq_txlog_group_replica_num, eloq_logserver_snapshot_interval); #endif -#endif /* USE_ROCKSDB_LOG_STATE */ +#endif /* LOG_STATE_TYPE_MEM */ err= txlog_server->Start(); if (err != 0) { @@ -3329,9 +3337,8 @@ static int eloq_done_func(void *p) } #if defined(DATA_STORE_TYPE_DYNAMODB) || \ - (defined(USE_ROCKSDB_LOG_STATE) && (WITH_ROCKSDB_CLOUD == CS_TYPE_S3)) || \ + defined(LOG_STATE_TYPE_RKDB_S3) || \ defined(DATA_STORE_TYPE_ELOQDSS_ROCKSDB_CLOUD_S3) - aws_deinit(); #endif diff --git a/storage/eloq/log_service b/storage/eloq/log_service index a085920ce6a..9caaf7cd9e7 160000 --- a/storage/eloq/log_service +++ b/storage/eloq/log_service @@ -1 +1 @@ -Subproject commit a085920ce6a71106c75703ff14c36033bae449b5 +Subproject commit 9caaf7cd9e78ee4a6e4eb6e6b35bff16f852e249 diff --git a/storage/eloq/store_handler b/storage/eloq/store_handler index 61c7c80119b..db720400d19 160000 --- a/storage/eloq/store_handler +++ b/storage/eloq/store_handler @@ -1 +1 @@ -Subproject commit 61c7c80119bf836c82006533d9bafffa482085bd +Subproject commit db720400d195c7164c386e13ca2811fa668123de diff --git a/storage/eloq/tx_service b/storage/eloq/tx_service index 055e500c5d9..f0634562130 160000 --- a/storage/eloq/tx_service +++ b/storage/eloq/tx_service @@ -1 +1 @@ -Subproject commit 055e500c5d947c6fc9b52587c50dbac70f28de39 +Subproject commit f06345621308065f8af22c54b85b97bdf794a9b6