Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion data_substrate
Submodule data_substrate updated 70 files
+0 −3 .gitignore
+18 −0 .pre-commit-config.yaml
+6 −7 core/include/data_substrate.h
+2 −1 core/src/data_substrate.cpp
+1 −1 core/src/storage_init.cpp
+16 −13 store_handler/bigtable_handler.cpp
+16 −13 store_handler/bigtable_handler.h
+97 −72 store_handler/data_store_service_client.cpp
+27 −23 store_handler/data_store_service_client.h
+4 −3 store_handler/data_store_service_client_closure.h
+16 −13 store_handler/dynamo_handler.cpp
+16 −13 store_handler/dynamo_handler.h
+2 −2 store_handler/dynamo_handler_typed.cpp
+2 −2 store_handler/dynamo_handler_typed.h
+1 −1 store_handler/eloq_data_store_service/INIReader.cpp
+1 −1 store_handler/eloq_data_store_service/INIReader.h
+2 −1 store_handler/eloq_data_store_service/data_store.h
+4 −2 store_handler/eloq_data_store_service/data_store_factory.h
+2 −1 store_handler/eloq_data_store_service/data_store_service.cpp
+2 −1 store_handler/eloq_data_store_service/data_store_service.h
+0 −1 store_handler/eloq_data_store_service/data_store_service_config.cpp
+4 −4 store_handler/eloq_data_store_service/eloq_store_config.cpp
+1 −1 store_handler/eloq_data_store_service/eloq_store_config.h
+82 −59 store_handler/eloq_data_store_service/ini.c
+1 −1 store_handler/eloq_data_store_service/ini.h
+2 −1 store_handler/eloq_data_store_service/internal_request.h
+1 −1 store_handler/eloq_data_store_service/main.cpp
+2 −2 store_handler/eloq_data_store_service/purger_event_listener.cpp
+3 −3 store_handler/eloq_data_store_service/purger_sliding_window.cpp
+7 −5 store_handler/eloq_data_store_service/purger_sliding_window.h
+2 −2 store_handler/eloq_data_store_service/rocksdb_cloud_data_store.cpp
+6 −3 store_handler/eloq_data_store_service/rocksdb_cloud_data_store.h
+3 −3 store_handler/eloq_data_store_service/rocksdb_cloud_dump.cpp
+4 −6 store_handler/eloq_data_store_service/rocksdb_config.cpp
+1 −1 store_handler/eloq_data_store_service/rocksdb_config.h
+13 −13 store_handler/eloq_data_store_service/rocksdb_data_store.cpp
+2 −2 store_handler/eloq_data_store_service/rocksdb_data_store_common.cpp
+1 −1 store_handler/eloq_data_store_service/rocksdb_data_store_common.h
+9 −10 store_handler/eloq_data_store_service/rocksdb_data_store_factory.h
+30 −25 store_handler/eloq_data_store_service/s3_file_downloader.cpp
+10 −8 store_handler/eloq_data_store_service/s3_file_downloader.h
+11 −8 store_handler/rocksdb_handler.cpp
+16 −13 store_handler/rocksdb_handler.h
+0 −18 tx_service/include/cc/cc_request.h
+1 −1 tx_service/include/cc/ccm_scanner.h
+1 −1 tx_service/include/cc/cluster_config_cc_map.h
+4 −0 tx_service/include/cc/object_cc_map.h
+1 −1 tx_service/include/cc/range_slice_type.h
+22 −32 tx_service/include/cc/template_cc_map.h
+1 −1 tx_service/include/cc_protocol.h
+1 −1 tx_service/include/checkpointer.h
+1 −1 tx_service/include/concurrent_queue_wsize.h
+12 −1 tx_service/include/data_sync_task.h
+1 −1 tx_service/include/invasive_head_list.h
+1 −1 tx_service/include/remote/cc_stream_sender.h
+1 −1 tx_service/include/standby.h
+18 −14 tx_service/include/store/data_store_handler.h
+9 −7 tx_service/include/store/int_mem_store.h
+1 −1 tx_service/include/tx_id.h
+1 −1 tx_service/include/tx_log_service.h
+2 −1 tx_service/src/cc/cc_shard.cpp
+1 −1 tx_service/src/cc/local_cc_shards.cpp
+1 −1 tx_service/src/checkpointer.cpp
+1 −1 tx_service/src/data_sync_task.cpp
+1 −1 tx_service/src/range_record.cpp
+0 −3 tx_service/src/sk_generator.cpp
+1 −1 tx_service/src/standby.cpp
+12 −2 tx_service/src/store/snapshot_manager.cpp
+1 −1 tx_service/src/tx_key.cpp
+1 −1 tx_service/tests/include/mock/mock_log_agent.h
9 changes: 6 additions & 3 deletions sql/mysqld.cc
Original file line number Diff line number Diff line change
Expand Up @@ -643,8 +643,7 @@ struct system_variables global_system_variables;
const char *current_dbug_option="";

// Declare gflags
DEFINE_string(eloqsql_config, "",
"Path to MySQL configuration file. All MySQL options must be in this config file.");
DECLARE_string(eloqsql_config);
DECLARE_bool(bootstrap);


Expand Down Expand Up @@ -5565,10 +5564,12 @@ int mysqld_main(int argc, char **argv)
// Build argv for load_defaults
// Copy all original arguments first, then append --eloqsql_config and --bootstrap if needed
int additional_args = 0;
#ifdef MYSQLD_LIBRARY_MODE
if (!FLAGS_eloqsql_config.empty())
additional_args++;
if (FLAGS_bootstrap)
additional_args++;
#endif

char **load_default_argv = (char**)malloc((argc + additional_args + 1) * sizeof(char*));
if (!load_default_argv)
Expand All @@ -5583,7 +5584,8 @@ int mysqld_main(int argc, char **argv)
{
load_default_argv[i] = argv[i];
}


#ifdef MYSQLD_LIBRARY_MODE
// Append --defaults-file if --eloqsql_config specified
std::string defaults_file_arg;
if (!FLAGS_eloqsql_config.empty())
Expand All @@ -5599,6 +5601,7 @@ int mysqld_main(int argc, char **argv)
load_default_argv[load_default_argc] = const_cast<char*>("--bootstrap");
load_default_argc++;
}
#endif

// Null-terminate the array
load_default_argv[load_default_argc] = nullptr;
Expand Down
12 changes: 4 additions & 8 deletions storage/eloq/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,12 @@ set(WITH_DATA_STORE "ELOQDSS_ROCKSDB" CACHE STRING "Which key-value storage to u
set_property(CACHE WITH_DATA_STORE PROPERTY STRINGS "DYNAMODB" "BIGTABLE" "ELOQDSS_ROCKSDB_CLOUD_S3" "ELOQDSS_ELOQSTORE" "ELOQDSS_ROCKSDB")
message(NOTICE "WITH_DATA_STORE" : ${WITH_DATA_STORE})

set(METRICS_LIB "eloq-metrics" CACHE STRING "metrics library name.")

option(WITH_LOG_SERVICE "Compile with built-in log service." ON)

if(BUILD_ELOQSQL_AS_LIBRARY)
add_compile_definitions(MYSQLD_LIBRARY_MODE)
endif()

if(WITH_LOG_SERVICE)
message("WITH_LOG_SERVICE: " ${WITH_LOG_SERVICE})
add_compile_definitions(WITH_LOG_SERVICE)
Expand Down Expand Up @@ -122,12 +124,6 @@ else()
endif()
add_compile_definitions(WITH_DATA_STORE=${KV_STORAGE_VAL})

if(USE_ONE_DATA_STORE_SHARD_ENABLED)
# add preprocessor defition USE_ONE_DATA_STORE_SHARD
add_compile_definitions(USE_ONE_DATA_STORE_SHARD)
endif()


MYSQL_ADD_PLUGIN(eloq ${MYELOQ_SOURCES}
STORAGE_ENGINE
MANDATORY
Expand Down
8 changes: 4 additions & 4 deletions storage/eloq/eloq_db.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -396,8 +396,8 @@ int eloq_create_database(THD *thd, LEX_CSTRING db,
}

auto [yield_func, resume_func]= thd_get_coro_functors(thd);
bool ok=
storage_hd->UpsertDatabase(key, {opt_binary.str, opt_binary.length}, yield_func, resume_func);
bool ok= storage_hd->UpsertDatabase(key, {opt_binary.str, opt_binary.length},
yield_func, resume_func);
if (!ok)
{
my_printf_error(HA_ERR_INTERNAL_ERROR, "Eloq upsert database '%s' failed",
Expand Down Expand Up @@ -439,8 +439,8 @@ int eloq_update_database(THD *thd, LEX_CSTRING db,
}

auto [yield_func, resume_func]= thd_get_coro_functors(thd);
bool ok=
storage_hd->UpsertDatabase(key, {opt_binary.str, opt_binary.length}, yield_func, resume_func);
bool ok= storage_hd->UpsertDatabase(key, {opt_binary.str, opt_binary.length},
yield_func, resume_func);
if (!ok)
{
my_printf_error(HA_ERR_INTERNAL_ERROR, "Eloq upsert database '%s' failed",
Expand Down
5 changes: 3 additions & 2 deletions storage/eloq/eloq_db_dl.h
Original file line number Diff line number Diff line change
Expand Up @@ -52,8 +52,9 @@ template <class Elem> class Dynamic_array;
* must match with that defined by ELOQ_DEFINEx.
*/
// Statically linked: declare the symbol and let the linker resolve it to the
// implementation provided in the static library (see eloq_db.cpp extern "C" block).
#define MONOCALL_DEFINEx(x, name, ...) \
// implementation provided in the static library (see eloq_db.cpp extern "C"
// block).
#define MONOCALL_DEFINEx(x, name, ...) \
extern "C" int name(__SC_DECL##x(__VA_ARGS__));
#define MONOCALL_DEFINE1(name, ...) MONOCALL_DEFINEx(1, name, __VA_ARGS__)
#define MONOCALL_DEFINE2(name, ...) MONOCALL_DEFINEx(2, name, __VA_ARGS__)
Expand Down
6 changes: 3 additions & 3 deletions storage/eloq/eloqsql_catalog_factory.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -435,9 +435,9 @@ MysqlTableSchema::MysqlTableSchema(const txservice::TableName &table_name,
}

size_t offset= 0;
kv_info_= DataSubstrate::GetGlobal()
->GetStoreHandler()
->DeserializeKVCatalogInfo(kv_info, offset);
kv_info_=
DataSubstrate::GetGlobal()->GetStoreHandler()->DeserializeKVCatalogInfo(
kv_info, offset);
record_schema_= EloqRecordSchema(&mysql_table_share_);
}

Expand Down
33 changes: 21 additions & 12 deletions storage/eloq/ha_eloq.cc
Original file line number Diff line number Diff line change
Expand Up @@ -281,10 +281,10 @@ static MYSQL_SYSVAR_ENUM(enum_var, // name
0, // def
&enum_var_typelib); // typelib


static MYSQL_SYSVAR_STR(config, eloq_config, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_MEMALLOC,
"Path to data substrate configuration file.", nullptr, nullptr,
"");
static MYSQL_SYSVAR_STR(config, eloq_config,
PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_MEMALLOC,
"Path to data substrate configuration file.", nullptr,
nullptr, "");
static MYSQL_SYSVAR_STR(insert_semantic, eloq_insert_semantic,
PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_MEMALLOC,
"Insert semantic: insert or upsert", nullptr, nullptr,
Expand Down Expand Up @@ -426,13 +426,21 @@ static MYSQL_SYSVAR_INT(signal_monitor, eloq_signal_monitor,
nullptr, nullptr, 0, 0, INT_MAX, 0);

static struct st_mysql_sys_var *eloq_system_variables[]= {
MYSQL_SYSVAR(cc_protocol), MYSQL_SYSVAR(enum_var),
MYSQL_SYSVAR(ulong_var), MYSQL_SYSVAR(int_var),
MYSQL_SYSVAR(double_var), MYSQL_SYSVAR(double_thdvar),
MYSQL_SYSVAR(varopt_default), MYSQL_SYSVAR(insert_semantic),
MYSQL_SYSVAR(auto_increment), MYSQL_SYSVAR(invalidate_cache_once),
MYSQL_SYSVAR(random_scan_sort), MYSQL_SYSVAR(report_debug_info),
MYSQL_SYSVAR(signal_monitor), MYSQL_SYSVAR(config), NULL};
MYSQL_SYSVAR(cc_protocol),
MYSQL_SYSVAR(enum_var),
MYSQL_SYSVAR(ulong_var),
MYSQL_SYSVAR(int_var),
MYSQL_SYSVAR(double_var),
MYSQL_SYSVAR(double_thdvar),
MYSQL_SYSVAR(varopt_default),
MYSQL_SYSVAR(insert_semantic),
MYSQL_SYSVAR(auto_increment),
MYSQL_SYSVAR(invalidate_cache_once),
MYSQL_SYSVAR(random_scan_sort),
MYSQL_SYSVAR(report_debug_info),
MYSQL_SYSVAR(signal_monitor),
MYSQL_SYSVAR(config),
NULL};

/**
Structure for CREATE TABLE options (table options).
Expand Down Expand Up @@ -1436,7 +1444,8 @@ static int eloq_init_func(void *p)
LOG(INFO) << "Data substrate initialized, MySQL continuing...";
}
#else
// Set the data substrate data path to mysql home directory in standalone mode.
// Set the data substrate data path to mysql home directory in standalone
// mode.
FLAGS_eloq_data_path= mysql_real_data_home_ptr;
// In standalone mode, initialize data substrate here
// Wait for mysqld to initialize before initializing data substrate
Expand Down