Converge

.gitignore

@@ -544,6 +544,7 @@ compile_commands.json
 
 # Visual Studio Code workspace
 .vscode/
+.cursor/
 
 # Clion && other JetBrains ides
 /.idea/

.gitmodules

@@ -17,15 +17,6 @@
 [submodule "storage/columnstore/columnstore"]
 	path = storage/columnstore/columnstore
 	url = https://github.com/mariadb-corporation/mariadb-columnstore-engine.git
-[submodule "storage/eloq/eloq_metrics"]
-	path = storage/eloq/eloq_metrics
-	url = https://github.com/eloqdata/eloq-metrics.git
-[submodule "storage/eloq/tx_service"]
-	path = storage/eloq/tx_service
+[submodule "data_substrate/tx_service"]
+	path = data_substrate
 	url = https://github.com/eloqdata/tx_service.git
-[submodule "storage/eloq/store_handler"]
-	path = storage/eloq/store_handler
-	url = https://github.com/eloqdata/store_handler.git
-[submodule "storage/eloq/log_service"]
-	path = storage/eloq/log_service
-	url = https://github.com/eloqdata/log_service.git

CMakeLists.txt

@@ -40,9 +40,16 @@ FOREACH(p CMP0022 CMP0046 CMP0040 CMP0048 CMP0054 CMP0075 CMP0069 CMP0135)
 ENDFOREACH()
 
 MESSAGE(STATUS "Running cmake version ${CMAKE_VERSION}")
+# Option to build as library for converged binary
+option(BUILD_ELOQSQL_AS_LIBRARY "Build eloqsql as library instead of executable" OFF)
+
+# IF(BUILD_ELOQSQL_AS_LIBRARY)
+  SET(ELOQSQL_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR})
+  SET(ELOQSQL_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR})
+# ENDIF()
 
 SET(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH}
-  ${CMAKE_SOURCE_DIR}/cmake ${CMAKE_SOURCE_DIR}/cmake/Internal/CPack)
+  ${ELOQSQL_SOURCE_DIR}/cmake ${ELOQSQL_SOURCE_DIR}/cmake/Internal/CPack)
 
 # Use a default manufacturer if no manufacturer was identified.
 IF(NOT DEFINED MANUFACTURER) 
@@ -98,9 +105,10 @@ SET(CPACK_PACKAGE_NAME "MariaDB")
 SET(CPACK_PACKAGE_DESCRIPTION_SUMMARY "MariaDB: a very fast and robust SQL database server")
 SET(CPACK_PACKAGE_URL "http://mariadb.org")
 
+option(WITH_DYNAMO_DB "With DynamoDB" OFF)
 IF(BUILD_CONFIG)
   INCLUDE(
-  ${CMAKE_SOURCE_DIR}/cmake/build_configurations/${BUILD_CONFIG}.cmake)
+  ${ELOQSQL_SOURCE_DIR}/cmake/build_configurations/${BUILD_CONFIG}.cmake)
 ENDIF()
 
 # Include the platform-specific file. To allow exceptions, this code
@@ -113,7 +121,7 @@ FOREACH(_base
     ${CMAKE_SYSTEM_NAME}-${CMAKE_SYSTEM_VERSION}-${CMAKE_SYSTEM_PROCESSOR}
     ${CMAKE_SYSTEM_NAME}-${CMAKE_SYSTEM_VERSION}
     ${CMAKE_SYSTEM_NAME})
-  SET(_file ${CMAKE_SOURCE_DIR}/cmake/os/${_base}.cmake)
+  SET(_file ${ELOQSQL_SOURCE_DIR}/cmake/os/${_base}.cmake)
   IF(EXISTS ${_file})
     INCLUDE(${_file})
     BREAK()
@@ -189,7 +197,7 @@ OPTION(WITHOUT_SERVER "Build only the client library and clients" OFF)
 IF(UNIX)
   OPTION(WITH_VALGRIND "Valgrind instrumentation" OFF)
 ENDIF()
-OPTION (WITH_UNIT_TESTS "Compile MySQL with unit tests" ON)
+OPTION (WITH_UNIT_TESTS "Compile MySQL with unit tests" OFF)
 
 IF (WITHOUT_SERVER)
   SET (SKIP_COMPONENTS "Server|IniFiles|SuportFiles|Readme")
@@ -450,10 +458,21 @@ INCLUDE(cpack_deb)
 
 SET(PYTHON_SHEBANG "/usr/bin/env python3" CACHE STRING "python shebang")
 MARK_AS_ADVANCED(PYTHON_SHEBANG)
-
 # Add storage engines and plugins.
 CONFIGURE_PLUGINS()
 
+add_compile_definitions(ELOQ_MODULE_ELOQSQL)
+# Build data substrate library first (before sql/ and storage/)
+if(NOT BUILD_ELOQSQL_AS_LIBRARY)
+    # Build data substrate library first (standalone mode)
+    ADD_SUBDIRECTORY(data_substrate)
+else()
+    # Use parent data_substrate (converged mode)
+    message(STATUS "EloqSQL: Using parent data_substrate")
+    # When building as library, mysqld will NOT initialize data substrate
+    # It will use synchronization with converged main instead
+    add_compile_definitions(MYSQLD_LIBRARY_MODE)
+endif()
 ADD_SUBDIRECTORY(include)
 ADD_SUBDIRECTORY(dbug)
 ADD_SUBDIRECTORY(strings)
@@ -505,15 +524,15 @@ IF(NOT CMAKE_CROSSCOMPILING OR DEFINED CMAKE_CROSSCOMPILING_EMULATOR)
   ENDIF()
   # minimal target to build only binaries for export
   ADD_CUSTOM_TARGET(import_executables DEPENDS ${EXPORTED})
-  EXPORT(TARGETS ${EXPORTED} FILE ${CMAKE_BINARY_DIR}/import_executables.cmake)
+  EXPORT(TARGETS ${EXPORTED} FILE ${ELOQSQL_BINARY_DIR}/import_executables.cmake)
 ENDIF()
 
-CONFIGURE_FILE(config.h.cmake   ${CMAKE_BINARY_DIR}/include/my_config.h)
-CONFIGURE_FILE(config.h.cmake   ${CMAKE_BINARY_DIR}/include/config.h)
-CONFIGURE_FILE(${CMAKE_SOURCE_DIR}/include/mysql_version.h.in
-               ${CMAKE_BINARY_DIR}/include/mysql_version.h )
-CONFIGURE_FILE(${CMAKE_SOURCE_DIR}/sql/sql_builtin.cc.in
-    ${CMAKE_BINARY_DIR}/sql/sql_builtin.cc)
+CONFIGURE_FILE(config.h.cmake   ${ELOQSQL_BINARY_DIR}/include/my_config.h)
+CONFIGURE_FILE(config.h.cmake   ${ELOQSQL_BINARY_DIR}/include/config.h)
+CONFIGURE_FILE(${ELOQSQL_SOURCE_DIR}/include/mysql_version.h.in
+               ${ELOQSQL_BINARY_DIR}/include/mysql_version.h )
+CONFIGURE_FILE(${ELOQSQL_SOURCE_DIR}/sql/sql_builtin.cc.in
+    ${ELOQSQL_BINARY_DIR}/sql/sql_builtin.cc)
 
 IF(GIT_EXECUTABLE AND EXISTS ${PROJECT_SOURCE_DIR}/.git)
   EXECUTE_PROCESS(
@@ -531,30 +550,30 @@ IF(SOURCE_REVISION OR
 ENDIF()
 
 CONFIGURE_FILE(
-    ${CMAKE_SOURCE_DIR}/cmake/info_macros.cmake.in
-    ${CMAKE_BINARY_DIR}/info_macros.cmake @ONLY)
+    ${ELOQSQL_SOURCE_DIR}/cmake/info_macros.cmake.in
+    ${ELOQSQL_BINARY_DIR}/info_macros.cmake @ONLY)
 
 # Handle the "INFO_*" files.
-INCLUDE(${CMAKE_BINARY_DIR}/info_macros.cmake)
+INCLUDE(${ELOQSQL_BINARY_DIR}/info_macros.cmake)
 # Source: This can be done during the cmake phase, all information is
 # available, but should be repeated on each "make" just in case someone
 # does "cmake ; make ; git pull ; make".
-CREATE_INFO_SRC(${CMAKE_BINARY_DIR}/Docs)
+CREATE_INFO_SRC(${ELOQSQL_BINARY_DIR}/Docs)
 ADD_CUSTOM_TARGET(INFO_SRC ALL
-  COMMAND ${CMAKE_COMMAND} -P ${CMAKE_SOURCE_DIR}/cmake/info_src.cmake
-  WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
+  COMMAND ${CMAKE_COMMAND} -P ${ELOQSQL_SOURCE_DIR}/cmake/info_src.cmake
+  WORKING_DIRECTORY ${ELOQSQL_BINARY_DIR}
 )
 # Build flags: This must be postponed to the make phase.
 ADD_CUSTOM_TARGET(INFO_BIN ALL
-  COMMAND ${CMAKE_COMMAND} -P ${CMAKE_SOURCE_DIR}/cmake/info_bin.cmake
-  WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
+  COMMAND ${CMAKE_COMMAND} -P ${ELOQSQL_SOURCE_DIR}/cmake/info_bin.cmake
+  WORKING_DIRECTORY ${ELOQSQL_BINARY_DIR}
 )
 
 INSTALL_DOCUMENTATION(README.md CREDITS COPYING THIRDPARTY COMPONENT Readme)
 
 # MDEV-6526 these files are not installed anymore
-#INSTALL_DOCUMENTATION(${CMAKE_BINARY_DIR}/Docs/INFO_SRC
-#                      ${CMAKE_BINARY_DIR}/Docs/INFO_BIN)
+#INSTALL_DOCUMENTATION(${ELOQSQL_BINARY_DIR}/Docs/INFO_SRC
+#                      ${ELOQSQL_BINARY_DIR}/Docs/INFO_BIN)
 
 IF(UNIX)
   INSTALL_DOCUMENTATION(Docs/INSTALL-BINARY Docs/README-wsrep COMPONENT Readme)
@@ -583,7 +602,7 @@ IF(NON_DISTRIBUTABLE_WARNING)
 You have linked MariaDB with ${NON_DISTRIBUTABLE_WARNING} libraries!  You may not distribute the resulting binary. If you do, you will put yourself into a legal problem with the Free Software Foundation.")
 ENDIF()
 
-IF(NOT WITHOUT_SERVER)
+IF(NOT WITHOUT_SERVER AND NOT BUILD_ELOQSQL_AS_LIBRARY)
   # Define target for minimal mtr-testable build
   ADD_CUSTOM_TARGET(minbuild)
   ADD_DEPENDENCIES(minbuild
@@ -617,8 +636,7 @@ IF(NOT WITHOUT_SERVER)
   ENDIF()
   ADD_CUSTOM_TARGET(smoketest
     COMMAND perl ./mysql-test-run.pl main.1st
-    WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/mysql-test)
+    WORKING_DIRECTORY ${ELOQSQL_BINARY_DIR}/mysql-test)
   ADD_DEPENDENCIES(smoketest minbuild)
 ENDIF()
 
-option(WITH_DYNAMO_DB "With DynamoDB" OFF)

README.md

@@ -5,7 +5,7 @@ A MySQL-compatible, high performance, elastic, distributed SQL database.
 ---
 
 ## Overview
-EloqSQL is a distributed SQL database designed to combine MySQL compatibility with the scalability and performance of modern distributed systems. Built on top of [Data Substrate](https://www.eloqdata.com/blog/2024/08/11/data-substrate), it replaces traditional storage engines like InnoDB with a flexible, distributed and high-performance eloq engine: [Transaction Service](https://github.com/eloqdata/tx_service). It has distributed buffer pool and support Cassandra, ScyllaDB and DynamoDB as the underlying data store.
+EloqSQL is a distributed SQL database designed to combine MySQL compatibility with the scalability and performance of modern distributed systems. Built on top of [Data Substrate](https://www.eloqdata.com/blog/2024/08/11/data-substrate), it replaces traditional storage engines like InnoDB with a flexible, distributed and high-performance eloq engine: [Transaction Service](https://github.com/eloqdata/tx_service). It has distributed buffer pool and support **RocksDB**, **RocksDB-Cloud**, and **DynamoDB** as storage.
 
 EloqSQL delivers full ACID transactions, elastic scaling, and efficient resource utilization, making it ideal for demanding workloads.
 
@@ -129,9 +129,9 @@ cmake --install . --config RelWithDebInfo
 ```
 
 ### 4. Set Up Storage Backend
-EloqSQL use s3 as storage backends. For testing, just deploy a s3 emulator.
+EloqSQL relies on S3-compatible object storage for durable data. In development you can emulate this by running [MinIO](https://min.io/).
 
-Download and start a MINIO instance:
+Download and start a MinIO instance:
 
 ```bash
 wget https://dl.min.io/server/minio/release/linux-amd64/minio
@@ -140,7 +140,12 @@ chmod +x minio
 ```
 
 ### 5. Configure EloqSQL
-Edit my-config.cnf with the following example settings:
+EloqSQL now uses **two** configuration files:
+
+1. A MySQL config (`my-config.cnf`) for server/bootstrap settings.
+2. A Data Substrate config (`data_substrate.cnf`) that contains data substrate related settings. This file must be in INI format, and MySQL references it through the `eloq_config` system variable.
+
+#### MySQL config (`my-config.cnf`)
 
 ```
 [mariadb]
@@ -153,16 +158,34 @@ socket=/tmp/mysqld3316.sock
 plugin_load_add=ha_eloq
 eloq
 eloq_kv_storage=eloqds
-eloq_dss_rocksdb_cloud_endpoint_url=http://127.0.0.1:9000
-eloq_dss_rocksdb_cloud_bucket_name=eloqsql
-eloq_dss_rocksdb_cloud_bucket_prefix=dss-
-eloq_dss_rocksdb_cloud_region=ap-northeast-1
-eloq_aws_access_key_id=minioadmin
-eloq_aws_secret_key=minioadmin
-eloq_local_ip=127.0.0.1:8000
-eloq_ip_list=127.0.0.1:8000
+eloq_config=${HOME}/data_substrate.cnf
 ```
 
+> **Note:** `eloq_config` must point to an absolute path that the `mysqld` process can read. Without this file, the Data Substrate initialization fails and the server will not start.
+
+#### Data Substrate config (`data_substrate.cnf`)
+
+```
+[local]
+tx_ip=127.0.0.1
+tx_port=8000
+txlog_rocksdb_cloud_s3_endpoint_url=http://127.0.0.1:9000
+txlog_rocksdb_cloud_bucket_name=eloqsql
+txlog_rocksdb_cloud_bucket_prefix=txlog-
+txlog_rocksdb_cloud_region=ap-northeast-1
+
+[store]
+aws_access_key_id=minioadmin
+aws_secret_key=minioadmin
+rocksdb_cloud_s3_endpoint_url=http://127.0.0.1:9000
+rocksdb_cloud_bucket_name=eloqsql
+rocksdb_cloud_bucket_prefix=dss-
+rocksdb_cloud_region=ap-northeast-1
+eloq_dss_config_file_path=
+```
+
+You can copy `storage/eloq/mysql-test/mono_main/data_substrate.cnf` as a starting point and then set the values to match your environment. Any option related to Data Substrate (TX service, storage backends, AWS credentials, etc.) must live in this INI file so that the `eloq_config` pointer can load it during server startup.
+
 ### 6. Bootstrap EloqSQL Node
 Initialize the database:
 
@@ -193,47 +216,39 @@ sudo ${INSTALL_DIR}/bin/mysql -u root -S /tmp/mysqld3316.sock
 ---
 
 ### 9. Run mtr test locally
-Shutdown EloqSQL before running mtr tests.
+MariaDB’s **mysql-test-runner (mtr)** suite still works with EloqSQL. Stop any running server before launching tests so the harness can manage its own instances.
 
 #### 1. mono_basic and mono_main:
-Edit eloqsql/concourse/scripts/mtr_bootstrap.cnf with the following example settings:
+Edit `eloqsql/concourse/scripts/mtr_bootstrap.cnf` to point at the shared Data Substrate template. Example:
 ```ini
 [mariadb]
 
 ...
 
-eloq_aws_access_key_id=minioadmin
-eloq_aws_secret_key=minioadmin
-eloq_txlog_rocksdb_cloud_endpoint_url=http://127.0.0.1:9000
-eloq_txlog_rocksdb_cloud_bucket_name = eloqsql-mtr-test
-eloq_txlog_rocksdb_cloud_bucket_prefix = txlog-
-eloq_txlog_rocksdb_cloud_region = ap-northeast-1
-eloq_dss_rocksdb_cloud_endpoint_url=http://127.0.0.1:9000
-eloq_dss_rocksdb_cloud_bucket_name = eloqsql-mtr-test
-eloq_dss_rocksdb_cloud_bucket_prefix = dss-
-eloq_dss_rocksdb_cloud_region = ap-northeast-1
-
-...
-
+eloq_config=/abs/path/to/eloqsql/concourse/scripts/mtr_bootstrap_ds.cnf
 ```
 
-Edit eloqsql/mysql-test/include/eloq_kv_dss.cnf with the following example settings:
+Then populate `eloqsql/concourse/scripts/mtr_bootstrap_ds.cnf` with your S3/Dynamo/AWS credentials:
 ```ini
-[mysqld]
-eloq_aws_access_key_id=minioadmin
-eloq_aws_secret_key=minioadmin
-eloq_txlog_rocksdb_cloud_endpoint_url=http://127.0.0.1:9000
-eloq_txlog_rocksdb_cloud_bucket_name = eloqsql-mtr-test
-eloq_txlog_rocksdb_cloud_bucket_prefix = txlog-
-eloq_txlog_rocksdb_cloud_region = ap-northeast-1
-eloq_dss_rocksdb_cloud_endpoint_url=http://127.0.0.1:9000
-eloq_dss_rocksdb_cloud_bucket_name = eloqsql-mtr-test
-eloq_dss_rocksdb_cloud_bucket_prefix = dss-
-eloq_dss_rocksdb_cloud_region = ap-northeast-1
-eloq_dss_rocksdb_cloud_sst_file_cache_size = 20GB
+[local]
+tx_ip=127.0.0.1
+tx_port=8000
+txlog_rocksdb_cloud_s3_endpoint_url=http://127.0.0.1:9000
+txlog_rocksdb_cloud_bucket_name=eloqsql-mtr-test
+txlog_rocksdb_cloud_bucket_prefix=txlog-
+txlog_rocksdb_cloud_region=ap-northeast-1
 
+[store]
+aws_access_key_id=minioadmin
+aws_secret_key=minioadmin
+rocksdb_cloud_s3_endpoint_url=http://127.0.0.1:9000
+rocksdb_cloud_bucket_name=eloqsql-mtr-test
+rocksdb_cloud_bucket_prefix=dss-
+rocksdb_cloud_region=ap-northeast-1
 ```
 
+> Tip: the same values should be copied into `storage/eloq/mysql-test/mono_basic/data_substrate.cnf` and `storage/eloq/mysql-test/mono_main/data_substrate.cnf` so the suite-specific configs stay consistent.
+
 Run mono_basic and mono_main test
 ```bash
 pkill -9 dss_server
@@ -251,26 +266,29 @@ build/mysql-test/mtr --clean-txlog-bucket-restart --suite=mono_basic,mono_main -
 
 #### 2. mono_multi:
 
-Edit eloqsql/concourse/scripts/mtr_multi_bootstrap.cnf with the following example settings:
-```ini
-[mariadb]
+Edit `eloqsql/concourse/scripts/mtr_multi_bootstrap.cnf` so that `eloq_config` references `concourse/scripts/mtr_multi_bootstrap_ds.cnf` (same pattern as mono_basic).
 
-...
+Configure `eloqsql/concourse/scripts/mtr_multi_bootstrap_ds.cnf` with the multi-node Data Substrate details:
+```ini
+[local]
+tx_ip=127.0.0.1
+tx_port=8000
+txlog_rocksdb_cloud_s3_endpoint_url=http://127.0.0.1:9000
+txlog_rocksdb_cloud_bucket_name=eloqsql-mtr-test
+txlog_rocksdb_cloud_bucket_prefix=txlog-
+txlog_rocksdb_cloud_region=ap-northeast-1
 
-eloq_aws_access_key_id=minioadmin
-eloq_aws_secret_key=minioadmin
-eloq_txlog_rocksdb_cloud_endpoint_url=http://127.0.0.1:9000
-eloq_txlog_rocksdb_cloud_bucket_name = eloqsql-mtr-test
-eloq_txlog_rocksdb_cloud_bucket_prefix = txlog-
-eloq_txlog_rocksdb_cloud_region = ap-northeast-1
-eloq_dss_rocksdb_cloud_endpoint_url=http://127.0.0.1:9000
-eloq_dss_rocksdb_cloud_bucket_name = eloqsql-mtr-test
-eloq_dss_rocksdb_cloud_bucket_prefix = dss-
-eloq_dss_rocksdb_cloud_region = ap-northeast-1
-eloq_dss_peer_node=localhost:9100 # add this line for mono_multi
-
-...
+[cluster]
+tx_ip_port_list=127.0.0.1:8000
+eloq_dss_peer_node=localhost:9100
 
+[store]
+aws_access_key_id=minioadmin
+aws_secret_key=minioadmin
+rocksdb_cloud_s3_endpoint_url=http://127.0.0.1:9000
+rocksdb_cloud_bucket_name=eloqsql-mtr-test
+rocksdb_cloud_bucket_prefix=dss-
+rocksdb_cloud_region=ap-northeast-1
 ```
 
 Edit eloqsql/concourse/scripts/dss_server.ini with the following example settings:
@@ -291,23 +309,7 @@ aws_secret_key=minioadmin
 
 ```
 
-Edit eloqsql/mysql-test/include/eloq_kv_dss.cnf with the following example settings:
-```ini
-[mysqld]
-eloq_aws_access_key_id=minioadmin
-eloq_aws_secret_key=minioadmin
-eloq_txlog_rocksdb_cloud_endpoint_url=http://127.0.0.1:9000
-eloq_txlog_rocksdb_cloud_bucket_name = eloqsql-mtr-test
-eloq_txlog_rocksdb_cloud_bucket_prefix = txlog-
-eloq_txlog_rocksdb_cloud_region = ap-northeast-1
-eloq_dss_rocksdb_cloud_endpoint_url=http://127.0.0.1:9000
-eloq_dss_rocksdb_cloud_bucket_name = eloqsql-mtr-test
-eloq_dss_rocksdb_cloud_bucket_prefix = dss-
-eloq_dss_rocksdb_cloud_region = ap-northeast-1
-eloq_dss_peer_node = localhost:9100 # add this line for mono_multi
-eloq_dss_rocksdb_cloud_sst_file_cache_size = 20GB
-
-```
+Each `storage/eloq/mysql-test/mono_multi/data_substrate*.cnf` file also mirrors these values for the individual mysqld instances in the suite.
 
 Run mono_multi test
 ```bash