diff --git a/include/PgSQL_Monitor.hpp b/include/PgSQL_Monitor.hpp index 70b6281db0..45661cc58c 100644 --- a/include/PgSQL_Monitor.hpp +++ b/include/PgSQL_Monitor.hpp @@ -16,6 +16,8 @@ #define MONITOR_SQLITE_TABLE_PGSQL_SERVER_READ_ONLY_LOG "CREATE TABLE pgsql_server_read_only_log ( hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 3306 , time_start_us INT NOT NULL DEFAULT 0 , success_time_us INT DEFAULT 0 , read_only INT DEFAULT 1 , error VARCHAR , PRIMARY KEY (hostname, port, time_start_us))" +#define MONITOR_SQLITE_TABLE_PGSQL_SERVER_REPLICATION_LAG_LOG "CREATE TABLE pgsql_server_replication_lag_log ( hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 3306 , time_start_us INT NOT NULL DEFAULT 0 , success_time_us INT DEFAULT 0 , repl_lag INT DEFAULT 0 , error VARCHAR , PRIMARY KEY (hostname, port, time_start_us))" + #define MONITOR_SQLITE_TABLE_PGSQL_SERVERS "CREATE TABLE pgsql_servers (hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 3306 , status INT CHECK (status IN (0, 1, 2, 3, 4)) NOT NULL DEFAULT 0 , use_ssl INT CHECK (use_ssl IN(0,1)) NOT NULL DEFAULT 0 , PRIMARY KEY (hostname, port) )" #define MONITOR_SQLITE_TABLE_PROXYSQL_SERVERS "CREATE TABLE proxysql_servers (hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 6032 , weight INT CHECK (weight >= 0) NOT NULL DEFAULT 0 , comment VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (hostname, port) )" @@ -39,6 +41,8 @@ struct PgSQL_Monitor { uint64_t ping_check_OK { 0 }; uint64_t readonly_check_ERR { 0 }; uint64_t readonly_check_OK { 0 }; + uint64_t repl_lag_check_ERR { 0 }; + uint64_t repl_lag_check_OK { 0 }; uint64_t ssl_connections_OK { 0 }; uint64_t non_ssl_connections_OK { 0 }; /////////////////////////////////////////////////////////////////////////// @@ -56,6 +60,10 @@ struct PgSQL_Monitor { const_cast("pgsql_server_read_only_log"), const_cast(MONITOR_SQLITE_TABLE_PGSQL_SERVER_READ_ONLY_LOG) }, + { + const_cast("pgsql_server_replication_lag_log"), + const_cast(MONITOR_SQLITE_TABLE_PGSQL_SERVER_REPLICATION_LAG_LOG) + }, }; std::vector tables_defs_monitor_internal { diff --git a/include/PgSQL_Thread.h b/include/PgSQL_Thread.h index 3cd4888a59..857f2d6644 100644 --- a/include/PgSQL_Thread.h +++ b/include/PgSQL_Thread.h @@ -904,6 +904,7 @@ class PgSQL_Threads_Handler bool monitor_replication_lag_group_by_host; //! How frequently a replication lag check is performed. Unit: 'ms'. int monitor_replication_lag_interval; + int monitor_replication_lag_interval_window; //! Read only check timeout. Unit: 'ms'. int monitor_replication_lag_timeout; int monitor_replication_lag_count; diff --git a/include/proxysql_structs.h b/include/proxysql_structs.h index dd09e9afa5..f1485c4df2 100644 --- a/include/proxysql_structs.h +++ b/include/proxysql_structs.h @@ -749,6 +749,7 @@ enum PROXYSQL_MYSQL_ERR { ER_PROXYSQL_CONNECT_TIMEOUT = 9020, ER_PROXYSQL_READONLY_TIMEOUT = 9021, ER_PROXYSQL_FAST_FORWARD_CONN_CREATE = 9022, + ER_PROXYSQL_REPL_LAG_TIMEOUT = 9023, }; enum proxysql_session_type { @@ -1205,6 +1206,11 @@ __thread int pgsql_thread___monitor_read_only_interval; __thread int pgsql_thread___monitor_read_only_interval_window; __thread int pgsql_thread___monitor_read_only_timeout; __thread int pgsql_thread___monitor_read_only_max_timeout_count; +__thread int pgsql_thread___monitor_replication_lag_interval; +__thread int pgsql_thread___monitor_replication_lag_interval_window; +__thread int pgsql_thread___monitor_replication_lag_timeout; +__thread int pgsql_thread___monitor_replication_lag_count; +__thread char* pgsql_thread___monitor_replication_lag_use_percona_heartbeat; __thread bool pgsql_thread___monitor_writer_is_also_reader; __thread int pgsql_thread___monitor_threads; __thread char* pgsql_thread___monitor_username; @@ -1527,6 +1533,11 @@ extern __thread int pgsql_thread___monitor_ping_timeout; extern __thread int pgsql_thread___monitor_read_only_interval; extern __thread int pgsql_thread___monitor_read_only_interval_window; extern __thread int pgsql_thread___monitor_read_only_timeout; +extern __thread int pgsql_thread___monitor_replication_lag_interval; +extern __thread int pgsql_thread___monitor_replication_lag_interval_window; +extern __thread int pgsql_thread___monitor_replication_lag_timeout; +extern __thread int pgsql_thread___monitor_replication_lag_count; +extern __thread char* pgsql_thread___monitor_replication_lag_use_percona_heartbeat; extern __thread int pgsql_thread___monitor_read_only_max_timeout_count; extern __thread bool pgsql_thread___monitor_writer_is_also_reader; extern __thread int pgsql_thread___monitor_threads; diff --git a/lib/PgSQL_HostGroups_Manager.cpp b/lib/PgSQL_HostGroups_Manager.cpp index d5507f0d75..89fb5bd95b 100644 --- a/lib/PgSQL_HostGroups_Manager.cpp +++ b/lib/PgSQL_HostGroups_Manager.cpp @@ -2667,7 +2667,7 @@ void PgSQL_HostGroups_Manager::replication_lag_action_inner(PgSQL_HGC *myhgc, co ) { // always increase the counter mysrvc->cur_replication_lag_count += 1; - if (mysrvc->cur_replication_lag_count >= (unsigned int)mysql_thread___monitor_replication_lag_count) { + if (mysrvc->cur_replication_lag_count >= (unsigned int)pgsql_thread___monitor_replication_lag_count) { proxy_warning("Shunning server %s:%d from HG %u with replication lag of %d second, count number: '%d'\n", address, port, myhgc->hid, current_replication_lag, mysrvc->cur_replication_lag_count); mysrvc->status=MYSQL_SERVER_STATUS_SHUNNED_REPLICATION_LAG; } else { @@ -2678,7 +2678,7 @@ void PgSQL_HostGroups_Manager::replication_lag_action_inner(PgSQL_HGC *myhgc, co myhgc->hid, current_replication_lag, mysrvc->cur_replication_lag_count, - mysql_thread___monitor_replication_lag_count + pgsql_thread___monitor_replication_lag_count ); } } else { @@ -2716,7 +2716,7 @@ void PgSQL_HostGroups_Manager::replication_lag_action(const std::list(server); const int current_replication_lag = std::get(server); - if (mysql_thread___monitor_replication_lag_group_by_host == false) { + if (/* pgsql_thread___monitor_replication_lag_group_by_host == */ false) { // feature currently not enabled // legacy check. 1 check per server per hostgroup PgSQL_HGC *myhgc = MyHGC_find(hid); replication_lag_action_inner(myhgc,address.c_str(),port,current_replication_lag); diff --git a/lib/PgSQL_Monitor.cpp b/lib/PgSQL_Monitor.cpp index fce00c5307..c693de0227 100644 --- a/lib/PgSQL_Monitor.cpp +++ b/lib/PgSQL_Monitor.cpp @@ -36,6 +36,22 @@ const char PING_QUERY[] { "" }; * @details If the server is not in this mode would be assumed to be a primary. */ const char READ_ONLY_QUERY[] { "SELECT pg_is_in_recovery()" }; +/** + * @brief Used to detect the current replication lag in a PostgreSQL instance. + * @details Lag measurement is based in a difference between the most recent WAL location that has been + * received and synced to disk (pg_last_wal_receive_lsn) and the most recent WAL location that has been + * replayed the most recent WAL location that has been replayed (pg_last_wal_replay_lsn). + */ +const char REPLICATION_LAG_QUERY[] { + "SELECT CASE WHEN pg_last_wal_receive_lsn() = pg_last_wal_replay_lsn() THEN 0 ELSE GREATEST" + " (0, EXTRACT (EPOCH FROM now() - pg_last_xact_replay_timestamp())) END AS replication_lag" +}; +/* + * @brief Used to detect current replication lag in a PostgreSQL instance when pt-heartbeat is used. + */ +const char REPLICATION_LAG_QUERY_PT_HEARTBEAT[] { + "SELECT EXTRACT(EPOCH FROM (LOCALTIMESTAMP - ts :: timestamp)) AS Seconds_Behind_Master FROM " +}; template void append(std::vector& dest, std::vector&& src) { @@ -258,9 +274,24 @@ void update_monitor_pgsql_servers(SQLite3_result* rs, SQLite3DB* db) { } } -enum class task_type_t { ping, connect, readonly }; +enum class task_type_t { ping, connect, readonly, repl_lag }; + +const char* get_task_type_str(task_type_t task_type) { + if (task_type == task_type_t::ping) { + return "ping"; + } else if (task_type == task_type_t::connect) { + return "connect"; + } else if (task_type == task_type_t::readonly) { + return "readonly"; + } else if (task_type == task_type_t::repl_lag) { + return "replication_lag"; + } else { + assert(0 && "Invalid task type"); + } +} struct mon_srv_t { + int32_t hostgroup_id; string addr; uint16_t port; bool ssl; @@ -290,6 +321,10 @@ struct readonly_res_t { int32_t val; }; +struct repl_lag_res_t { + int32_t val; +}; + struct ping_conf_t { unique_ptr srvs_info; ping_params_t params; @@ -323,10 +358,50 @@ struct readonly_conf_t { readonly_params_t params; }; +struct repl_lag_params_t { + int32_t interval; + double interval_window; + int32_t timeout; + int32_t ping_max_failures; + int32_t ping_interval; + mf_unique_ptr pt_heartbeat {}; + + repl_lag_params_t( + int32_t _interval, + double _interval_window, + int32_t _timeout, + int32_t _ping_max_failures, + int32_t _ping_interval, + char* _pt_heartbeat + ) : + interval(_interval), + interval_window(_interval_window), + timeout(_timeout), + ping_max_failures(_ping_max_failures), + ping_interval(_ping_interval), + pt_heartbeat(_pt_heartbeat ? strdup(_pt_heartbeat) : nullptr) + {} + + repl_lag_params_t(const repl_lag_params_t& o) : + interval(o.interval), + interval_window(o.interval_window), + timeout(o.timeout), + ping_max_failures(o.ping_max_failures), + ping_interval(o.ping_interval), + pt_heartbeat(o.pt_heartbeat ? strdup(o.pt_heartbeat.get()) : nullptr) + {} +}; + +struct repl_lag_conf_t { + unique_ptr srvs_info; + repl_lag_params_t params; +}; + struct tasks_conf_t { ping_conf_t ping; connect_conf_t connect; readonly_conf_t readonly; + repl_lag_conf_t repl_lag; mon_user_t user_info; }; @@ -361,9 +436,10 @@ vector ext_srvs(const unique_ptr& srvs_info) { srvs.reserve(srvs_info->rows.size()); for (const auto& row : srvs_info->rows) { srvs.push_back({ - string { row->fields[0] }, - static_cast(std::atoi(row->fields[1])), - static_cast(std::atoi(row->fields[2])), + static_cast(std::atoi(row->fields[0])), + string { row->fields[1] }, + static_cast(std::atoi(row->fields[2])), + static_cast(std::atoi(row->fields[3])), mon_srv_t::ssl_opts_t { string { pgsql_thread___ssl_p2s_key ? pgsql_thread___ssl_p2s_key : ""}, string { pgsql_thread___ssl_p2s_cert ? pgsql_thread___ssl_p2s_cert : "" }, @@ -394,22 +470,28 @@ tasks_conf_t fetch_updated_conf(PgSQL_Monitor* mon, PgSQL_HostGroups_Manager* hg } unique_ptr ping_srvrs { fetch_mon_srvs_conf(mon, - "SELECT hostname, port, MAX(use_ssl) use_ssl FROM monitor_internal.pgsql_servers" + "SELECT 0 hostgroup_id, hostname, port, MAX(use_ssl) use_ssl FROM monitor_internal.pgsql_servers" " GROUP BY hostname, port ORDER BY RANDOM()" )}; unique_ptr connect_srvrs { fetch_mon_srvs_conf(mon, - "SELECT hostname, port, MAX(use_ssl) use_ssl FROM monitor_internal.pgsql_servers" + "SELECT 0 hostgroup_id, hostname, port, MAX(use_ssl) use_ssl FROM monitor_internal.pgsql_servers" " GROUP BY hostname, port ORDER BY RANDOM()" )}; unique_ptr readonly_srvs { fetch_hgm_srvs_conf(hgm, - "SELECT hostname, port, MAX(use_ssl) use_ssl, check_type, reader_hostgroup" + "SELECT hostgroup_id, hostname, port, MAX(use_ssl) use_ssl, check_type, reader_hostgroup" " FROM pgsql_servers JOIN pgsql_replication_hostgroups" " ON hostgroup_id=writer_hostgroup OR hostgroup_id=reader_hostgroup" " WHERE status NOT IN (2,3) GROUP BY hostname, port ORDER BY RANDOM()" )}; + unique_ptr repl_srvs { fetch_hgm_srvs_conf(hgm, + "SELECT hostgroup_id, hostname, port, MAX(use_ssl) use_ssl FROM pgsql_servers" + " JOIN pgsql_replication_hostgroups ON hostgroup_id=writer_hostgroup OR hostgroup_id=reader_hostgroup" + " WHERE max_replication_lag > 0 AND status NOT IN (2,3)" + " GROUP BY hostgroup_id, hostname, port ORDER BY RANDOM()" + )}; return tasks_conf_t { ping_conf_t { @@ -446,11 +528,22 @@ tasks_conf_t fetch_updated_conf(PgSQL_Monitor* mon, PgSQL_HostGroups_Manager* hg pgsql_thread___monitor_writer_is_also_reader } }, + repl_lag_conf_t { + std::move(repl_srvs), + repl_lag_params_t { + pgsql_thread___monitor_replication_lag_interval * 1000, + pgsql_thread___monitor_replication_lag_interval_window / 100.0, + pgsql_thread___monitor_replication_lag_timeout * 1000, + pgsql_thread___monitor_ping_max_failures, + pgsql_thread___monitor_ping_interval * 1000, + pgsql_thread___monitor_replication_lag_use_percona_heartbeat + } + }, mon_user_t { pgsql_thread___monitor_username, pgsql_thread___monitor_password, pgsql_thread___monitor_dbname - } + }, }; } @@ -562,15 +655,29 @@ short handle_async_check_cont(state_t& st, short _) { int row_count = PQntuples(res); if (row_count > 0) { - const char* value_str { PQgetvalue(res, 0, 0) }; - bool value { strcmp(value_str, "t") == 0 }; - - set_finish_st(st, ASYNC_QUERY_END, - op_result_t { - new readonly_res_t { value }, - [] (void* v) { delete static_cast(v); } - } - ); + if (st.task.type == task_type_t::readonly) { + const char* value_str { PQgetvalue(res, 0, 0) }; + bool value { strcmp(value_str, "t") == 0 }; + + set_finish_st(st, ASYNC_QUERY_END, + op_result_t { + new readonly_res_t { value }, + [] (void* v) { delete static_cast(v); } + } + ); + } else if (st.task.type == task_type_t::repl_lag) { + const char* value_str { PQgetvalue(res, 0, 0) }; + int32_t value { std::atoi(value_str) }; + + set_finish_st(st, ASYNC_QUERY_END, + op_result_t { + new repl_lag_res_t { value }, + [] (void* v) { delete static_cast(v); } + } + ); + } else { + assert(0 && "Invalid task type"); + } } else { const mon_srv_t& srv { st.task.op_st.srv_info }; const char err_t[] { "Invalid number of rows '%d'" }; @@ -578,8 +685,8 @@ short handle_async_check_cont(state_t& st, short _) { cstr_format(err_b, err_t, row_count); proxy_error( - "Monitor readonly failed addr='%s:%d' status=%d error='%s'\n", - srv.addr.c_str(), srv.port, status, err_b + "Monitor %s failed addr='%s:%d' status=%d error='%s'\n", + get_task_type_str(st.task.type), srv.addr.c_str(), srv.port, status, err_b ); set_failed_st(st, ASYNC_QUERY_FAILED, mf_unique_ptr(strdup(err_b))); } @@ -602,6 +709,12 @@ short handle_async_check_cont(state_t& st, short _) { srv.addr.c_str(), srv.port, status, err.get() ); set_failed_st(st, ASYNC_QUERY_FAILED, std::move(err)); + } else if (st.task.type == task_type_t::repl_lag) { + proxy_error( + "Monitor repl_lag failed addr='%s:%d' status=%d error='%s'\n", + srv.addr.c_str(), srv.port, status, err.get() + ); + set_failed_st(st, ASYNC_QUERY_FAILED, std::move(err)); } else { assert(0 && "Invalid task type"); } @@ -648,6 +761,8 @@ pair handle_async_connect_cont(state_t& st, short revent) { proc_again = true; } else if (st.task.type == task_type_t::readonly) { proc_again = true; + } else if (st.task.type == task_type_t::repl_lag) { + proc_again = true; } else { assert(0 && "Non-implemented task-type"); } @@ -669,13 +784,38 @@ pair handle_async_connect_cont(state_t& st, short revent) { return { req_events, proc_again }; } +string get_task_query(const state_t& st) { + const task_type_t task_type { st.task.type }; + + if (task_type == task_type_t::ping) { + return PING_QUERY; + } else if (task_type == task_type_t::readonly) { + return READ_ONLY_QUERY; + } else if (task_type == task_type_t::repl_lag) { + repl_lag_params_t* params { + static_cast(st.task.op_st.op_params.get()) + }; + + if (params->pt_heartbeat && strlen(params->pt_heartbeat.get())) { + // FIXME: This is a SQL injection vulnerability. + // pt-heartbeat support for PostgreSQL is currently disabled. + // return string { REPLICATION_LAG_QUERY_PT_HEARTBEAT } + params->pt_heartbeat.get(); + return REPLICATION_LAG_QUERY; + } else { + return REPLICATION_LAG_QUERY; + } + } else { + assert(0 && "Invalid task type"); + } +} + short handle_async_connect_end(state_t& st, short _) { pgsql_conn_t& pgconn { st.conn }; short req_events { 0 }; - const char* QUERY { st.task.type == task_type_t::ping ? PING_QUERY : READ_ONLY_QUERY }; + const string QUERY { get_task_query(st) }; - int rc = PQsendQuery(pgconn.conn, QUERY); + int rc = PQsendQuery(pgconn.conn, QUERY.c_str()); if (rc == 0) { const mon_srv_t& srv { st.task.op_st.srv_info }; auto err { strdup_no_lf(PQerrorMessage(pgconn.conn)) }; @@ -692,6 +832,12 @@ short handle_async_connect_end(state_t& st, short _) { srv.addr.c_str(), srv.port, err.get() ); set_failed_st(st, ASYNC_QUERY_FAILED, std::move(err)); + } else if (st.task.type == task_type_t::repl_lag) { + proxy_error( + "Monitor repl_lag start failed addr='%s:%d' error='%s'\n", + srv.addr.c_str(), srv.port, err.get() + ); + set_failed_st(st, ASYNC_QUERY_FAILED, std::move(err)); } else { assert(0 && "Invalid task type"); } @@ -714,6 +860,12 @@ short handle_async_connect_end(state_t& st, short _) { srv.addr.c_str(), srv.port, err.get() ); set_failed_st(st, ASYNC_QUERY_FAILED, std::move(err)); + } else if (st.task.type == task_type_t::repl_lag) { + proxy_error( + "Monitor repl_lag start failed addr='%s:%d' error='%s'\n", + srv.addr.c_str(), srv.port, err.get() + ); + set_failed_st(st, ASYNC_QUERY_FAILED, std::move(err)); } else { assert(0 && "Invalid task type"); } @@ -724,6 +876,8 @@ short handle_async_connect_end(state_t& st, short _) { pgconn.state = ASYNC_ST::ASYNC_PING_CONT; } else if (st.task.type == task_type_t::readonly) { pgconn.state = ASYNC_ST::ASYNC_QUERY_CONT; + } else if (st.task.type == task_type_t::repl_lag) { + pgconn.state = ASYNC_ST::ASYNC_QUERY_CONT; } else { assert(0 && "Invalid task type"); } @@ -869,6 +1023,12 @@ uint64_t get_connpool_cleanup_intv(task_st_t& task) { static_cast(task.op_st.op_params.get()) }; + res = params->ping_interval; + } else if (task.type == task_type_t::repl_lag){ + repl_lag_params_t* params { + static_cast(task.op_st.op_params.get()) + }; + res = params->ping_interval; } else { assert(0 && "Non-implemented task-type"); @@ -1041,6 +1201,7 @@ struct tasks_intvs_t { uint64_t next_ping_at; uint64_t next_connect_at; uint64_t next_readonly_at; + uint64_t next_repl_lag_at; }; struct task_poll_t { @@ -1285,7 +1446,8 @@ bool is_task_success(pgsql_conn_t& c, task_st_t& st) { || (c.state != ASYNC_ST::ASYNC_QUERY_FAILED && c.state != ASYNC_QUERY_TIMEOUT)) && ((c.state == ASYNC_ST::ASYNC_CONNECT_END && st.type == task_type_t::connect) || (c.state == ASYNC_ST::ASYNC_PING_END && st.type == task_type_t::ping) - || (c.state == ASYNC_ST::ASYNC_QUERY_END && st.type == task_type_t::readonly)); + || (c.state == ASYNC_ST::ASYNC_QUERY_END && st.type == task_type_t::readonly) + || (c.state == ASYNC_ST::ASYNC_QUERY_END && st.type == task_type_t::repl_lag)); } bool is_task_finish(pgsql_conn_t& c, task_st_t& st) { @@ -1295,7 +1457,8 @@ bool is_task_finish(pgsql_conn_t& c, task_st_t& st) { || (c.state == ASYNC_ST::ASYNC_QUERY_FAILED || c.state == ASYNC_ST::ASYNC_QUERY_TIMEOUT)) || (c.state == ASYNC_ST::ASYNC_CONNECT_END && st.type == task_type_t::connect) || (c.state == ASYNC_ST::ASYNC_PING_END && st.type == task_type_t::ping) - || (c.state == ASYNC_ST::ASYNC_QUERY_END && st.type == task_type_t::readonly); + || (c.state == ASYNC_ST::ASYNC_QUERY_END && st.type == task_type_t::readonly) + || (c.state == ASYNC_ST::ASYNC_QUERY_END && st.type == task_type_t::repl_lag); } void update_connect_table(SQLite3DB* db, state_t& state) { @@ -1444,6 +1607,62 @@ void update_readonly_table(SQLite3DB* db, state_t& state) { } } +void update_repl_lag_table(SQLite3DB* db, state_t& state) { + repl_lag_res_t* op_result { + static_cast(state.task.op_st.op_result.get()) + }; + + auto [rc, stmt_unique] = db->prepare_v2( + "INSERT OR REPLACE INTO pgsql_server_replication_lag_log VALUES (?1, ?2, ?3, ?4, ?5, ?6)" + ); + ASSERT_SQLITE_OK(rc, db); + sqlite3_stmt* stmt = stmt_unique.get(); + + uint64_t op_dur_us { state.task.end - state.task.start }; + + sqlite_bind_text(stmt, 1, state.task.op_st.srv_info.addr.c_str()); + sqlite_bind_int(stmt, 2, state.task.op_st.srv_info.port); + + uint64_t time_start_us { realtime_time() - op_dur_us }; + sqlite_bind_int64(stmt, 3, time_start_us); + + uint64_t succ_time_us { is_task_success(state.conn, state.task) ? op_dur_us : 0 }; + sqlite_bind_int64(stmt, 4, succ_time_us); + + if (op_result) { + sqlite_bind_int64(stmt, 5, op_result->val); + } else { + sqlite_bind_null(stmt, 5); + } + + sqlite_bind_text(stmt, 6, state.conn.err.get()); + + SAFE_SQLITE3_STEP2(stmt); + + sqlite_clear_bindings(stmt); + sqlite_reset_statement(stmt); + // RAII auto-finalizes stmt + + if (state.conn.err) { + const mon_srv_t& srv { state.task.op_st.srv_info }; + char* srv_addr { const_cast(srv.addr.c_str()) }; + int err_code { 0 }; + + if (state.conn.state != ASYNC_ST::ASYNC_QUERY_TIMEOUT) { + err_code = 9100 + state.conn.state; + } else { + err_code = ER_PROXYSQL_REPL_LAG_TIMEOUT; + }; + + PgHGM->p_update_pgsql_error_counter( + p_pgsql_error_type::proxysql, 0, srv_addr, srv.port, err_code + ); + __sync_fetch_and_add(&GloPgMon->repl_lag_check_ERR, 1); + } else { + __sync_fetch_and_add(&GloPgMon->repl_lag_check_OK, 1); + } +} + const char CHECK_HOST_ERR_LIMIT_QUERY[] { "SELECT 1" " FROM" @@ -1621,6 +1840,28 @@ void perf_readonly_actions(SQLite3DB* db, state_t& state) { } } +void perf_repl_lag_actions(SQLite3DB* db, state_t& state) { + // Update table entries + update_repl_lag_table(db, state); + + // Perform the repl_lag actions + { + const op_st_t& op_st { state.task.op_st }; + const mon_srv_t& srv { state.task.op_st.srv_info }; + + if (is_task_success(state.conn, state.task)) { + repl_lag_res_t* op_result { static_cast(op_st.op_result.get()) }; + + // TODO: Override replication is hardcoded to 'false', this should be revisited. + PgHGM->replication_lag_action({{ srv.hostgroup_id, srv.addr, srv.port, op_result->val, false }}); + } else { + proxy_error( + "Replication lag checked failed error='%s'\n", state.conn.err.get() + ); + } + } +} + uint64_t get_task_timeout(task_st_t& task) { uint64_t task_to = 0; @@ -1641,6 +1882,12 @@ uint64_t get_task_timeout(task_st_t& task) { static_cast(task.op_st.op_params.get()) }; + task_to = params->timeout; + } else if (task.type == task_type_t::repl_lag) { + repl_lag_params_t* params { + static_cast(task.op_st.op_params.get()) + }; + task_to = params->timeout; } else { assert(0 && "Non-implemented task-type"); @@ -1669,6 +1916,12 @@ uint64_t get_task_max_ping_fails(task_st_t& task) { static_cast(task.op_st.op_params.get()) }; + max_fails = params->ping_max_failures; + } else if (task.type == task_type_t::repl_lag) { + repl_lag_params_t* params { + static_cast(task.op_st.op_params.get()) + }; + max_fails = params->ping_max_failures; } else { assert(0 && "Non-implemented task-type"); @@ -1717,6 +1970,18 @@ void proc_task_state(state_t& state, uint64_t task_start) { } else if (is_task_finish(state.conn, state.task)) { perf_readonly_actions(&GloPgMon->monitordb, state); } + } else if (state.task.type == task_type_t::repl_lag) { + if (monotonic_time() - state.task.start > get_task_timeout(state.task)) { + // TODO: Unified state processing + pg_conn.state = ASYNC_ST::ASYNC_QUERY_TIMEOUT; + pg_conn.err = mf_unique_ptr(strdup("Operation timed out")); + state.task.end = monotonic_time(); + + // TODO: proxy_error + metrics update + perf_repl_lag_actions(&GloPgMon->monitordb, state); + } else if (is_task_finish(state.conn, state.task)) { + perf_repl_lag_actions(&GloPgMon->monitordb, state); + } } else { assert(0 && "Non-implemented task-type"); } @@ -1751,6 +2016,20 @@ void* worker_thread(void* args) { bool recv_stop_signal = 0; uint64_t prev_it_time = 0; + // VARIABLE SYNCHRONIZATION + /////////////////////////////////////////////////////////////////////////// + // NOTE: Ideally this section should be removed. It's required since some monitoring + // actions can internally use `pgsql_thread___` variables. This actions normally + // belong to 'PgSQL_HostGroups_Manager' and are out of the scope of this module, for + // now, until a refactor of those actions can take place and parametrize this + // configurations, this sync mechanism is required. + /////////////////////////////////////////////////////////////////////////// + // Initial Monitor thread variables version + unsigned int PgSQL_Thread__variables_version = GloPTH->get_global_version(); + // PgSQL thread structure used for variable refreshing + unique_ptr pgsql_thread { init_pgsql_thread_struct() }; + /////////////////////////////////////////////////////////////////////////// + // Insert dummy task for scheduler comms add_scheduler_comm_task(tasks_queue, task_poll); @@ -1765,6 +2044,18 @@ void* worker_thread(void* args) { } } + // VARIABLE SYNCHRONIZATION + /////////////////////////////////////////////////////////////////////// + // See NOTE above on this section. + /////////////////////////////////////////////////////////////////////// + // Check variable version changes; refresh if needed + unsigned int glover = GloPTH->get_global_version(); + if (PgSQL_Thread__variables_version < glover) { + PgSQL_Thread__variables_version = glover; + pgsql_thread->refresh_variables(); + } + /////////////////////////////////////////////////////////////////////// + // Fetch the next tasks from the queue { std::lock_guard tasks_mutex { tasks_queue.mutex }; @@ -1972,6 +2263,10 @@ const char MAINT_READONLY_LOG_QUERY[] { "DELETE FROM pgsql_server_read_only_log WHERE time_start_us < ?1" }; +const char MAINT_REPLICATION_LAG_LOG_QUERY[] { + "DELETE FROM pgsql_server_replication_lag_log WHERE time_start_us < ?1" +}; + /** * @brief Performs the required maintenance in the monitor log tables. * @param tasks_conf The updated tasks config for the interval. @@ -2007,6 +2302,15 @@ void maint_mon_tables( &GloPgMon->monitordb, MAINT_READONLY_LOG_QUERY, tasks_conf.ping.params ); } + + if (next_intvs.next_repl_lag_at <= intv_start) { + proxy_debug(PROXY_DEBUG_MONITOR, 5, + "Performed REPLICATION_LAG table maintenance intv_start=%lu\n", intv_start + ); + maint_monitor_table( + &GloPgMon->monitordb, MAINT_REPLICATION_LAG_LOG_QUERY, tasks_conf.ping.params + ); + } } /** @@ -2072,6 +2376,23 @@ vector build_intv_batches( ); } + if (next_intvs.next_repl_lag_at <= intv_start && tasks_conf.repl_lag.srvs_info->rows_count) { + intv_tasks.push_back({ + task_type_t::repl_lag, + uint64_t(tasks_conf.repl_lag.srvs_info->rows_count), + tasks_conf.repl_lag.params.interval, + tasks_conf.repl_lag.params.interval_window, + intv_start, + create_simple_tasks( + intv_start, tasks_conf.user_info, tasks_conf.repl_lag, task_type_t::repl_lag + ) + }); + proxy_debug(PROXY_DEBUG_MONITOR, 5, + "Created REPL_LAG tasks tasks=%lu intv_start=%lu\n", + intv_tasks.back().tasks.size(), intv_start + ); + } + return intv_tasks; } @@ -2108,6 +2429,13 @@ tasks_intvs_t compute_next_intvs( upd_intvs.next_readonly_at = ULONG_MAX; } } + if (next_intvs.next_repl_lag_at <= intv_start && conf.repl_lag.params.interval != 0) { + if (conf.repl_lag.params.interval != 0) { + upd_intvs.next_repl_lag_at = intv_start + conf.repl_lag.params.interval; + } else { + upd_intvs.next_repl_lag_at = ULONG_MAX; + } + } return upd_intvs; } @@ -2149,18 +2477,20 @@ void* PgSQL_monitor_scheduler_thread() { std::min({ next_intvs.next_ping_at, next_intvs.next_connect_at, - next_intvs.next_readonly_at + next_intvs.next_readonly_at, + next_intvs.next_repl_lag_at }) }; if (cur_intv_start >= closest_intv) { proxy_debug(PROXY_DEBUG_MONITOR, 5, - "Scheduling interval time=%lu delta=%lu ping=%lu connect=%lu readonly=%lu\n", + "Scheduling interval time=%lu delta=%lu ping=%lu connect=%lu readonly=%lu repl_lag=%lu\n", cur_intv_start, cur_intv_start - closest_intv, next_intvs.next_ping_at, next_intvs.next_connect_at, - next_intvs.next_readonly_at + next_intvs.next_readonly_at, + next_intvs.next_repl_lag_at ); // Quick exit during shutdown/restart @@ -2241,7 +2571,8 @@ void* PgSQL_monitor_scheduler_thread() { std::min({ next_intvs.next_ping_at, next_intvs.next_connect_at, - next_intvs.next_readonly_at + next_intvs.next_readonly_at, + next_intvs.next_repl_lag_at }) }; const uint64_t next_intv_diff { upd_closest_intv < curtime ? 0 : upd_closest_intv - curtime }; diff --git a/lib/PgSQL_Thread.cpp b/lib/PgSQL_Thread.cpp index 79d4be49ee..acc0578db1 100644 --- a/lib/PgSQL_Thread.cpp +++ b/lib/PgSQL_Thread.cpp @@ -329,12 +329,15 @@ static char* pgsql_thread_variables_names[] = { (char*)"monitor_read_only_interval_window", (char*)"monitor_read_only_timeout", (char*)"monitor_read_only_max_timeout_count", -/* - (char*)"monitor_aws_rds_topology_discovery_interval", - (char*)"monitor_replication_lag_group_by_host", (char*)"monitor_replication_lag_interval", + (char*)"monitor_replication_lag_interval_window", (char*)"monitor_replication_lag_timeout", (char*)"monitor_replication_lag_count", + // NOTE: Disabled until 'pt-heartbeat' supports PostgreSQL is fixed: https://perconadev.atlassian.net/browse/PT-2030 + // (char*)"monitor_replication_lag_use_percona_heartbeat", +/* + (char*)"monitor_aws_rds_topology_discovery_interval", + (char*)"monitor_replication_lag_group_by_host", (char*)"monitor_groupreplication_healthcheck_interval", (char*)"monitor_groupreplication_healthcheck_timeout", (char*)"monitor_groupreplication_healthcheck_max_timeout_count", @@ -348,7 +351,6 @@ static char* pgsql_thread_variables_names[] = { (char*)"monitor_password", (char*)"monitor_dbname", /* - (char*)"monitor_replication_lag_use_percona_heartbeat", (char*)"monitor_query_interval", (char*)"monitor_query_timeout", (char*)"monitor_slave_lag_when_null", @@ -1064,6 +1066,7 @@ PgSQL_Threads_Handler::PgSQL_Threads_Handler() { variables.monitor_read_only_max_timeout_count = 3; variables.monitor_replication_lag_group_by_host = false; variables.monitor_replication_lag_interval = 10000; + variables.monitor_replication_lag_interval_window = 10; variables.monitor_replication_lag_timeout = 1000; variables.monitor_replication_lag_count = 1; /* TODO: Remove @@ -1089,9 +1092,7 @@ PgSQL_Threads_Handler::PgSQL_Threads_Handler() { variables.monitor_username = strdup((char*)"monitor"); variables.monitor_password = strdup((char*)"monitor"); variables.monitor_dbname = strdup((char*)"postgres"); -/* TODO: Remove variables.monitor_replication_lag_use_percona_heartbeat = strdup((char*)""); -*/ variables.monitor_wait_timeout = true; variables.monitor_writer_is_also_reader = true; variables.max_allowed_packet = 64 * 1024 * 1024; @@ -1310,9 +1311,8 @@ char* PgSQL_Threads_Handler::get_variable_string(char* name) { if (!strcmp(name, "monitor_username")) return strdup(variables.monitor_username); if (!strcmp(name, "monitor_password")) return strdup(variables.monitor_password); if (!strcmp(name, "monitor_dbname")) return strdup(variables.monitor_dbname); -/* - if (!strcmp(name, "monitor_replication_lag_use_percona_heartbeat")) return strdup(variables.monitor_replication_lag_use_percona_heartbeat); -*/ + // NOTE: Disabled until 'pt-heartbeat' supports PostgreSQL is fixed: https://perconadev.atlassian.net/browse/PT-2030 + // if (!strcmp(name, "monitor_replication_lag_use_percona_heartbeat")) return strdup(variables.monitor_replication_lag_use_percona_heartbeat); } if (!strncmp(name, "ssl_", 4)) { if (!strcmp(name, "ssl_p2s_ca")) { @@ -1606,9 +1606,8 @@ char* PgSQL_Threads_Handler::get_variable(char* name) { // this is the public fu if (!strcasecmp(name, "monitor_username")) return strdup(variables.monitor_username); if (!strcasecmp(name, "monitor_password")) return strdup(variables.monitor_password); if (!strcasecmp(name, "monitor_dbname")) return strdup(variables.monitor_dbname); -/* - if (!strcasecmp(name, "monitor_replication_lag_use_percona_heartbeat")) return strdup(variables.monitor_replication_lag_use_percona_heartbeat); -*/ + // NOTE: Disabled until 'pt-heartbeat' supports PostgreSQL is fixed: https://perconadev.atlassian.net/browse/PT-2030 + // if (!strcasecmp(name, "monitor_replication_lag_use_percona_heartbeat")) return strdup(variables.monitor_replication_lag_use_percona_heartbeat); } if (!strcasecmp(name, "threads")) { sprintf(intbuf, "%d", (num_threads ? num_threads : DEFAULT_NUM_THREADS)); @@ -2240,11 +2239,13 @@ char** PgSQL_Threads_Handler::get_variables_list() { VariablesPointers_int["monitor_read_only_interval_window"] = make_tuple(&variables.monitor_read_only_interval_window, 0, 100, false); VariablesPointers_int["monitor_read_only_timeout"] = make_tuple(&variables.monitor_read_only_timeout, 100, 600 * 1000, false); VariablesPointers_int["monitor_read_only_max_timeout_count"] = make_tuple(&variables.monitor_read_only_max_timeout_count, 1, 1000 * 1000, false); -/* VariablesPointers_int["monitor_replication_lag_interval"] = make_tuple(&variables.monitor_replication_lag_interval, 100, 7 * 24 * 3600 * 1000, false); + VariablesPointers_int["monitor_replication_lag_interval_window"] = make_tuple(&variables.monitor_replication_lag_interval_window, 0, 100, false); VariablesPointers_int["monitor_replication_lag_timeout"] = make_tuple(&variables.monitor_replication_lag_timeout, 100, 600 * 1000, false); VariablesPointers_int["monitor_replication_lag_count"] = make_tuple(&variables.monitor_replication_lag_count, 1, 10, false); +/* + VariablesPointers_int["monitor_groupreplication_healthcheck_interval"] = make_tuple(&variables.monitor_groupreplication_healthcheck_interval, 100, 7 * 24 * 3600 * 1000, false); VariablesPointers_int["monitor_groupreplication_healthcheck_timeout"] = make_tuple(&variables.monitor_groupreplication_healthcheck_timeout, 100, 600 * 1000, false); VariablesPointers_int["monitor_groupreplication_healthcheck_max_timeout_count"] = make_tuple(&variables.monitor_groupreplication_healthcheck_max_timeout_count, 1, 10, false); @@ -2858,13 +2859,17 @@ PgSQL_Thread::~PgSQL_Thread() { if (pgsql_thread___monitor_username) { free(pgsql_thread___monitor_username); pgsql_thread___monitor_username = NULL; } if (pgsql_thread___monitor_password) { free(pgsql_thread___monitor_password); pgsql_thread___monitor_password = NULL; } if (pgsql_thread___monitor_dbname) { free(pgsql_thread___monitor_dbname); pgsql_thread___monitor_dbname = NULL; } + if (pgsql_thread___monitor_replication_lag_use_percona_heartbeat) { + free(pgsql_thread___monitor_replication_lag_use_percona_heartbeat); + pgsql_thread___monitor_replication_lag_use_percona_heartbeat = NULL; + } /* if (mysql_thread___monitor_username) { free(mysql_thread___monitor_username); mysql_thread___monitor_username = NULL; } if (mysql_thread___monitor_password) { free(mysql_thread___monitor_password); mysql_thread___monitor_password = NULL; } - if (mysql_thread___monitor_replication_lag_use_percona_heartbeat) { - free(mysql_thread___monitor_replication_lag_use_percona_heartbeat); - mysql_thread___monitor_replication_lag_use_percona_heartbeat = NULL; + if (pgsql_thread___monitor_replication_lag_use_percona_heartbeat) { + free(pgsql_thread___monitor_replication_lag_use_percona_heartbeat); + pgsql_thread___monitor_replication_lag_use_percona_heartbeat = NULL; } */ //if (pgsql_thread___default_schema) { free(pgsql_thread___default_schema); pgsql_thread___default_schema = NULL; } @@ -4009,9 +4014,6 @@ void PgSQL_Thread::refresh_variables() { mysql_thread___monitor_username = GloPTH->get_variable_string((char*)"monitor_username"); if (mysql_thread___monitor_password) free(mysql_thread___monitor_password); mysql_thread___monitor_password = GloPTH->get_variable_string((char*)"monitor_password"); - if (mysql_thread___monitor_replication_lag_use_percona_heartbeat) free(mysql_thread___monitor_replication_lag_use_percona_heartbeat); - mysql_thread___monitor_replication_lag_use_percona_heartbeat = GloPTH->get_variable_string((char*)"monitor_replication_lag_use_percona_heartbeat"); - mysql_thread___monitor_wait_timeout = (bool)GloPTH->get_variable_int((char*)"monitor_wait_timeout"); */ pgsql_thread___monitor_writer_is_also_reader = (bool)GloPTH->get_variable_int((char*)"monitor_writer_is_also_reader"); @@ -4026,9 +4028,17 @@ void PgSQL_Thread::refresh_variables() { pgsql_thread___monitor_ping_timeout = GloPTH->get_variable_int((char*)"monitor_ping_timeout"); pgsql_thread___monitor_read_only_interval = GloPTH->get_variable_int((char*)"monitor_read_only_interval"); pgsql_thread___monitor_read_only_interval_window = GloPTH->get_variable_int((char*)"monitor_read_only_interval_window"); + pgsql_thread___monitor_replication_lag_interval = GloPTH->get_variable_int((char*)"monitor_replication_lag_interval"); + pgsql_thread___monitor_replication_lag_interval_window = GloPTH->get_variable_int((char*)"monitor_replication_lag_interval_window"); + pgsql_thread___monitor_replication_lag_timeout = GloPTH->get_variable_int((char*)"monitor_replication_lag_timeout"); + pgsql_thread___monitor_replication_lag_count = GloPTH->get_variable_int((char*)"monitor_replication_lag_count"); pgsql_thread___monitor_read_only_timeout = GloPTH->get_variable_int((char*)"monitor_read_only_timeout"); pgsql_thread___monitor_read_only_max_timeout_count = GloPTH->get_variable_int((char*)"monitor_read_only_max_timeout_count"); pgsql_thread___monitor_threads = GloPTH->get_variable_int((char*)"monitor_threads"); + /* NOTE: Disabled until 'pt-heartbeat' supports PostgreSQL is fixed: https://perconadev.atlassian.net/browse/PT-2030 + if (pgsql_thread___monitor_replication_lag_use_percona_heartbeat) free(pgsql_thread___monitor_replication_lag_use_percona_heartbeat); + pgsql_thread___monitor_replication_lag_use_percona_heartbeat = GloPTH->get_variable_string((char*)"monitor_replication_lag_use_percona_heartbeat"); + */ if (pgsql_thread___monitor_username) free(pgsql_thread___monitor_username); pgsql_thread___monitor_username = GloPTH->get_variable_string((char*)"monitor_username"); if (pgsql_thread___monitor_password) free(pgsql_thread___monitor_password); @@ -4041,7 +4051,6 @@ void PgSQL_Thread::refresh_variables() { mysql_thread___monitor_replication_lag_group_by_host = (bool)GloPTH->get_variable_int((char*)"monitor_replication_lag_group_by_host"); mysql_thread___monitor_replication_lag_interval = GloPTH->get_variable_int((char*)"monitor_replication_lag_interval"); mysql_thread___monitor_replication_lag_timeout = GloPTH->get_variable_int((char*)"monitor_replication_lag_timeout"); - mysql_thread___monitor_replication_lag_count = GloPTH->get_variable_int((char*)"monitor_replication_lag_count"); mysql_thread___monitor_groupreplication_healthcheck_interval = GloPTH->get_variable_int((char*)"monitor_groupreplication_healthcheck_interval"); mysql_thread___monitor_groupreplication_healthcheck_timeout = GloPTH->get_variable_int((char*)"monitor_groupreplication_healthcheck_timeout"); mysql_thread___monitor_groupreplication_healthcheck_max_timeout_count = GloPTH->get_variable_int((char*)"monitor_groupreplication_healthcheck_max_timeout_count"); @@ -4677,6 +4686,18 @@ SQLite3_result* PgSQL_Threads_Handler::SQL3_GlobalStatus(bool _memory) { pta[1] = buf; result->add_row(pta); } + { + pta[0] = (char*)"PgSQL_Monitor_replication_lag_check_OK"; + sprintf(buf, "%lu", GloPgMon->repl_lag_check_OK); + pta[1] = buf; + result->add_row(pta); + } + { + pta[0] = (char*)"PgSQL_Monitor_replication_lag_check_ERR"; + sprintf(buf, "%lu", GloPgMon->repl_lag_check_ERR); + pta[1] = buf; + result->add_row(pta); + } { pta[0] = (char*)"PgSQL_Monitor_ssl_connections_OK"; sprintf(buf, "%lu", GloPgMon->ssl_connections_OK); diff --git a/test/deps/Makefile b/test/deps/Makefile index 5d39e09c3a..6ab8a22e84 100644 --- a/test/deps/Makefile +++ b/test/deps/Makefile @@ -6,7 +6,7 @@ DEPS_PATH := $(PROXYSQL_PATH)/deps include $(PROXYSQL_PATH)/include/makefiles_vars.mk include $(PROXYSQL_PATH)/include/makefiles_paths.mk -CMAKE3 ?= cmake3 +CMAKE3 := $(shell command -v cmake3 2>/dev/null || command -v cmake 2>/dev/null) .DEFAULT: default .PHONY: default diff --git a/test/tap/groups/groups.json b/test/tap/groups/groups.json index 7b75d34dd9..891d43649e 100644 --- a/test/tap/groups/groups.json +++ b/test/tap/groups/groups.json @@ -299,6 +299,8 @@ "test_mcp_claude_headless_flow-t": [ "ai-g1" ], "test_mcp_llm_discovery_phaseb-t": [ "ai-g1" ], "test_mcp_static_harvest-t": [ "ai-g1" ], + "unit-strip_schema_from_query-t": [ "unit-tests-g1" ], + "test_pgsql_replication_lag-t": [ "pgsql-repl" ], "mcp_query_rules-t": [ "ai-g1" ], "unit-strip_schema_from_query-t": [ "unit-tests-g1" ] diff --git a/test/tap/groups/pgsql-repl/Dockerfile b/test/tap/groups/pgsql-repl/Dockerfile new file mode 100644 index 0000000000..935358e244 --- /dev/null +++ b/test/tap/groups/pgsql-repl/Dockerfile @@ -0,0 +1,3 @@ +FROM postgres:17 + +RUN apt-get update && apt-get install -y iproute2 diff --git a/test/tap/groups/pgsql-repl/README.md b/test/tap/groups/pgsql-repl/README.md new file mode 100644 index 0000000000..4735d1ca3a --- /dev/null +++ b/test/tap/groups/pgsql-repl/README.md @@ -0,0 +1,29 @@ +# PostgreSQL Primary-Replica Infra + +## Images build + +```bash +docker build -t postgres-tc:17 . +``` + +This step is automatically performed by `docker-compose-init.bash`. Images should be reused between +executions. + +## Start / Stop + +To start the infra just execute the group startup script: + +```bash +pre-proxysql.bash +``` + +To stop the infra just execute the group shutdown script: + +```bash +post-proxysql.bash +``` + +## Folder structure + +* `conf`: Config files for both infra and `ProxySQL`. +* `scripts`: Collection of scripts used to prepare the infra. diff --git a/test/tap/groups/pgsql-repl/bin/docker-pgsql-post.bash b/test/tap/groups/pgsql-repl/bin/docker-pgsql-post.bash new file mode 100755 index 0000000000..568d03a39e --- /dev/null +++ b/test/tap/groups/pgsql-repl/bin/docker-pgsql-post.bash @@ -0,0 +1,33 @@ +#!/bin/bash + +set -e + +POSTGRE_SETUP_DIR=$(dirname "$(realpath "$0")")/../scripts/ + +WAITED=0 +TIMEOUT=300 +RC=1 + +set +e + +printf "[$(date)] Waiting for PostgreSQL service to initialize" +while [ $RC -ne 0 ]; do + if [ $WAITED -gt $TIMEOUT ]; then + echo "[ERROR] Timeout of $TIMEOUT seconds reached while connecting to PostgreSQL" + exit 1 + else + printf "." + PGPASSWORD=$PGSQL_PWD ON_ERROR_STOP=1 psql -h$PGSQL_HOST -p$PGSQL_PORT -U$PGSQL_DB -c"SELECT" > /dev/null 2>&1 + RC=$? + WAITED=$((WAITED+1)) + sleep 1 + fi +done +printf "\n" + +set -e + +echo "[$(date)] Creating table structures for testing ..." +set -x +PGPASSWORD=$PGSQL_PWD ON_ERROR_STOP=1 psql -h$PGSQL_HOST -p$PGSQL_PORT -U$PGSQL_DB < "$POSTGRE_SETUP_DIR"/create_test_tables.sql +set +x diff --git a/test/tap/groups/pgsql-repl/bin/docker-proxy-post.bash b/test/tap/groups/pgsql-repl/bin/docker-proxy-post.bash new file mode 100755 index 0000000000..f9c88ab6af --- /dev/null +++ b/test/tap/groups/pgsql-repl/bin/docker-proxy-post.bash @@ -0,0 +1,33 @@ +#!/bin/bash + +set -e + +PROXY_CONF_DIR=$(dirname "$(realpath "$0")")/../conf/proxysql + +WAITED=0 +TIMEOUT=300 +RC=1 + +set +e + +printf "[$(date)] Waiting for ProxySQL service to initialize" +while [ $RC -eq 1 ]; do + if [ $WAITED -gt $TIMEOUT ]; then + echo "[ERROR] Timeout of $TIMEOUT seconds reached while connecting to ProxySQL" + exit 1 + else + printf "." + mysql -h$ADMIN_HOST -P$ADMIN_PORT -u$ADMIN_USER -p$ADMIN_PWD -e"\s" > /dev/null 2>&1 + RC=$? + WAITED=$((WAITED+1)) + sleep 1 + fi +done +printf "\n" + +set -e + +echo "[$(date)] Applying initial config for ProxySQL ..." +set -x +mysql --prompt="admin> " -u$ADMIN_USER -p$ADMIN_PWD --table -h$ADMIN_HOST -P$ADMIN_PORT < "$PROXY_CONF_DIR"/config.sql +set +x diff --git a/test/tap/groups/pgsql-repl/conf/postgres/00_init.sql b/test/tap/groups/pgsql-repl/conf/postgres/00_init.sql new file mode 100644 index 0000000000..1ae32a5593 --- /dev/null +++ b/test/tap/groups/pgsql-repl/conf/postgres/00_init.sql @@ -0,0 +1,9 @@ +CREATE USER repluser WITH replication encrypted password 'replpass'; +SELECT pg_create_physical_replication_slot('replication_slot'); + +CREATE USER proxymon WITH encrypted password 'proxymon'; + +GRANT pg_monitor TO proxymon; + +-- For testing 'pgsql-monitor_dbname' +CREATE DATABASE proxymondb; diff --git a/test/tap/groups/pgsql-repl/conf/proxysql/config.sql b/test/tap/groups/pgsql-repl/conf/proxysql/config.sql new file mode 100644 index 0000000000..5c4fe1a0f2 --- /dev/null +++ b/test/tap/groups/pgsql-repl/conf/proxysql/config.sql @@ -0,0 +1,36 @@ +SET pgsql-monitor_password='proxymon'; +SET pgsql-monitor_username='proxymon'; + +LOAD PGSQL VARIABLES TO RUNTIME; +SAVE PGSQL VARIABLES TO DISK; + +DELETE FROM pgsql_users; +INSERT INTO pgsql_users (username,password,default_hostgroup) VALUES ('postgres','postgres',0); + +LOAD PGSQL USERS TO RUNTIME; +SAVE PGSQL USERS TO DISK; + +SET pgsql-monitor_replication_lag_interval=1000; + +LOAD PGSQL VARIABLES TO RUNTIME; +SAVE PGSQL VARIABLES TO DISK; + +DELETE FROM pgsql_replication_hostgroups; +INSERT INTO + pgsql_replication_hostgroups (writer_hostgroup, reader_hostgroup, check_type, comment) +VALUES + (0, 1, 'read_only', 'pg-replication'); + +LOAD PGSQL SERVERS TO RUNTIME; +SAVE PGSQL SERVERS TO DISK; + +DELETE FROM pgsql_servers; + +INSERT INTO + pgsql_servers (hostgroup_id, hostname, port, max_replication_lag, comment) +VALUES + (0, '127.0.0.1', 15432, 3, 'pg-primary'), + (1, '127.0.0.1', 15433, 3, 'pg-replica'); + +LOAD PGSQL SERVERS TO RUNTIME; +SAVE PGSQL SERVERS TO DISK; diff --git a/test/tap/groups/pgsql-repl/constants b/test/tap/groups/pgsql-repl/constants new file mode 100644 index 0000000000..caf5afe2cc --- /dev/null +++ b/test/tap/groups/pgsql-repl/constants @@ -0,0 +1,12 @@ +#!/bin/bash + +export PGSQL_USER=${TAP_PGSQLROOT_USERNAME:=postgres} +export PGSQL_PWD=${TAP_PGSQLROOT_PASSWORD:=postgres} +export PGSQL_DB=${TAP_PGSQLROOT_DATABASE:=postgres} +export PGSQL_HOST=${TAP_PGSQLSERVER_HOST:=127.0.0.1} +export PGSQL_PORT=${TAP_PGSQLSERVER_PORT:=15432} + +export ADMIN_HOST=${TAP_ADMINHOST:=127.0.0.1} +export ADMIN_PORT=${TAP_ADMINPORT:=6032} +export ADMIN_USER=${TAP_ADMINUSERNAME:=radmin} +export ADMIN_PWD=${TAP_ADMINPASSWORD:=radmin} diff --git a/test/tap/groups/pgsql-repl/docker-compose-destroy.bash b/test/tap/groups/pgsql-repl/docker-compose-destroy.bash new file mode 100755 index 0000000000..0ec548cf87 --- /dev/null +++ b/test/tap/groups/pgsql-repl/docker-compose-destroy.bash @@ -0,0 +1,3 @@ +#!/bin/bash + +docker compose down -v diff --git a/test/tap/groups/pgsql-repl/docker-compose-init.bash b/test/tap/groups/pgsql-repl/docker-compose-init.bash new file mode 100755 index 0000000000..d509b2e2f0 --- /dev/null +++ b/test/tap/groups/pgsql-repl/docker-compose-init.bash @@ -0,0 +1,10 @@ +#!/bin/bash + +set -e + +. constants + +docker build -t postgres-tc:17 . +docker compose up -d + +./bin/docker-pgsql-post.bash && ./bin/docker-proxy-post.bash diff --git a/test/tap/groups/pgsql-repl/docker-compose.yaml b/test/tap/groups/pgsql-repl/docker-compose.yaml new file mode 100644 index 0000000000..988a1a53cc --- /dev/null +++ b/test/tap/groups/pgsql-repl/docker-compose.yaml @@ -0,0 +1,70 @@ +x-pg-common: + &pg-common + image: postgres-tc:17 + user: postgres + restart: always + healthcheck: + test: 'pg_isready -U postgres --dbname=postgres' + interval: 10s + timeout: 5s + retries: 5 + +services: + pg_primary: + <<: *pg-common + ports: + - 15432:5432 + environment: + POSTGRES_USER: postgres + POSTGRES_DB: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_HOST_AUTH_METHOD: "scram-sha-256\nhost replication all 0.0.0.0/0 md5" + POSTGRES_INITDB_ARGS: "--auth-host=scram-sha-256" + networks: + default: {} + pg_backend: + aliases: + - pg_primary_int + cap_add: + - NET_ADMIN + command: | + postgres + -c wal_level=replica + -c hot_standby=on + -c max_wal_senders=10 + -c max_connections=500 + -c max_replication_slots=10 + -c hot_standby_feedback=on + volumes: + - ./conf/postgres/00_init.sql:/docker-entrypoint-initdb.d/00_init.sql + + pg_replica: + <<: *pg-common + ports: + - 15433:5432 + environment: + PGUSER: repluser + PGPASSWORD: replpass + networks: + - default + - pg_backend + cap_add: + - NET_ADMIN + command: | + bash -c " + until pg_basebackup --pgdata=/var/lib/postgresql/data -R --slot=replication_slot --host=pg_primary_int --port=5432 + do + echo 'Waiting for primary to connect...' + sleep 1s + done + echo 'Backup done, starting replica...' + chmod 0700 /var/lib/postgresql/data + postgres -c max_connections=500 + " + depends_on: + - pg_primary + +networks: + pg_backend: + driver: bridge + internal: true diff --git a/test/tap/groups/pgsql-repl/post-proxysql.bash b/test/tap/groups/pgsql-repl/post-proxysql.bash new file mode 100755 index 0000000000..6b78b7daf1 --- /dev/null +++ b/test/tap/groups/pgsql-repl/post-proxysql.bash @@ -0,0 +1,8 @@ +#!/bin/bash + +set -e + +. constants + +echo "[$(date)] Shutting down PGSQL_REPL testing infra ..." +./docker-compose-destroy.bash diff --git a/test/tap/groups/pgsql-repl/pre-proxysql.bash b/test/tap/groups/pgsql-repl/pre-proxysql.bash new file mode 100755 index 0000000000..e8ab606ff7 --- /dev/null +++ b/test/tap/groups/pgsql-repl/pre-proxysql.bash @@ -0,0 +1,11 @@ +#!/bin/bash + +set -e + +. constants + +echo "[$(date)] Cleaning infra prior to PGSQL_REPL group testing" +./docker-compose-destroy.bash + +echo "[$(date)] Starting infra required for PGSQL_REPL group testing" +./docker-compose-init.bash diff --git a/test/tap/groups/pgsql-repl/scripts/create_test_tables.sql b/test/tap/groups/pgsql-repl/scripts/create_test_tables.sql new file mode 100644 index 0000000000..1e52bfe4db --- /dev/null +++ b/test/tap/groups/pgsql-repl/scripts/create_test_tables.sql @@ -0,0 +1,13 @@ +\set ON_ERROR_STOP on + +DROP DATABASE IF EXISTS sysbench; +CREATE DATABASE sysbench; + +\connect sysbench; + +CREATE TABLE sbtest1 ( + id INTEGER PRIMARY KEY GENERATED ALWAYS AS IDENTITY, + k integer DEFAULT 0 NOT NULL, + c character(120) DEFAULT ''::bpchar NOT NULL, + pad character(60) DEFAULT ''::bpchar NOT NULL +); diff --git a/test/tap/proxysql-ca.pem b/test/tap/proxysql-ca.pem deleted file mode 100644 index 256a3158d4..0000000000 --- a/test/tap/proxysql-ca.pem +++ /dev/null @@ -1,18 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIC8zCCAdugAwIBAgIEaWQj8TANBgkqhkiG9w0BAQsFADAxMS8wLQYDVQQDDCZQ -cm94eVNRTF9BdXRvX0dlbmVyYXRlZF9DQV9DZXJ0aWZpY2F0ZTAeFw0yNjAxMTEy -MjI4MDFaFw0zNjAxMDkyMjI4MDFaMDExLzAtBgNVBAMMJlByb3h5U1FMX0F1dG9f -R2VuZXJhdGVkX0NBX0NlcnRpZmljYXRlMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A -MIIBCgKCAQEAm+yYXZdv9Q1ifx7QRxR7icJMyOqnEIcFTT4zpStJx586mKrtNLbl -dWf8wpxVLoEbmwTcfrKTL7ys7QZEQiX1JVEYkCWjlhy90uo2czOhag91WgBdJe9D -9x9wGLUscgxj8bxQU0tT0ZjRVcvGMf45frFw26f2PPaHJ5eCyU1hRx9PGp6XUct8 -xDWPUrUU4ilxdsgxIjNLGKrXT3HgmaiePEn+wn0ASKkaiSrtE5VwYkmCnbv3qBQ8 -/hT2K1W81zfpvQIa6gMEOs3FExfhuEIGWs7PcipT7XSK6n+fZY40jdN3NVRLQvfE -8z+mHXEqDM+SNTZuG2W7QegSaEZncaXVUQIDAQABoxMwETAPBgNVHRMBAf8EBTAD -AQH/MA0GCSqGSIb3DQEBCwUAA4IBAQAmP+o3MGKoNpjnxW1tkjcUZaDuAjPVBJoX -EzjVahV0Hnb9ALptIeGXkpTP9LcvOgOMFMWNRFdQTyUfgiajCBVOjc0LgkbWfpiS -UV9QEbtN9uXdzxMO0ZvAAbZsB+TAfRo6zQeU++vWVochnn/J4J0ax641Gq1tSH2M -If4KUhTLP1fZoGKllm2pr/YJr56e+nsy3gVmolR9o5P+2aYfDd0TPy8tgH+uPHTZ -o1asy6oB/8a47nQVUU82ljJgoe1iVYwYRchLjYQLCJCoYN6AMPxpPxQVME4AgBrx -OHyDVPBvWU/NgN3banbrlRTJtCtp3spoKO8oGtAvPqGV0h1860mw ------END CERTIFICATE----- diff --git a/test/tap/proxysql-cert.pem b/test/tap/proxysql-cert.pem deleted file mode 100644 index 0aff3a8fff..0000000000 --- a/test/tap/proxysql-cert.pem +++ /dev/null @@ -1,18 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIC9DCCAdygAwIBAgIEaWQj8TANBgkqhkiG9w0BAQsFADAxMS8wLQYDVQQDDCZQ -cm94eVNRTF9BdXRvX0dlbmVyYXRlZF9DQV9DZXJ0aWZpY2F0ZTAeFw0yNjAxMTEy -MjI4MDFaFw0zNjAxMDkyMjI4MDFaMDUxMzAxBgNVBAMMKlByb3h5U1FMX0F1dG9f -R2VuZXJhdGVkX1NlcnZlcl9DZXJ0aWZpY2F0ZTCCASIwDQYJKoZIhvcNAQEBBQAD -ggEPADCCAQoCggEBAJvsmF2Xb/UNYn8e0EcUe4nCTMjqpxCHBU0+M6UrScefOpiq -7TS25XVn/MKcVS6BG5sE3H6yky+8rO0GREIl9SVRGJAlo5YcvdLqNnMzoWoPdVoA -XSXvQ/cfcBi1LHIMY/G8UFNLU9GY0VXLxjH+OX6xcNun9jz2hyeXgslNYUcfTxqe -l1HLfMQ1j1K1FOIpcXbIMSIzSxiq109x4JmonjxJ/sJ9AEipGokq7ROVcGJJgp27 -96gUPP4U9itVvNc36b0CGuoDBDrNxRMX4bhCBlrOz3IqU+10iup/n2WONI3TdzVU -S0L3xPM/ph1xKgzPkjU2bhtlu0HoEmhGZ3Gl1VECAwEAAaMQMA4wDAYDVR0TAQH/ -BAIwADANBgkqhkiG9w0BAQsFAAOCAQEAL2fQnE9vUK7/t6tECL7LMSs2Y5pBUZsA -sCQigyU7CQ9e6GTG5lPonWVX4pOfriDEWOkAuWlgRSxZpbvPJBpqN1CpR1tFBpMn -2H7gXZGkx+O2fvVvBMPFxusZZRoFfKWwO7Vr+YU3q8pai4ra3lFMMzzrIKku65pt -Vv2U4Sb4RsdXYDsjiAUSsPNqJsQTvum5QTEzqMSUSrKEvpOtVVvGr7KULZt4md/C -GQcuZujr2VTiclDhAP7rvMhmWE8FhGCcBce+k3/PMq9ui+NsMLGmWvp4BUmr8mD3 -xTwclMHIahUrxFEgp/AA+NspGCFm48xyvSpmfttAW83JYDs7R5fJEQ== ------END CERTIFICATE----- diff --git a/test/tap/proxysql-key.pem b/test/tap/proxysql-key.pem deleted file mode 100644 index c5c9eed8a6..0000000000 --- a/test/tap/proxysql-key.pem +++ /dev/null @@ -1,27 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEogIBAAKCAQEAm+yYXZdv9Q1ifx7QRxR7icJMyOqnEIcFTT4zpStJx586mKrt -NLbldWf8wpxVLoEbmwTcfrKTL7ys7QZEQiX1JVEYkCWjlhy90uo2czOhag91WgBd -Je9D9x9wGLUscgxj8bxQU0tT0ZjRVcvGMf45frFw26f2PPaHJ5eCyU1hRx9PGp6X -Uct8xDWPUrUU4ilxdsgxIjNLGKrXT3HgmaiePEn+wn0ASKkaiSrtE5VwYkmCnbv3 -qBQ8/hT2K1W81zfpvQIa6gMEOs3FExfhuEIGWs7PcipT7XSK6n+fZY40jdN3NVRL -QvfE8z+mHXEqDM+SNTZuG2W7QegSaEZncaXVUQIDAQABAoIBABbreNwtEgp5/LQF -8gS4yI4P7xyLjaI6zrczgQDy84Xx7HmbioG4rtMKxZdPxp+u38FyPf0rv8IBIIQ4 -6xi0HqxtFsi9l6XNtMOHpRhbCwudmRjxO8ADQ0DUsLQZEZ70Hk7e6QnNZVVGeuL7 -MLeRkJ8Eczv+nQ4KCQTzWwi/JKEBCOoYtPDwkecydbxMsOVM5204rXwmQxW9l2Sr -uGrtfWp5C+xW041spRGskV/7jNhNNKethO1obQlBN6LJKD48p8uEvH+FuHWndm/E -F5GgttSLOemeJrjpXjE4RCdRCT/ZSyE120mxv7YgctMGC1ouFWolgc4hGzJURBtu -H/8KbXcCgYEAzjEp8b9I4QUCopc+bYO5FAVN+I5e/uvVFbgu1QLhknK488DIj2XH -uKj52lGMOkdtgdEQdpk/9fYd0kwn2k7U8/6mb5kQqtuzSll6UCC+OwaCbke3DPp1 -JXmGapUYVIZ8TIxnVaZcKSWv3VqjuwV2GQqOcaSSbAt3BQ5whIzn4F8CgYEAwZbj -IHx0GmrvxjF0JpC1duk65zMKWyLddYeAIuq9hgB7jCVOqmmDElTcZOWKboMUvVg7 -SvteIZjQLB93ktqHf40n1hfmYMaSNLJYxe/JMXWYEDL9++qBPz0rLpScZGxOmNyj -jIl8pwilATs2ZAjQEfy5qL1GeOHe/X6N896vaE8CgYBNNfHL+eIziOnEsrgI0GOU -0Kuy4LVH5k3DtVWsJEkNyvHhLRatQ+K3DmeJTjIhfK/QBdaRYq+lzgS6xBPEVvK9 -b2Upsvqf0Gdh9wGrUaeKeNSMsUQlkwAdCVXBQZV7yWRwUb88PnCSY+9oB1H6bYAc -vmw6t/KwjNaDyTVvHUiTJwKBgHZ2hvZSMhoYZjG6AYG3+9OQVWM1cJjkdPB+woKb -cu6VTQUtrz3I41RMabG0ZUnLHN3hKCdyOuAESx81Ak7zOwdqsX3pkiiWWtG0cW5u -lYeWlj8TdSi7D+xK2ine9vTc8hvIqKxPVeBBAfgG6/m7Cth29oWzjXRbg8FLuEIL -evsxAoGASKbnZznS0tI8mLBrnZWISlpbdiXwHcIOcuF06rEVHTFHd+Ab5eRCFwY9 -idQnAEUUUK8FTHvj5pdPNYv3s9koRF2FHgBilF4k3ESMR2yoPuUQHQ0M7uySy2+c -u7owHRtq0phoywgtZnbKpg1h0kafTkYdRG3eF3I8pBy7jDGrG4k= ------END RSA PRIVATE KEY----- diff --git a/test/tap/tap/Makefile b/test/tap/tap/Makefile index ed0344bbdb..a6126978ac 100644 --- a/test/tap/tap/Makefile +++ b/test/tap/tap/Makefile @@ -106,7 +106,7 @@ cpp-dotenv/static/cpp-dotenv/libcpp_dotenv.a: cd cpp-dotenv/static/cpp-dotenv && patch src/dotenv.cpp < ../../dotenv.cpp.patch cd cpp-dotenv/static/cpp-dotenv && patch include/dotenv.h < ../../dotenv.h.patch cd cpp-dotenv/static/cpp-dotenv && patch -p0 < ../../nm_clang_fix.patch - cd cpp-dotenv/static/cpp-dotenv && cmake . -DBUILD_TESTING=OFF -DBUILD_SHARED_LIBS=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_BUILD_TYPE=Debug + cd cpp-dotenv/static/cpp-dotenv && cmake . -DBUILD_TESTING=OFF -DBUILD_SHARED_LIBS=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_BUILD_TYPE=Debug -DCMAKE_POLICY_VERSION_MINIMUM=3.5 cd cpp-dotenv/static/cpp-dotenv && CC=${CC} CXX=${CXX} ${MAKE} cpp-dotenv/dynamic/cpp-dotenv/libcpp_dotenv.so: @@ -115,7 +115,7 @@ cpp-dotenv/dynamic/cpp-dotenv/libcpp_dotenv.so: cd cpp-dotenv/dynamic/cpp-dotenv && patch src/dotenv.cpp < ../../dotenv.cpp.patch cd cpp-dotenv/dynamic/cpp-dotenv && patch include/dotenv.h < ../../dotenv.h.patch cd cpp-dotenv/dynamic/cpp-dotenv && patch -p0 < ../../nm_clang_fix.patch - cd cpp-dotenv/dynamic/cpp-dotenv && cmake . -DBUILD_TESTING=OFF -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_RPATH="../tap:../../tap" -DCMAKE_BUILD_TYPE=Debug + cd cpp-dotenv/dynamic/cpp-dotenv && cmake . -DBUILD_TESTING=OFF -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_RPATH="../tap:../../tap" -DCMAKE_BUILD_TYPE=Debug -DCMAKE_POLICY_VERSION_MINIMUM=3.5 cd cpp-dotenv/dynamic/cpp-dotenv && CC=${CC} CXX=${CXX} ${MAKE} diff --git a/test/tap/tap/utils.h b/test/tap/tap/utils.h index 3d066f0a1e..75e7660590 100644 --- a/test/tap/tap/utils.h +++ b/test/tap/tap/utils.h @@ -261,10 +261,23 @@ sq3_res_t sqlite3_execute_stmt(sqlite3* db, const std::string& query); * @param val The 'ext_val_t' to be checked. * @return In case of failure, 'EXIT_FAILURE' after logging the error, continues otherwise. */ -#define CHECK_EXT_VAL(val)\ +#define CHECK_EXT_VAL(conn, val)\ do {\ if (val.err) {\ - diag("%s:%d: Query failed err=\"%s\"", __func__, __LINE__, val.str.c_str());\ + diag("%s:%d: Query failed err=\"%s\"", __func__, __LINE__, get_ext_val_err(conn, val).c_str());\ + return EXIT_FAILURE;\ + }\ + } while(0) + +/** + * @brief Utility one-liner macro to check for query failure on a 'ext_val_t'. + * @param val The 'ext_val_t' to be checked. + * @return In case of failure, 'EXIT_FAILURE' after logging the error, continues otherwise. + */ +#define SQ3_CHECK_EXT_VAL(val)\ + do {\ + if (val.err) {\ + diag("%s:%d: Query failed err=\"%s\"", __func__, __LINE__, sq3_get_ext_val_err(val).c_str());\ return EXIT_FAILURE;\ }\ } while(0) diff --git a/test/tap/tests/test_match_eof_conn_cap.cpp b/test/tap/tests/test_match_eof_conn_cap.cpp index 220733f3bd..791e74fecd 100644 --- a/test/tap/tests/test_match_eof_conn_cap.cpp +++ b/test/tap/tests/test_match_eof_conn_cap.cpp @@ -57,14 +57,6 @@ using std::vector; #define _S(s) ( std::string {s} ) #define _TO_S(s) ( std::to_string(s) ) -#define CHECK_EXT_VAL(val)\ - do {\ - if (val.err) {\ - diag("%s:%d: Query failed err=\"%s\"", __func__, __LINE__, val.str.c_str());\ - return EXIT_FAILURE;\ - }\ - } while(0) - MYSQL* create_mysql_conn(const conn_opts_t& opts) { const char* host { opts.host.c_str() }; const char* user { opts.user.c_str() }; @@ -399,11 +391,11 @@ int test_conn_acquisition(MYSQL* admin, const test_cnf_t& test_conf) { const ext_val_t retries_delay { mysql_query_ext_val(admin, SELECT_RUNTIME_VAR"'mysql-connect_retries_delay'", -1) }; - CHECK_EXT_VAL(retries_delay); + CHECK_EXT_VAL(admin, retries_delay); const ext_val_t to_server_max { mysql_query_ext_val(admin, SELECT_RUNTIME_VAR"'mysql-connect_timeout_server_max'", -1) }; - CHECK_EXT_VAL(to_server_max); + CHECK_EXT_VAL(admin, to_server_max); /////////////////////////////////////////////////////////////////////////// diag( @@ -433,12 +425,12 @@ int test_conn_acquisition(MYSQL* admin, const test_cnf_t& test_conf) { diag("Get pre-conn attempt stats from target hostgroup tg=%d", SQLITE3_HG); const ext_val_t pre_hg_st { get_conn_pool_hg_stats(admin, SQLITE3_HG) }; - CHECK_EXT_VAL(pre_hg_st); + CHECK_EXT_VAL(admin, pre_hg_st); const ext_val_t pre_srv_conns { mysql_query_ext_val(admin, "SELECT variable_value FROM stats.stats_mysql_global WHERE variable_name='Server_Connections_created'", int64_t(-1) )}; - CHECK_EXT_VAL(pre_srv_conns); + CHECK_EXT_VAL(admin, pre_srv_conns); const string audit_regex { "SQLite3_Connect_OK.*" + _S(FF_USER) }; // Audit logs are per-thread buffered. PROXYSQL FLUSH LOGS ensures buffers are written @@ -586,7 +578,7 @@ int test_conn_acquisition(MYSQL* admin, const test_cnf_t& test_conf) { diag("Get post-conn attempt stats from target hostgroup tg=%d", SQLITE3_HG); const ext_val_t post_hg_st { get_conn_pool_hg_stats(admin, SQLITE3_HG) }; - CHECK_EXT_VAL(post_hg_st); + CHECK_EXT_VAL(admin, post_hg_st); ok( pre_hg_st.val.conn_ok + exp_conns == post_hg_st.val.conn_ok, @@ -602,7 +594,7 @@ int test_conn_acquisition(MYSQL* admin, const test_cnf_t& test_conf) { int64_t(-1) ) }; - CHECK_EXT_VAL(post_srv_conns); + CHECK_EXT_VAL(admin, post_srv_conns); ok( pre_srv_conns.val + exp_conns == post_srv_conns.val, @@ -631,11 +623,11 @@ int test_conn_acquisition( const ext_val_t retries_delay { mysql_query_ext_val(admin, SELECT_RUNTIME_VAR"'mysql-connect_retries_delay'", -1) }; - CHECK_EXT_VAL(retries_delay); + CHECK_EXT_VAL(admin, retries_delay); const ext_val_t to_server_max { mysql_query_ext_val(admin, SELECT_RUNTIME_VAR"'mysql-connect_timeout_server_max'", -1) }; - CHECK_EXT_VAL(to_server_max); + CHECK_EXT_VAL(admin, to_server_max); ////// diag("Update 'fast-forward' for testing user user=\"%s\" fast_forward=%d", FF_USER, ff); @@ -659,12 +651,12 @@ int test_conn_acquisition( diag("Get pre-conn attempt stats from target hostgroup tg=%d", SQLITE3_HG); const ext_val_t pre_hg_st { get_conn_pool_hg_stats(admin, SQLITE3_HG) }; - CHECK_EXT_VAL(pre_hg_st); + CHECK_EXT_VAL(admin, pre_hg_st); const ext_val_t pre_srv_conns { mysql_query_ext_val(admin, "SELECT variable_value FROM stats.stats_mysql_global WHERE variable_name='Server_Connections_created'", int64_t(-1) )}; - CHECK_EXT_VAL(pre_srv_conns); + CHECK_EXT_VAL(admin, pre_srv_conns); const string audit_regex { "SQLite3_Connect_OK.*" + _S(FF_USER) }; // ProxySQL audit logs are per-thread buffered. PROXYSQL FLUSH LOGS ensures buffers @@ -736,7 +728,7 @@ int test_conn_acquisition( diag("Get post-conn attempt stats from target hostgroup tg=%d", SQLITE3_HG); const ext_val_t post_hg_st { get_conn_pool_hg_stats(admin, SQLITE3_HG) }; - CHECK_EXT_VAL(post_hg_st); + CHECK_EXT_VAL(admin, post_hg_st); ok( pre_hg_st.val.conn_ok + exp_conns == post_hg_st.val.conn_ok, @@ -752,7 +744,7 @@ int test_conn_acquisition( int64_t(-1) ) }; - CHECK_EXT_VAL(post_srv_conns); + CHECK_EXT_VAL(admin, post_srv_conns); ok( pre_srv_conns.val + exp_conns == post_srv_conns.val, @@ -885,12 +877,12 @@ int test_conn_ff_conv(MYSQL* admin, const CommandLine& cl, bool client_eof) { diag("Get pre-conn attempt stats from target hostgroup tg=%d", SQLITE3_HG); const ext_val_t pre_hg_st { get_conn_pool_hg_stats(admin, SQLITE3_HG) }; - CHECK_EXT_VAL(pre_hg_st); + CHECK_EXT_VAL(admin, pre_hg_st); const ext_val_t pre_srv_conns { mysql_query_ext_val(admin, "SELECT variable_value FROM stats.stats_mysql_global WHERE variable_name='Server_Connections_created'", int64_t(-1) )}; - CHECK_EXT_VAL(pre_srv_conns); + CHECK_EXT_VAL(admin, pre_srv_conns); diag("Issue query (start trx) to create new backend conn query=\"%s\"", "BEGIN"); rc = mysql_query_t(proxy, "/* hostgroup=" + std::to_string(SQLITE3_HG) + " */ BEGIN"); @@ -901,7 +893,7 @@ int test_conn_ff_conv(MYSQL* admin, const CommandLine& cl, bool client_eof) { diag("Get post-conn attempt stats from target hostgroup tg=%d", SQLITE3_HG); const ext_val_t post_hg_st { get_conn_pool_hg_stats(admin, SQLITE3_HG) }; - CHECK_EXT_VAL(post_hg_st); + CHECK_EXT_VAL(admin, post_hg_st); ok( pre_hg_st.val.conn_used + 1 == post_hg_st.val.conn_used, diff --git a/test/tap/tests/test_pgsql_replication_lag-t.cpp b/test/tap/tests/test_pgsql_replication_lag-t.cpp new file mode 100644 index 0000000000..308e7bdfc5 --- /dev/null +++ b/test/tap/tests/test_pgsql_replication_lag-t.cpp @@ -0,0 +1,339 @@ +/** + * @file test_pgsql_replication_lag-t.cpp + * @brief POC: Test for PostgreSQL replication lag monitoring and shunning. + * @details This is a test POC for new infra for testing PostgreSQL monitoring and SHUNNING. Correctness can + * be improved in multiple points, but for now, this provides coverage and automated testing for the feature. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "libpq-fe.h" +#include "mysql.h" + +#include "tap.h" +#include "command_line.h" +#include "utils.h" + +using std::string; +using std::vector; + +#define _TO_S(s) ( std::to_string(s) ) + +// PostgreSQL query execution with logging - similar to MYSQL_QUERY_T +#define PG_QUERY_T(conn, query) \ + do { \ + diag("%s:%d: Executing query: %s", __func__, __LINE__, query); \ + PGresult* res = PQexec(conn, query); \ + if (PQresultStatus(res) != PGRES_COMMAND_OK && PQresultStatus(res) != PGRES_TUPLES_OK) { \ + fprintf(stderr, "File %s, line %d, Error: %s\n", \ + __FILE__, __LINE__, PQerrorMessage(conn)); \ + PQclear(res); \ + return EXIT_FAILURE; \ + } \ + PQclear(res); \ + } while(0) + +PGconn* create_new_conn(const CommandLine& cl, bool with_ssl) { + std::stringstream ss; + + ss << "host=" << cl.pgsql_host << " port=" << cl.pgsql_port; + ss << " user=" << cl.pgsql_username << " password=" << cl.pgsql_password; + ss << " sslmode=disable"; + ss << " dbname=sysbench"; + + diag("Creating PostgreSQL connection params=\"%s\"", ss.str().c_str()); + PGconn* conn { PQconnectdb(ss.str().c_str()) }; + const bool res = (conn && PQstatus(conn) == CONNECTION_OK); + + if (res) { + return conn; + } else { + fprintf(stderr, "File %s, line %d, Error: Connection failed\n", __FILE__, __LINE__); + PQfinish(conn); + return nullptr; + } +} + +const char* REPL_PORT { get_env_str("TAP_REPLICA_PORT", "15433") }; +const string TEST_DATADIR { get_env_str("TAP_WORKDIR", ".") + _S("/test_pgsql_replication_lag") }; +const int32_t MAX_REPL_LAG { 3 }; + +int test_replication_lag( + const CommandLine& cl, MYSQL* admin, PGconn* pgconn, int32_t max_repl_lag, int32_t max_count +) { + diag("Testing PostgreSQL replication lag check max_count=%d", max_count); + + MYSQL_QUERY_T(admin, ("SET pgsql-monitor_replication_lag_count=" + _TO_S(max_count)).c_str()); + MYSQL_QUERY_T(admin, "LOAD PGSQL VARIABLES TO RUNTIME"); + + // Get ProxySQL log path + const string PROXYSQL_LOG_PATH { + get_env_str("REGULAR_INFRA_DATADIR", "/tmp") + _S("/proxysql.log") + }; + + // Get pgsql-monitor_replication_lag_interval + diag("Retrieving pgsql-monitor_replication_lag_interval"); + const ext_val_t monitor_interval { + mysql_query_ext_val(admin, + "SELECT variable_value FROM runtime_global_variables WHERE" + " variable_name='pgsql-monitor_replication_lag_interval'", + int32_t(-1)) + }; + CHECK_EXT_VAL(admin, monitor_interval); + diag("pgsql-monitor_replication_lag_interval = %d ms", monitor_interval.val); + + // Wait for servers to be ONLINE (in case of previous execution) + diag("Waiting until target replica is ONLINE (catchup with replication)..."); + { + const string q_st_check { + "SELECT IF(" + "(SELECT status FROM runtime_pgsql_servers WHERE port=" + _S(REPL_PORT) + ")=\"ONLINE\"," + " TRUE," + " FALSE" + ")" + }; + uint32_t timeout = 60; + uint32_t j = 0; + + for (uint32_t i = 0; i < 5; i++) { + diag("Issuing check query query=\"%s\"", q_st_check.c_str()); + const ext_val_t is_online { mysql_query_ext_val(admin, q_st_check, 0) }; + CHECK_EXT_VAL(admin, is_online); + + diag("Check finished with result val=\"%d\"", is_online.val); + if (!is_online.val) { + i = 0; + } + + if (j > timeout) { + break; + } else { + j++; + } + + sleep(1); + } + } + + // Open log file before waiting + diag("Opening ProxySQL log file for SHUNNED detection logfile=\"%s\"", PROXYSQL_LOG_PATH.c_str()); + std::fstream logfile_shunn; + int of_err = open_file_and_seek_end(PROXYSQL_LOG_PATH, logfile_shunn); + if (of_err != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + diag("Opening ProxySQL log file for NOT_SHUNNING detection"); + std::fstream logfile_not_shunn; + of_err = open_file_and_seek_end(PROXYSQL_LOG_PATH, logfile_not_shunn); + if (of_err != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + // Delete and insert data + diag("Inserting test data to trigger replication lag"); + PG_QUERY_T(pgconn, "DELETE FROM sbtest1"); + PG_QUERY_T(pgconn, "INSERT INTO sbtest1 (k, c, pad) VALUES (1, 'x', 'y')"); + PG_QUERY_T(pgconn, + "WITH RECURSIVE gen(n) AS (" + " SELECT 1" + " UNION ALL" + " SELECT n + 1" + " FROM gen" + " WHERE n < 48000" + ")" + "INSERT INTO sbtest1 (k, c, pad)" + "SELECT" + " (random() * 10000)::int," + " lpad(md5(random()::text), 120, 'x')::char(120)," + " lpad(md5(random()::text), 60, 'y')::char(60)" + "FROM gen" + ); + + // Wait for monitoring interval + diag("Waiting for replication lag detection"); + vector repl_lags {}; + + for (int32_t i = 0; i < 20; i++) { + const ext_val_t repl_lag { + mysql_query_ext_val(admin, + "SELECT repl_lag FROM pgsql_server_replication_lag_log " + "WHERE port=" + _S(REPL_PORT) + " ORDER BY time_start_us DESC LIMIT 1", + 0 + ) + }; + CHECK_EXT_VAL(admin, repl_lag); + diag("Fetched current replication lag repl_lag=%d", repl_lag.val); + + if (repl_lag.val > max_repl_lag) { + repl_lags.push_back(repl_lag.val); + } else { + repl_lags.clear(); + } + + if (repl_lags.size() >= max_count && repl_lags.size() > 3) { + diag( + "Consistent replication lag detected; server should be SHUNNED repl_lags=%ld", + repl_lags.size() + ); + break; + } else { + diag( + "Found replication lag; waiting for more entries repl_lags=%ld", + repl_lags.size() + ); + } + sleep(1); + } + + // Check server status via runtime_pgsql_servers + diag("Checking server status in runtime_pgsql_servers"); + vector shunned {}; + for (uint32_t i = 0; i < 6; i++) { + const ext_val_t server_status { + mysql_query_ext_val(admin, + "SELECT status FROM runtime_pgsql_servers WHERE port=" + _S(REPL_PORT), + string("UNKNOWN") + ) + }; + CHECK_EXT_VAL(admin, server_status); + diag("Fetched current runtime_server_status status=\"%s\"", server_status.val.c_str()); + + shunned.push_back(server_status.val == "SHUNNED"); + sleep(1); + } + + const int32_t times { std::reduce(shunned.begin(), shunned.end()) }; + ok(times >= 1, "Server status should have been SHUNNED at least one check times=%d", times); + + // Check logs for shunning message + diag("Checking ProxySQL log for shunning message"); + const string shun_regex { + ".*\\[WARNING\\] Shunning server [^\\s]+:" + _S(REPL_PORT) + " from HG 1 with replication lag of.*" + }; + const string not_shun_regex { + ".*\\[INFO\\] Not shunning server [^\\s]+:" + _S(REPL_PORT) + " from HG 1 with replication lag of.*" + }; + + const auto& [_0, not_shun_lines] = get_matching_lines(logfile_not_shunn, not_shun_regex); + const auto& [_1, shun_lines] = get_matching_lines(logfile_shunn, shun_regex); + + ok( + shun_lines.size() > 0, + "ProxySQL should have SHUNNED the server shunn_count=%ld", + shun_lines.size() + ); + for (const auto& [pos, line, match] : shun_lines) { + diag("Found log line line=`%s`", line.c_str()); + } + + if (max_count > 1) { + ok( + not_shun_lines.size() >= max_count - 1, + "SHUNNING attempts should exceed 'monitor_replication_lag_count' nshunn_count=%ld", + not_shun_lines.size() + ); + for (const auto& [pos, line, match] : not_shun_lines) { + diag("Found log line line=`%s`", line.c_str()); + } + } else { + ok( + not_shun_lines.size() == 0, + "SHUNNING attempts should be only one; no logging required nshunn_count=%ld", + not_shun_lines.size() + ); + } + + // Cleanup + logfile_shunn.close(); + logfile_not_shunn.close(); + + return EXIT_SUCCESS; +} + +int main(int argc, char** argv) { + CommandLine cl; + + if (cl.getEnv()) { + diag("Failed to get the required environmental variables."); + return EXIT_FAILURE; + } + + plan(3 * 2); + + // Connect to ProxySQL admin interface + MYSQL* admin = mysql_init(NULL); + + diag("Creating connection to Admin host=\"%s\" port=%d", cl.host, cl.admin_port); + if (!mysql_real_connect(admin, cl.host, cl.admin_username, cl.admin_password, NULL, cl.admin_port, NULL, 0)) { + fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(admin)); + return EXIT_FAILURE; + } + + MYSQL_QUERY_T(admin, "UPDATE mysql_servers SET max_replication_lag=0"); + MYSQL_QUERY_T(admin, + ("UPDATE pgsql_servers SET max_replication_lag=" + _TO_S(MAX_REPL_LAG) + + " WHERE port=" + _S(REPL_PORT)).c_str() + ); + MYSQL_QUERY_T(admin, "LOAD PGSQL SERVERS TO RUNTIME"); + + // Create connection to PostgreSQL + PGconn* pgconn = create_new_conn(cl, false); + if (pgconn == nullptr) { + return EXIT_FAILURE; + } + + PG_QUERY_T(pgconn, + "CREATE TABLE IF NOT EXISTS sbtest1 (" + " id INTEGER PRIMARY KEY GENERATED ALWAYS AS IDENTITY," + " k integer DEFAULT 0 NOT NULL," + " c character(120) DEFAULT ''::bpchar NOT NULL," + " pad character(60) DEFAULT ''::bpchar NOT NULL" + ")" + ); + + MYSQL_QUERY_T(admin, "SET pgsql-monitor_replication_lag_interval=1000"); + MYSQL_QUERY_T(admin, "LOAD PGSQL VARIABLES TO RUNTIME"); + + // Execute the setup script + { + const string setup_script_path { TEST_DATADIR + "/setup_primary_vintf_throttle.sh" }; + diag("Executing vintf throttling setup script path=\"%s\"", setup_script_path.c_str()); + + int exec_rc = system(setup_script_path.c_str()); + if (exec_rc != 0) { + diag("ERROR: Script execution failed; Aborting further testing err=%d", exec_rc); + // Continue anyway - the script may have already been run + return exit_status(); + } + } + + test_replication_lag(cl, admin, pgconn, MAX_REPL_LAG, 1); + test_replication_lag(cl, admin, pgconn, MAX_REPL_LAG, 3); + + // Execute the cleanup script + { + const string del_script_path { TEST_DATADIR + "/delete_primary_vintf_throttle.sh" }; + diag("Executing vintf throttling setup script path=\"%s\"", del_script_path.c_str()); + + int exec_rc = system(del_script_path.c_str()); + if (exec_rc != 0) { + diag("ERROR: Script execution failed; This could compromise further tests! err=%d", exec_rc); + // Continue anyway - the script may have already been run + return exit_status(); + } + } + + PQfinish(pgconn); + mysql_close(admin); + + return exit_status(); +} diff --git a/test/tap/tests/test_pgsql_replication_lag-t.env b/test/tap/tests/test_pgsql_replication_lag-t.env new file mode 100644 index 0000000000..2e40199e89 --- /dev/null +++ b/test/tap/tests/test_pgsql_replication_lag-t.env @@ -0,0 +1,2 @@ +TAP_PGSQL_USERNAME=postgres +TAP_PGSQL_PASSWORD=postgres diff --git a/test/tap/tests/test_pgsql_replication_lag/delete_primary_vintf_throttle.sh b/test/tap/tests/test_pgsql_replication_lag/delete_primary_vintf_throttle.sh new file mode 100755 index 0000000000..9a5e6d9d7a --- /dev/null +++ b/test/tap/tests/test_pgsql_replication_lag/delete_primary_vintf_throttle.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +set -e + +rsubnet=$(docker exec --user root pgsql-repl-pg_replica-1 sh -c \ + 'getent hosts pg_primary_int | cut -d" " -f1') +vintf=$(docker exec --user root pgsql-repl-pg_primary-1 sh -c \ + "ip route show to match \"$rsubnet\" | grep -E \"$rsubnet\" | cut -d' ' -f3") + +tbf_count=$(docker exec --user root pgsql-repl-pg_primary-1 sh -c \ + "tc qdisc list dev $vintf | grep 'qdisc tbf' | wc -l") + +if [[ $tbf_count -ne 0 ]]; then + echo "[$(date)] Deleting found TBF rule to interface... vintf=$vintf count=$tbf_count" + docker exec --user root pgsql-repl-pg_primary-1 sh -c \ + "tc qdisc del dev $vintf root" +fi diff --git a/test/tap/tests/test_pgsql_replication_lag/setup_primary_vintf_throttle.sh b/test/tap/tests/test_pgsql_replication_lag/setup_primary_vintf_throttle.sh new file mode 100755 index 0000000000..3f8395e4a5 --- /dev/null +++ b/test/tap/tests/test_pgsql_replication_lag/setup_primary_vintf_throttle.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +set -e + +rsubnet=$(docker exec --user root pgsql-repl-pg_replica-1 sh -c \ + 'getent hosts pg_primary_int | cut -d" " -f1') +vintf=$(docker exec --user root pgsql-repl-pg_primary-1 sh -c \ + "ip route show to match \"$rsubnet\" | grep -E \"$rsubnet\" | cut -d' ' -f3") + +tbf_count=$(docker exec --user root pgsql-repl-pg_primary-1 sh -c \ + "tc qdisc list dev $vintf | grep 'qdisc tbf' | wc -l") + +if [[ $tbf_count -ne 0 ]]; then + echo "[$(date)] Deleting found TBF rule to interface... vintf=$vintf count=$tbf_count" + docker exec --user root pgsql-repl-pg_primary-1 sh -c \ + "tc qdisc del dev $vintf root" +fi + +echo "[$(date)] Adding new TBF rule to interface... vintf=$vintf" +docker exec --user root pgsql-repl-pg_primary-1 sh -c \ + "tc qdisc add dev $vintf root tbf rate 8mbit latency 10ms burst 2k" diff --git a/test/tap/tests/test_query_rules_fast_routing_algorithm-t.cpp b/test/tap/tests/test_query_rules_fast_routing_algorithm-t.cpp index 1ae41d7178..2a9d3406d0 100644 --- a/test/tap/tests/test_query_rules_fast_routing_algorithm-t.cpp +++ b/test/tap/tests/test_query_rules_fast_routing_algorithm-t.cpp @@ -200,7 +200,7 @@ int check_fast_routing_rules( diag("Getting last 'debug_log' entry id"); ext_val_t last_id { sq3_query_ext_val(sq3_db, SELECT_LAST_DEBUG_ID, uint32_t(0)) }; - CHECK_EXT_VAL(last_id); + SQ3_CHECK_EXT_VAL(last_id); diag("Fetched last 'debug_log' entry id id=%d", last_id.val); // Check that fast_routing rules are properly working for the defined range @@ -231,7 +231,7 @@ int check_fast_routing_rules( }; ext_val_t entries { sq3_query_ext_val(sq3_db, select_count, int64_t(-1))}; if (entries.err) { - const string err { get_ext_val_err(admin, entries) }; + const string err { sq3_get_ext_val_err(entries) }; diag("%s:%d: Query failed err=\"%s\"", __func__, __LINE__, err.c_str()); return { -1, 0 }; } @@ -272,12 +272,12 @@ int threads_warmup(const CommandLine& cl, MYSQL* admin, sqlite3* sq3_db) { "SELECT variable_value FROM global_variables WHERE variable_name='mysql-threads'", 0 ) }; - CHECK_EXT_VAL(mysql_threads); + CHECK_EXT_VAL(admin, mysql_threads); const ext_val_t qlog_fname { mysql_query_ext_val(admin, SELECT_RUNTIME_VAR"'mysql-eventslog_filename'", _S("query.log")) }; - CHECK_EXT_VAL(qlog_fname); + CHECK_EXT_VAL(admin, qlog_fname); const string PROXYSQL_AUDIT_LOG { PROXYSQL_QLOG_DIR + "/" + qlog_fname.str }; diag("Flush debug logs to ensure getting the latest id"); @@ -285,7 +285,7 @@ int threads_warmup(const CommandLine& cl, MYSQL* admin, sqlite3* sq3_db) { diag("Getting last 'debug_log' entry id"); ext_val_t last_id { sq3_query_ext_val(sq3_db, SELECT_LAST_DEBUG_ID, uint32_t(0)) }; - CHECK_EXT_VAL(last_id); + SQ3_CHECK_EXT_VAL(last_id); diag("Fetched last 'debug_log' entry id id=%d", last_id.val); int conns { find_min_elems(1.0 - pow(10, -6), mysql_threads.val) /* * 2 */ }; @@ -409,7 +409,7 @@ int test_fast_routing_algorithm( MYSQL_QUERY_T(admin, "LOAD MYSQL QUERY RULES TO RUNTIME"); ext_val_t init_mem_stats { mysql_query_ext_val(admin, q_query_rules_mem_stats, -1) }; - CHECK_EXT_VAL(init_mem_stats); + CHECK_EXT_VAL(admin, init_mem_stats); diag("Initial 'mysql_query_rules_memory' of '%d'", init_mem_stats.val); c_err = create_fast_routing_rules_range(cl, admin, rng_init, rng_end); @@ -426,7 +426,7 @@ int test_fast_routing_algorithm( printf("\n"); ext_val_t old_mem_stats { mysql_query_ext_val(admin, q_query_rules_mem_stats, -1) }; - CHECK_EXT_VAL(old_mem_stats); + CHECK_EXT_VAL(admin, old_mem_stats); diag("*ONLY* Changing the algorithm shouldn't have any effect"); diag("Testing 'query_rules_fast_routing_algorithm=%d'", new_algo); @@ -439,7 +439,7 @@ int test_fast_routing_algorithm( printf("\n"); ext_val_t new_mem_stats { mysql_query_ext_val(admin, q_query_rules_mem_stats, -1) }; - CHECK_EXT_VAL(new_mem_stats); + CHECK_EXT_VAL(admin, new_mem_stats); diag("Memory SHOULDN'T have changed just because of a variable change"); ok( @@ -459,7 +459,7 @@ int test_fast_routing_algorithm( if (chk_res) { return EXIT_FAILURE; } new_mem_stats = mysql_query_ext_val(admin, q_query_rules_mem_stats, -1); - CHECK_EXT_VAL(new_mem_stats); + CHECK_EXT_VAL(admin, new_mem_stats); bool mem_check_res = false; string exp_change { "" };