diff --git a/include/ProxySQL_Cluster.hpp b/include/ProxySQL_Cluster.hpp index 05621f543e..9805bae3b7 100644 --- a/include/ProxySQL_Cluster.hpp +++ b/include/ProxySQL_Cluster.hpp @@ -5,6 +5,7 @@ #include "thread.h" #include "wqueue.h" #include +#include #include "prometheus/counter.h" #include "prometheus/gauge.h" @@ -12,6 +13,15 @@ #define PROXYSQL_NODE_METRICS_LEN 5 /** + * @file ProxySQL_Cluster.hpp + * @brief ProxySQL Cluster synchronization and management definitions. + * + * This file contains definitions for ProxySQL's clustering functionality, including: + * - Cluster query definitions for MySQL and PostgreSQL module synchronization + * - Node management and metrics collection + * - Checksum computation and comparison algorithms + * - Peer selection and synchronization logic + * * CLUSTER QUERIES DEFINITION * ========================== * @@ -19,11 +29,20 @@ * the queries issued for generating the checksum for each of the target modules, for simpler reasoning, they should * also represent the actual resultset being received when issuing them, since this resultset is used for computing the * 'expected checksum' for the fetched config before loading it to runtime. This is done for the following modules: + * + * MySQL modules: * - 'runtime_mysql_servers': tables 'mysql_servers' * - 'runtime_mysql_users'. * - 'runtime_mysql_query_rules'. * - 'mysql_servers_v2': tables admin 'mysql_servers', 'mysql_replication_hostgroups', 'mysql_group_replication_hostroups', * 'mysql_galera_hostgroups', 'mysql_aws_aurora_hostgroups', 'mysql_hostgroup_attributes'. + * + * PostgreSQL modules: + * - 'runtime_pgsql_servers': runtime PostgreSQL server status and configuration + * - 'runtime_pgsql_users': runtime PostgreSQL user authentication settings + * - 'runtime_pgsql_query_rules': runtime PostgreSQL query routing rules + * - 'pgsql_servers_v2': static PostgreSQL server configuration + * * IMPORTANT: For further clarify this means that it's important that the actual resultset produced by the intercepted * query preserve the filtering and ordering expressed in this queries. */ @@ -61,6 +80,131 @@ /* @brief Query to be intercepted by 'ProxySQL_Admin' for 'runtime_mysql_query_rules_fast_routing'. See top comment for details. */ #define CLUSTER_QUERY_MYSQL_QUERY_RULES_FAST_ROUTING "PROXY_SELECT username, schemaname, flagIN, destination_hostgroup, comment FROM runtime_mysql_query_rules_fast_routing ORDER BY username, schemaname, flagIN" +/** + * @brief Query to be intercepted by 'ProxySQL_Admin' for 'runtime_pgsql_servers'. + * + * This query retrieves the current operational status and configuration of PostgreSQL servers + * from the runtime_pgsql_servers table. It includes server health metrics, connection settings, + * and current operational status. The query filters out OFFLINE_HARD servers and converts + * numeric status values to human-readable format. + * + * Result columns: + * - hostgroup_id: Logical grouping identifier for PostgreSQL servers + * - hostname: Server hostname or IP address + * - port: PostgreSQL server port number + * - status: Converted status string (ONLINE, OFFLINE_SOFT, OFFLINE_HARD) + * - weight: Load balancing weight for the server + * - compression: Whether compression is enabled + * - max_connections: Maximum allowed connections + * - max_replication_lag: Maximum acceptable replication lag + * - use_ssl: SSL/TLS connection requirement + * - max_latency_ms: Maximum acceptable latency + * - comment: Administrative comments + * + * @see runtime_pgsql_servers + * @see pull_runtime_pgsql_servers_from_peer() + */ +#define CLUSTER_QUERY_RUNTIME_PGSQL_SERVERS "PROXY_SELECT hostgroup_id, hostname, port, CASE status WHEN 0 THEN \"ONLINE\" WHEN 1 THEN \"ONLINE\" WHEN 2 THEN \"OFFLINE_SOFT\" WHEN 3 THEN \"OFFLINE_HARD\" WHEN 4 THEN \"ONLINE\" END status, weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM runtime_pgsql_servers WHERE status<>'OFFLINE_HARD' ORDER BY hostgroup_id, hostname, port" + +/** + * @brief Query to be intercepted by 'ProxySQL_Admin' for 'pgsql_servers_v2'. + * + * This query retrieves the static configuration of PostgreSQL servers from the pgsql_servers_v2 table. + * It includes connection parameters, load balancing settings, and server metadata. The query + * filters out OFFLINE_HARD servers and converts SHUNNED status to ONLINE for cluster synchronization. + * + * Result columns: + * - hostgroup_id: Logical grouping identifier for PostgreSQL servers + * - hostname: Server hostname or IP address + * - port: PostgreSQL server port number + * - status: Server status (SHUNNED converted to ONLINE for sync) + * - weight: Load balancing weight for the server + * - compression: Whether compression is enabled + * - max_connections: Maximum allowed connections + * - max_replication_lag: Maximum acceptable replication lag + * - use_ssl: SSL/TLS connection requirement + * - max_latency_ms: Maximum acceptable latency + * - comment: Administrative comments + * + * @see pgsql_servers_v2 + * @see pull_pgsql_servers_v2_from_peer() + */ +#define CLUSTER_QUERY_PGSQL_SERVERS_V2 "PROXY_SELECT hostgroup_id, hostname, port, CASE WHEN status=\"SHUNNED\" THEN \"ONLINE\" ELSE status END AS status, weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM pgsql_servers_v2 WHERE status<>'OFFLINE_HARD' ORDER BY hostgroup_id, hostname, port" + +/** + * @brief Query to be intercepted by 'ProxySQL_Admin' for 'runtime_pgsql_users'. + * + * This query retrieves PostgreSQL user authentication configuration from the runtime_pgsql_users table. + * It includes credentials, connection settings, and user behavior preferences that are used for + * authenticating and managing PostgreSQL client connections. + * + * Result columns: + * - username: PostgreSQL username + * - password: Authentication password/hash + * - use_ssl: SSL/TLS connection requirement + * - default_hostgroup: Default hostgroup for routing + * - transaction_persistent: Whether transactions persist across connections + * - fast_forward: Fast forwarding mode setting + * - backend: Backend connection settings + * - frontend: Frontend connection settings + * - max_connections: Maximum connections per user + * - attributes: Additional user attributes (JSON) + * - comment: Administrative comments + * + * @see runtime_pgsql_users + * @see pull_pgsql_users_from_peer() + */ +#define CLUSTER_QUERY_PGSQL_USERS "PROXY_SELECT username, password, use_ssl, default_hostgroup, transaction_persistent, fast_forward, backend, frontend, max_connections, attributes, comment FROM runtime_pgsql_users" + +/** + * @brief Query to be intercepted by 'ProxySQL_Admin' for 'runtime_pgsql_query_rules'. + * + * This query retrieves PostgreSQL query routing rules from the runtime_pgsql_query_rules table. + * It includes comprehensive rule definitions for query matching, routing, caching, and behavior + * control. Rules are ordered by rule_id to ensure consistent processing and checksum generation. + * + * Key result columns: + * - rule_id: Unique identifier for the rule + * - username: Filter by PostgreSQL username + * - database: Filter by database name (PostgreSQL-specific, replaces schemaname) + * - flagIN, flagOUT: Rule processing flags + * - match_digest, match_pattern: Query matching criteria + * - destination_hostgroup: Target hostgroup for matching queries + * - cache_ttl, cache_timeout: Query caching settings + * - timeout, retries, delay: Query execution parameters + * - mirror_hostgroup: Query mirroring destination + * - error_msg, ok_msg: Custom response messages + * - sticky_conn, multiplex: Connection pooling behavior + * - log, apply: Logging and application flags + * - attributes: Additional rule attributes (JSON) + * - comment: Administrative comments + * + * @see runtime_pgsql_query_rules + * @see pull_pgsql_query_rules_from_peer() + */ +#define CLUSTER_QUERY_PGSQL_QUERY_RULES "PROXY_SELECT rule_id, username, database, flagIN, client_addr, proxy_addr, proxy_port, digest, match_digest, match_pattern, negate_match_pattern, re_modifiers, flagOUT, replace_pattern, destination_hostgroup, cache_ttl, cache_empty_result, cache_timeout, reconnect, timeout, retries, delay, next_query_flagIN, mirror_flagOUT, mirror_hostgroup, error_msg, ok_msg, sticky_conn, multiplex, log, apply, attributes, comment FROM runtime_pgsql_query_rules ORDER BY rule_id" + +/** + * @brief Query to be intercepted by 'ProxySQL_Admin' for 'runtime_pgsql_query_rules_fast_routing'. + * + * This query retrieves PostgreSQL fast routing rules from the runtime_pgsql_query_rules_fast_routing table. + * Fast routing provides a lightweight mechanism for direct query routing based on username, database, + * and processing flags without complex pattern matching. This enables efficient routing for common + * use cases and reduces processing overhead. + * + * Result columns: + * - username: PostgreSQL username for routing rule + * - database: Database name for routing rule (PostgreSQL-specific) + * - flagIN: Input processing flag for rule matching + * - destination_hostgroup: Target hostgroup for direct routing + * - comment: Administrative comments + * + * @see runtime_pgsql_query_rules_fast_routing + * @see pull_pgsql_query_rules_from_peer() + * @see CLUSTER_QUERY_PGSQL_QUERY_RULES + */ +#define CLUSTER_QUERY_PGSQL_QUERY_RULES_FAST_ROUTING "PROXY_SELECT username, database, flagIN, destination_hostgroup, comment FROM runtime_pgsql_query_rules_fast_routing ORDER BY username, database, flagIN" + class ProxySQL_Checksum_Value_2: public ProxySQL_Checksum_Value { public: time_t last_updated; @@ -161,6 +305,11 @@ class ProxySQL_Node_Entry { ProxySQL_Checksum_Value_2 mysql_users; ProxySQL_Checksum_Value_2 proxysql_servers; ProxySQL_Checksum_Value_2 mysql_servers_v2; + ProxySQL_Checksum_Value_2 pgsql_query_rules; + ProxySQL_Checksum_Value_2 pgsql_servers; + ProxySQL_Checksum_Value_2 pgsql_users; + ProxySQL_Checksum_Value_2 pgsql_servers_v2; + ProxySQL_Checksum_Value_2 pgsql_variables; } checksums_values; uint64_t global_checksum; }; @@ -255,10 +404,16 @@ class ProxySQL_Cluster_Nodes { void get_peer_to_sync_mysql_servers_v2(char** host, uint16_t* port, char** peer_mysql_servers_v2_checksum, char** peer_runtime_mysql_servers_checksum, char** ip_address); void get_peer_to_sync_mysql_users(char **host, uint16_t *port, char** ip_address); + void get_peer_to_sync_variables_module(const char* module_name, char **host, uint16_t *port, char** ip_address); void get_peer_to_sync_mysql_variables(char **host, uint16_t *port, char** ip_address); void get_peer_to_sync_admin_variables(char **host, uint16_t* port, char** ip_address); void get_peer_to_sync_ldap_variables(char **host, uint16_t *port, char** ip_address); void get_peer_to_sync_proxysql_servers(char **host, uint16_t *port, char ** ip_address); + void get_peer_to_sync_pgsql_query_rules(char **host, uint16_t *port, char** ip_address); + void get_peer_to_sync_runtime_pgsql_servers(char **host, uint16_t *port, char **peer_checksum, char** ip_address); + void get_peer_to_sync_pgsql_servers_v2(char** host, uint16_t* port, char** peer_pgsql_servers_v2_checksum, + char** peer_runtime_pgsql_servers_checksum, char** ip_address); + void get_peer_to_sync_pgsql_users(char **host, uint16_t *port, char** ip_address); }; struct p_cluster_counter { @@ -298,6 +453,15 @@ struct p_cluster_counter { pulled_ldap_variables_success, pulled_ldap_variables_failure, + pulled_pgsql_query_rules_success, + pulled_pgsql_query_rules_failure, + pulled_pgsql_servers_success, + pulled_pgsql_servers_failure, + pulled_pgsql_users_success, + pulled_pgsql_users_failure, + pulled_pgsql_variables_success, + pulled_pgsql_variables_failure, + pulled_mysql_ldap_mapping_success, pulled_mysql_ldap_mapping_failure, @@ -308,6 +472,7 @@ struct p_cluster_counter { sync_conflict_mysql_variables_share_epoch, sync_conflict_admin_variables_share_epoch, sync_conflict_ldap_variables_share_epoch, + sync_conflict_pgsql_variables_share_epoch, sync_delayed_mysql_query_rules_version_one, sync_delayed_mysql_servers_version_one, @@ -316,6 +481,7 @@ struct p_cluster_counter { sync_delayed_mysql_variables_version_one, sync_delayed_admin_variables_version_one, sync_delayed_ldap_variables_version_one, + sync_delayed_pgsql_variables_version_one, __size }; @@ -418,13 +584,17 @@ class ProxySQL_Cluster { char* admin_mysql_ifaces; int cluster_check_interval_ms; int cluster_check_status_frequency; - int cluster_mysql_query_rules_diffs_before_sync; - int cluster_mysql_servers_diffs_before_sync; - int cluster_mysql_users_diffs_before_sync; - int cluster_proxysql_servers_diffs_before_sync; - int cluster_mysql_variables_diffs_before_sync; - int cluster_ldap_variables_diffs_before_sync; - int cluster_admin_variables_diffs_before_sync; + std::atomic cluster_mysql_query_rules_diffs_before_sync; + std::atomic cluster_mysql_servers_diffs_before_sync; + std::atomic cluster_mysql_users_diffs_before_sync; + std::atomic cluster_proxysql_servers_diffs_before_sync; + std::atomic cluster_mysql_variables_diffs_before_sync; + std::atomic cluster_ldap_variables_diffs_before_sync; + std::atomic cluster_admin_variables_diffs_before_sync; + std::atomic cluster_pgsql_query_rules_diffs_before_sync; + std::atomic cluster_pgsql_servers_diffs_before_sync; + std::atomic cluster_pgsql_users_diffs_before_sync; + std::atomic cluster_pgsql_variables_diffs_before_sync; int cluster_mysql_servers_sync_algorithm; bool cluster_mysql_query_rules_save_to_disk; bool cluster_mysql_servers_save_to_disk; @@ -433,6 +603,10 @@ class ProxySQL_Cluster { bool cluster_mysql_variables_save_to_disk; bool cluster_ldap_variables_save_to_disk; bool cluster_admin_variables_save_to_disk; + bool cluster_pgsql_query_rules_save_to_disk; + bool cluster_pgsql_servers_save_to_disk; + bool cluster_pgsql_users_save_to_disk; + bool cluster_pgsql_variables_save_to_disk; ProxySQL_Cluster(); ~ProxySQL_Cluster(); void init() {}; @@ -491,5 +665,11 @@ class ProxySQL_Cluster { */ void pull_global_variables_from_peer(const std::string& type, const std::string& expected_checksum, const time_t epoch); void pull_proxysql_servers_from_peer(const std::string& expected_checksum, const time_t epoch); + void pull_pgsql_query_rules_from_peer(const std::string& expected_checksum, const time_t epoch); + void pull_runtime_pgsql_servers_from_peer(const runtime_pgsql_servers_checksum_t& peer_runtime_pgsql_server); + void pull_pgsql_servers_v2_from_peer(const pgsql_servers_v2_checksum_t& peer_pgsql_server_v2, + const runtime_pgsql_servers_checksum_t& peer_runtime_pgsql_server = {}, bool fetch_runtime_pgsql_servers = false); + void pull_pgsql_users_from_peer(const std::string& expected_checksum, const time_t epoch); + void pull_pgsql_variables_from_peer(const std::string& expected_checksum, const time_t epoch); }; #endif /* CLASS_PROXYSQL_CLUSTER_H */ diff --git a/include/proxysql_admin.h b/include/proxysql_admin.h index bc8f35675b..f7827e6ab3 100644 --- a/include/proxysql_admin.h +++ b/include/proxysql_admin.h @@ -313,6 +313,9 @@ class ProxySQL_Admin { int cluster_mysql_variables_diffs_before_sync; int cluster_admin_variables_diffs_before_sync; int cluster_ldap_variables_diffs_before_sync; + int cluster_pgsql_query_rules_diffs_before_sync; + int cluster_pgsql_servers_diffs_before_sync; + int cluster_pgsql_users_diffs_before_sync; int cluster_mysql_servers_sync_algorithm; bool cluster_mysql_query_rules_save_to_disk; bool cluster_mysql_servers_save_to_disk; @@ -321,6 +324,9 @@ class ProxySQL_Admin { bool cluster_mysql_variables_save_to_disk; bool cluster_admin_variables_save_to_disk; bool cluster_ldap_variables_save_to_disk; + bool cluster_pgsql_query_rules_save_to_disk; + bool cluster_pgsql_servers_save_to_disk; + bool cluster_pgsql_users_save_to_disk; int stats_mysql_connection_pool; int stats_mysql_connections; int stats_mysql_query_cache; diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index daebe79ced..2aa0658669 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -388,6 +388,9 @@ static char * admin_variables_names[]= { (char *)"cluster_mysql_variables_diffs_before_sync", (char *)"cluster_admin_variables_diffs_before_sync", (char *)"cluster_ldap_variables_diffs_before_sync", + (char *)"cluster_pgsql_query_rules_diffs_before_sync", + (char *)"cluster_pgsql_servers_diffs_before_sync", + (char *)"cluster_pgsql_users_diffs_before_sync", (char *)"cluster_mysql_query_rules_save_to_disk", (char *)"cluster_mysql_servers_save_to_disk", (char *)"cluster_mysql_users_save_to_disk", @@ -395,6 +398,9 @@ static char * admin_variables_names[]= { (char *)"cluster_mysql_variables_save_to_disk", (char *)"cluster_admin_variables_save_to_disk", (char *)"cluster_ldap_variables_save_to_disk", + (char *)"cluster_pgsql_query_rules_save_to_disk", + (char *)"cluster_pgsql_servers_save_to_disk", + (char *)"cluster_pgsql_users_save_to_disk", (char *)"cluster_mysql_servers_sync_algorithm", (char *)"checksum_mysql_query_rules", (char *)"checksum_mysql_servers", @@ -2623,6 +2629,9 @@ ProxySQL_Admin::ProxySQL_Admin() : variables.cluster_mysql_variables_diffs_before_sync = 3; variables.cluster_admin_variables_diffs_before_sync = 3; variables.cluster_ldap_variables_diffs_before_sync = 3; + variables.cluster_pgsql_query_rules_diffs_before_sync = 3; + variables.cluster_pgsql_servers_diffs_before_sync = 3; + variables.cluster_pgsql_users_diffs_before_sync = 3; variables.cluster_mysql_servers_sync_algorithm = 1; checksum_variables.checksum_mysql_query_rules = true; checksum_variables.checksum_mysql_servers = true; @@ -2637,6 +2646,9 @@ ProxySQL_Admin::ProxySQL_Admin() : variables.cluster_mysql_variables_save_to_disk = true; variables.cluster_admin_variables_save_to_disk = true; variables.cluster_ldap_variables_save_to_disk = true; + variables.cluster_pgsql_query_rules_save_to_disk = true; + variables.cluster_pgsql_servers_save_to_disk = true; + variables.cluster_pgsql_users_save_to_disk = true; variables.stats_mysql_connection_pool = 60; variables.stats_mysql_connections = 60; variables.stats_mysql_query_cache = 60; @@ -3360,6 +3372,18 @@ char * ProxySQL_Admin::get_variable(char *name) { sprintf(intbuf,"%d",variables.cluster_ldap_variables_diffs_before_sync); return strdup(intbuf); } + if (!strcasecmp(name,"cluster_pgsql_query_rules_diffs_before_sync")) { + sprintf(intbuf,"%d",variables.cluster_pgsql_query_rules_diffs_before_sync); + return strdup(intbuf); + } + if (!strcasecmp(name,"cluster_pgsql_servers_diffs_before_sync")) { + sprintf(intbuf,"%d",variables.cluster_pgsql_servers_diffs_before_sync); + return strdup(intbuf); + } + if (!strcasecmp(name,"cluster_pgsql_users_diffs_before_sync")) { + sprintf(intbuf,"%d",variables.cluster_pgsql_users_diffs_before_sync); + return strdup(intbuf); + } if (!strcasecmp(name,"cluster_mysql_servers_sync_algorithm")) { sprintf(intbuf, "%d", variables.cluster_mysql_servers_sync_algorithm); return strdup(intbuf); @@ -3385,6 +3409,15 @@ char * ProxySQL_Admin::get_variable(char *name) { if (!strcasecmp(name,"cluster_ldap_variables_save_to_disk")) { return strdup((variables.cluster_ldap_variables_save_to_disk ? "true" : "false")); } + if (!strcasecmp(name,"cluster_pgsql_query_rules_save_to_disk")) { + return strdup((variables.cluster_pgsql_query_rules_save_to_disk ? "true" : "false")); + } + if (!strcasecmp(name,"cluster_pgsql_servers_save_to_disk")) { + return strdup((variables.cluster_pgsql_servers_save_to_disk ? "true" : "false")); + } + if (!strcasecmp(name,"cluster_pgsql_users_save_to_disk")) { + return strdup((variables.cluster_pgsql_users_save_to_disk ? "true" : "false")); + } if (!strcasecmp(name,"refresh_interval")) { sprintf(intbuf,"%d",variables.refresh_interval); return strdup(intbuf); @@ -3798,7 +3831,7 @@ bool ProxySQL_Admin::set_variable(char *name, char *value, bool lock) { // this GloProxyCluster->Reset_Global_Checksums(lock); } variables.cluster_mysql_query_rules_diffs_before_sync=intv; - __sync_lock_test_and_set(&GloProxyCluster->cluster_mysql_query_rules_diffs_before_sync, intv); + GloProxyCluster->cluster_mysql_query_rules_diffs_before_sync = intv; return true; } else { return false; @@ -3813,7 +3846,7 @@ bool ProxySQL_Admin::set_variable(char *name, char *value, bool lock) { // this GloProxyCluster->Reset_Global_Checksums(lock); } variables.cluster_mysql_servers_diffs_before_sync=intv; - __sync_lock_test_and_set(&GloProxyCluster->cluster_mysql_servers_diffs_before_sync, intv); + GloProxyCluster->cluster_mysql_servers_diffs_before_sync = intv; return true; } else { return false; @@ -3828,7 +3861,7 @@ bool ProxySQL_Admin::set_variable(char *name, char *value, bool lock) { // this GloProxyCluster->Reset_Global_Checksums(lock); } variables.cluster_mysql_users_diffs_before_sync=intv; - __sync_lock_test_and_set(&GloProxyCluster->cluster_mysql_users_diffs_before_sync, intv); + GloProxyCluster->cluster_mysql_users_diffs_before_sync = intv; return true; } else { return false; @@ -3842,7 +3875,7 @@ bool ProxySQL_Admin::set_variable(char *name, char *value, bool lock) { // this GloProxyCluster->Reset_Global_Checksums(lock); } variables.cluster_proxysql_servers_diffs_before_sync=intv; - __sync_lock_test_and_set(&GloProxyCluster->cluster_proxysql_servers_diffs_before_sync, intv); + GloProxyCluster->cluster_proxysql_servers_diffs_before_sync = intv; return true; } else { return false; @@ -3857,7 +3890,7 @@ bool ProxySQL_Admin::set_variable(char *name, char *value, bool lock) { // this GloProxyCluster->Reset_Global_Checksums(lock); } variables.cluster_mysql_variables_diffs_before_sync=intv; - __sync_lock_test_and_set(&GloProxyCluster->cluster_mysql_variables_diffs_before_sync, intv); + GloProxyCluster->cluster_mysql_variables_diffs_before_sync = intv; return true; } else { return false; @@ -3872,7 +3905,7 @@ bool ProxySQL_Admin::set_variable(char *name, char *value, bool lock) { // this GloProxyCluster->Reset_Global_Checksums(lock); } variables.cluster_admin_variables_diffs_before_sync=intv; - __sync_lock_test_and_set(&GloProxyCluster->cluster_admin_variables_diffs_before_sync, intv); + GloProxyCluster->cluster_admin_variables_diffs_before_sync = intv; return true; } else { return false; @@ -3887,7 +3920,52 @@ bool ProxySQL_Admin::set_variable(char *name, char *value, bool lock) { // this GloProxyCluster->Reset_Global_Checksums(lock); } variables.cluster_ldap_variables_diffs_before_sync=intv; - __sync_lock_test_and_set(&GloProxyCluster->cluster_ldap_variables_diffs_before_sync, intv); + GloProxyCluster->cluster_ldap_variables_diffs_before_sync = intv; + return true; + } else { + return false; + } + } + if (!strcasecmp(name,"cluster_pgsql_query_rules_diffs_before_sync")) { + int intv=atoi(value); + if (intv >= 0 && intv <= 1000) { + intv = checksum_variables.checksum_mysql_query_rules ? intv : 0; // Reuse mysql_query_rules checksum + if (variables.cluster_pgsql_query_rules_diffs_before_sync == 0 && intv != 0) { + proxy_info("Re-enabled previously disabled 'admin-cluster_pgsql_query_rules_diffs_before_sync'. Resetting global checksums to force Cluster re-sync.\n"); + GloProxyCluster->Reset_Global_Checksums(lock); + } + variables.cluster_pgsql_query_rules_diffs_before_sync=intv; + GloProxyCluster->cluster_pgsql_query_rules_diffs_before_sync = intv; + return true; + } else { + return false; + } + } + if (!strcasecmp(name,"cluster_pgsql_servers_diffs_before_sync")) { + int intv=atoi(value); + if (intv >= 0 && intv <= 1000) { + intv = checksum_variables.checksum_mysql_servers ? intv : 0; // Reuse mysql_servers checksum + if (variables.cluster_pgsql_servers_diffs_before_sync == 0 && intv != 0) { + proxy_info("Re-enabled previously disabled 'admin-cluster_pgsql_servers_diffs_before_sync'. Resetting global checksums to force Cluster re-sync.\n"); + GloProxyCluster->Reset_Global_Checksums(lock); + } + variables.cluster_pgsql_servers_diffs_before_sync=intv; + GloProxyCluster->cluster_pgsql_servers_diffs_before_sync = intv; + return true; + } else { + return false; + } + } + if (!strcasecmp(name,"cluster_pgsql_users_diffs_before_sync")) { + int intv=atoi(value); + if (intv >= 0 && intv <= 1000) { + intv = checksum_variables.checksum_mysql_users ? intv : 0; // Reuse mysql_users checksum + if (variables.cluster_pgsql_users_diffs_before_sync == 0 && intv != 0) { + proxy_info("Re-enabled previously disabled 'admin-cluster_pgsql_users_diffs_before_sync'. Resetting global checksums to force Cluster re-sync.\n"); + GloProxyCluster->Reset_Global_Checksums(lock); + } + variables.cluster_pgsql_users_diffs_before_sync=intv; + GloProxyCluster->cluster_pgsql_users_diffs_before_sync = intv; return true; } else { return false; @@ -4092,6 +4170,48 @@ bool ProxySQL_Admin::set_variable(char *name, char *value, bool lock) { // this } return rt; } + if (!strcasecmp(name,"cluster_pgsql_query_rules_save_to_disk")) { + bool rt = false; + if (strcasecmp(value,"true")==0 || strcasecmp(value,"1")==0) { + variables.cluster_pgsql_query_rules_save_to_disk=true; + rt = __sync_lock_test_and_set(&GloProxyCluster->cluster_pgsql_query_rules_save_to_disk, true); + return true; + } + if (strcasecmp(value,"false")==0 || strcasecmp(value,"0")==0) { + variables.cluster_pgsql_query_rules_save_to_disk=false; + rt = __sync_lock_test_and_set(&GloProxyCluster->cluster_pgsql_query_rules_save_to_disk, false); + return true; + } + return rt; + } + if (!strcasecmp(name,"cluster_pgsql_servers_save_to_disk")) { + bool rt = false; + if (strcasecmp(value,"true")==0 || strcasecmp(value,"1")==0) { + variables.cluster_pgsql_servers_save_to_disk=true; + rt = __sync_lock_test_and_set(&GloProxyCluster->cluster_pgsql_servers_save_to_disk, true); + return true; + } + if (strcasecmp(value,"false")==0 || strcasecmp(value,"0")==0) { + variables.cluster_pgsql_servers_save_to_disk=false; + rt = __sync_lock_test_and_set(&GloProxyCluster->cluster_pgsql_servers_save_to_disk, false); + return true; + } + return rt; + } + if (!strcasecmp(name,"cluster_pgsql_users_save_to_disk")) { + bool rt = false; + if (strcasecmp(value,"true")==0 || strcasecmp(value,"1")==0) { + variables.cluster_pgsql_users_save_to_disk=true; + rt = __sync_lock_test_and_set(&GloProxyCluster->cluster_pgsql_users_save_to_disk, true); + return true; + } + if (strcasecmp(value,"false")==0 || strcasecmp(value,"0")==0) { + variables.cluster_pgsql_users_save_to_disk=false; + rt = __sync_lock_test_and_set(&GloProxyCluster->cluster_pgsql_users_save_to_disk, false); + return true; + } + return rt; + } if (!strcasecmp(name,"checksum_mysql_query_rules")) { if (strcasecmp(value,"true")==0 || strcasecmp(value,"1")==0) { checksum_variables.checksum_mysql_query_rules=true; @@ -4100,7 +4220,7 @@ bool ProxySQL_Admin::set_variable(char *name, char *value, bool lock) { // this if (strcasecmp(value,"false")==0 || strcasecmp(value,"0")==0) { checksum_variables.checksum_mysql_query_rules=false; variables.cluster_mysql_query_rules_diffs_before_sync = 0; - __sync_lock_test_and_set(&GloProxyCluster->cluster_mysql_query_rules_diffs_before_sync, 0); + GloProxyCluster->cluster_mysql_query_rules_diffs_before_sync = 0; proxy_warning("Disabling deprecated 'admin-checksum_mysql_query_rules', setting 'admin-cluster_mysql_query_rules_diffs_before_sync=0'\n"); return true; } @@ -4114,7 +4234,7 @@ bool ProxySQL_Admin::set_variable(char *name, char *value, bool lock) { // this if (strcasecmp(value,"false")==0 || strcasecmp(value,"0")==0) { checksum_variables.checksum_mysql_servers=false; variables.cluster_mysql_servers_diffs_before_sync = 0; - __sync_lock_test_and_set(&GloProxyCluster->cluster_mysql_servers_diffs_before_sync, 0); + GloProxyCluster->cluster_mysql_servers_diffs_before_sync = 0; proxy_warning("Disabling deprecated 'admin-checksum_mysql_servers', setting 'admin-cluster_mysql_servers_diffs_before_sync=0'\n"); return true; } @@ -4129,7 +4249,7 @@ bool ProxySQL_Admin::set_variable(char *name, char *value, bool lock) { // this if (strcasecmp(value,"false")==0 || strcasecmp(value,"0")==0) { checksum_variables.checksum_mysql_users=false; variables.cluster_mysql_users_diffs_before_sync = 0; - __sync_lock_test_and_set(&GloProxyCluster->cluster_mysql_users_diffs_before_sync, 0); + GloProxyCluster->cluster_mysql_users_diffs_before_sync = 0; proxy_warning("Disabling deprecated 'admin-checksum_mysql_users', setting 'admin-cluster_mysql_users_diffs_before_sync=0'\n"); return true; } @@ -4143,7 +4263,7 @@ bool ProxySQL_Admin::set_variable(char *name, char *value, bool lock) { // this if (strcasecmp(value,"false")==0 || strcasecmp(value,"0")==0) { checksum_variables.checksum_mysql_variables=false; variables.cluster_mysql_variables_diffs_before_sync = 0; - __sync_lock_test_and_set(&GloProxyCluster->cluster_mysql_variables_diffs_before_sync, 0); + GloProxyCluster->cluster_mysql_variables_diffs_before_sync = 0; proxy_warning("Disabling deprecated 'admin-checksum_mysql_variables', setting 'admin-cluster_mysql_variables_diffs_before_sync=0'\n"); return true; } @@ -4157,7 +4277,7 @@ bool ProxySQL_Admin::set_variable(char *name, char *value, bool lock) { // this if (strcasecmp(value,"false")==0 || strcasecmp(value,"0")==0) { checksum_variables.checksum_admin_variables=false; variables.cluster_admin_variables_diffs_before_sync = 0; - __sync_lock_test_and_set(&GloProxyCluster->cluster_admin_variables_diffs_before_sync, 0); + GloProxyCluster->cluster_admin_variables_diffs_before_sync = 0; proxy_warning("Disabling deprecated 'admin-checksum_admin_variables', setting 'admin-cluster_admin_variables_diffs_before_sync=0'\n"); return true; } @@ -4171,7 +4291,7 @@ bool ProxySQL_Admin::set_variable(char *name, char *value, bool lock) { // this if (strcasecmp(value,"false")==0 || strcasecmp(value,"0")==0) { checksum_variables.checksum_ldap_variables=false; variables.cluster_ldap_variables_diffs_before_sync = 0; - __sync_lock_test_and_set(&GloProxyCluster->cluster_ldap_variables_diffs_before_sync, 0); + GloProxyCluster->cluster_ldap_variables_diffs_before_sync = 0; proxy_warning("Disabling deprecated 'admin-checksum_ldap_variables', setting 'admin-cluster_ldap_variables_diffs_before_sync=0'\n"); return true; } @@ -6129,6 +6249,14 @@ void ProxySQL_Admin::dump_checksums_values_table() { SAFE_SQLITE3_STEP2(statement1); rc = (*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, admindb); rc = (*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, admindb); + + rc = (*proxy_sqlite3_bind_text)(statement1, 1, "pgsql_servers_v2", -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_int64)(statement1, 2, GloVars.checksums_values.pgsql_servers_v2.version); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_int64)(statement1, 3, GloVars.checksums_values.pgsql_servers_v2.epoch); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_text)(statement1, 4, GloVars.checksums_values.pgsql_servers_v2.checksum, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + SAFE_SQLITE3_STEP2(statement1); + rc = (*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, admindb); // diff --git a/lib/ProxySQL_Cluster.cpp b/lib/ProxySQL_Cluster.cpp index 647a522a7a..966d866d98 100644 --- a/lib/ProxySQL_Cluster.cpp +++ b/lib/ProxySQL_Cluster.cpp @@ -1,4 +1,6 @@ #include +#include +#include #include "proxysql.h" #include "proxysql_utils.h" @@ -13,6 +15,8 @@ #include "ProxySQL_Cluster.hpp" #include "MySQL_Authentication.hpp" #include "MySQL_LDAP_Authentication.hpp" +#include "PgSQL_Authentication.h" +#include "PgSQL_Query_Processor.h" #ifdef DEBUG #define DEB "_DEBUG" @@ -43,6 +47,8 @@ extern ProxySQL_Cluster * GloProxyCluster; extern ProxySQL_Admin *GloAdmin; extern MySQL_LDAP_Authentication* GloMyLdapAuth; extern MySQL_Authentication* GloMyAuth; +extern PgSQL_Authentication *GloPgAuth; +extern PgSQL_Query_Processor* GloPgQPro; void * ProxySQL_Cluster_Monitor_thread(void *args) { pthread_attr_t thread_attr; @@ -400,439 +406,319 @@ ProxySQL_Node_Metrics * ProxySQL_Node_Entry::get_metrics_prev() { return m; } +/** + * @brief Processes checksum updates from a cluster peer and triggers synchronization when needed. + * + * This function is the core of ProxySQL's cluster monitoring and synchronization system. It processes + * checksum data received from peer nodes, compares it with local checksums, and initiates synchronization + * when differences are detected and thresholds are met. + * + * The function processes checksums for the following modules: + * MySQL modules: + * - admin_variables: ProxySQL admin configuration + * - mysql_query_rules: MySQL query routing rules + * - mysql_servers_v2: MySQL server configuration + * - runtime_mysql_servers: Runtime MySQL server status + * - mysql_users: MySQL user credentials + * - mysql_variables: MySQL server variables + * - ldap_variables: LDAP authentication settings + * - proxysql_servers: ProxySQL cluster node configuration + * + * PostgreSQL modules: + * - pgsql_query_rules: PostgreSQL query routing rules + * - pgsql_servers_v2: PostgreSQL server configuration + * - runtime_pgsql_servers: Runtime PostgreSQL server status + * - pgsql_users: PostgreSQL user credentials + * + * Synchronization Logic: + * 1. For each module, it compares local and peer checksums + * 2. If checksums differ, it checks the epoch timestamp to determine recency + * 3. If the peer is more recent and diff_check exceeds configured thresholds, sync is triggered + * 4. Conflict resolution handles cases where multiple nodes have the same epoch + * 5. Appropriate pull functions are called to fetch updated configuration + * + * Thresholds and Configuration: + * Each module has a corresponding *_diffs_before_sync variable that controls how many + * consecutive differences must be observed before triggering synchronization. This prevents + * excessive network traffic due to transient changes. + * + * @param _r MySQL result set containing checksum data from a peer node + * + * @note This function is thread-safe and requires GloVars.checksum_mutex to be held + * @note The function logs detailed information about checksum changes and synchronization decisions + * @note Metrics are updated to track successful and failed synchronization attempts + * @see ProxySQL_Cluster::pull_mysql_query_rules_from_peer() + * @see ProxySQL_Cluster::pull_pgsql_query_rules_from_peer() + * @see ProxySQL_Cluster::pull_pgsql_users_from_peer() + * @see ProxySQL_Cluster::pull_pgsql_servers_v2_from_peer() + * @see cluster_*_diffs_before_sync variables + */ +/** + * @brief Helper function to process checksum updates for cluster components + * + * @param row MySQL row containing checksum data (row[0] contains component name) + * @param checksum Reference to the node's checksum value + * @param global_checksum Reference to the global checksum value + * @param now Current timestamp + * @param diff_flag Flag indicating if sync should be delayed + * @param diff_sync_msg Message for when sync is disabled + * @param hostname Peer hostname for logging + * @param port Peer port for logging + */ +static void process_component_checksum( + MYSQL_ROW row, + ProxySQL_Checksum_Value_2& checksum, + ProxySQL_Checksum_Value& global_checksum, + time_t now, + bool diff_flag, + const char* diff_sync_msg, + const char* hostname, + int port +) { + checksum.version = atoll(row[1]); + checksum.epoch = atoll(row[2]); + checksum.last_updated = now; + + if (strcmp(checksum.checksum, row[3])) { + strcpy(checksum.checksum, row[3]); + checksum.last_changed = now; + checksum.diff_check = 1; + const char* no_sync_message = NULL; + + if (diff_flag) { + no_sync_message = "Not syncing yet ...\n"; + } else { + no_sync_message = diff_sync_msg; + } + + proxy_info( + "Cluster: detected a new checksum for %s from peer %s:%d, version %llu, epoch %llu, checksum %s . %s", + row[0], hostname, port, checksum.version, checksum.epoch, checksum.checksum, no_sync_message + ); + + if (strcmp(checksum.checksum, global_checksum.checksum) == 0) { + proxy_info( + "Cluster: checksum for %s from peer %s:%d matches with local checksum %s , we won't sync.\n", + row[0], hostname, port, global_checksum.checksum + ); + } + } else { + checksum.diff_check++; + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Checksum for %s from peer %s:%d, version %llu, epoch %llu, checksum %s is different from local checksum %s. Incremented diff_check %d ...\n", + row[0], hostname, port, checksum.version, checksum.epoch, checksum.checksum, global_checksum.checksum, checksum.diff_check); + } + + if (strcmp(checksum.checksum, global_checksum.checksum) == 0) { + checksum.diff_check = 0; + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Checksum for %s from peer %s:%d matches with local checksum %s, reset diff_check to 0.\n", + row[0], hostname, port, global_checksum.checksum); + } +} + + void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) { MYSQL_ROW row; time_t now = time(NULL); // Fetch the cluster_*_diffs_before_sync variables to ensure consistency at local scope - unsigned int diff_av = (unsigned int)__sync_fetch_and_add(&GloProxyCluster->cluster_admin_variables_diffs_before_sync,0); - unsigned int diff_mqr = (unsigned int)__sync_fetch_and_add(&GloProxyCluster->cluster_mysql_query_rules_diffs_before_sync,0); - unsigned int diff_ms = (unsigned int)__sync_fetch_and_add(&GloProxyCluster->cluster_mysql_servers_diffs_before_sync,0); - unsigned int diff_mu = (unsigned int)__sync_fetch_and_add(&GloProxyCluster->cluster_mysql_users_diffs_before_sync,0); - unsigned int diff_ps = (unsigned int)__sync_fetch_and_add(&GloProxyCluster->cluster_proxysql_servers_diffs_before_sync,0); - unsigned int diff_mv = (unsigned int)__sync_fetch_and_add(&GloProxyCluster->cluster_mysql_variables_diffs_before_sync,0); - unsigned int diff_lv = (unsigned int)__sync_fetch_and_add(&GloProxyCluster->cluster_ldap_variables_diffs_before_sync,0); + unsigned int diff_av = (unsigned int)GloProxyCluster->cluster_admin_variables_diffs_before_sync; + unsigned int diff_mqr = (unsigned int)GloProxyCluster->cluster_mysql_query_rules_diffs_before_sync; + unsigned int diff_ms = (unsigned int)GloProxyCluster->cluster_mysql_servers_diffs_before_sync; + unsigned int diff_mu = (unsigned int)GloProxyCluster->cluster_mysql_users_diffs_before_sync; + unsigned int diff_ps = (unsigned int)GloProxyCluster->cluster_proxysql_servers_diffs_before_sync; + unsigned int diff_mv = (unsigned int)GloProxyCluster->cluster_mysql_variables_diffs_before_sync; + unsigned int diff_lv = (unsigned int)GloProxyCluster->cluster_ldap_variables_diffs_before_sync; + unsigned int diff_pqr = (unsigned int)GloProxyCluster->cluster_pgsql_query_rules_diffs_before_sync; + unsigned int diff_ms_pgsql = (unsigned int)GloProxyCluster->cluster_pgsql_servers_diffs_before_sync; + unsigned int diff_mu_pgsql = (unsigned int)GloProxyCluster->cluster_pgsql_users_diffs_before_sync; + unsigned int diff_mv_pgsql = (unsigned int)GloProxyCluster->cluster_pgsql_variables_diffs_before_sync; pthread_mutex_lock(&GloVars.checksum_mutex); - while ( _r && (row = mysql_fetch_row(_r))) { - if (strcmp(row[0],"admin_variables")==0) { - ProxySQL_Checksum_Value_2& checksum = checksums_values.admin_variables; - ProxySQL_Checksum_Value& global_checksum = GloVars.checksums_values.admin_variables; - checksums_values.admin_variables.version = atoll(row[1]); - checksums_values.admin_variables.epoch = atoll(row[2]); - checksums_values.admin_variables.last_updated = now; - if (strcmp(checksums_values.admin_variables.checksum, row[3])) { - strcpy(checksums_values.admin_variables.checksum, row[3]); - checksums_values.admin_variables.last_changed = now; - checksums_values.admin_variables.diff_check = 1; - const char* no_sync_message = NULL; - - if (diff_av) { - no_sync_message = "Not syncing yet ...\n"; - } else { - no_sync_message = "Not syncing due to 'admin-cluster_admin_variables_diffs_before_sync=0'.\n"; - } + // Data-driven mapping of module names to their checksum fields and sync configuration + struct ChecksumModuleInfo { + const char* module_name; + ProxySQL_Checksum_Value_2* local_checksum; + ProxySQL_Checksum_Value* global_checksum; + std::atomic ProxySQL_Cluster::*diff_member; - proxy_info( - "Cluster: detected a new checksum for %s from peer %s:%d, version %llu, epoch %llu, checksum %s . %s", - row[0], hostname, port, checksum.version, checksum.epoch, checksum.checksum, no_sync_message - ); + // Sync decision fields (used only for modules that need special sync processing) + const char* sync_command; // "admin", "mysql", "ldap" for pull_global_variables_from_peer() + const char* load_runtime_command; // Command name for warning messages + int sync_conflict_counter; // Counter for epoch conflicts + int sync_delayed_counter; // Counter for version=1 delays - if (strcmp(checksum.checksum, global_checksum.checksum) == 0) { - proxy_info( - "Cluster: checksum for %s from peer %s:%d matches with local checksum %s , we won't sync.\n", - row[0], hostname, port, global_checksum.checksum - ); - } - } else { - checksums_values.admin_variables.diff_check++; - proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Checksum for admin_variables from peer %s:%d, version %llu, epoch %llu, checksum %s is different from local checksum %s. Incremented diff_check %d ...\n", hostname, port, checksums_values.admin_variables.version, checksums_values.admin_variables.epoch, - checksums_values.admin_variables.checksum, GloVars.checksums_values.admin_variables.checksum, checksums_values.admin_variables.diff_check); - } - if (strcmp(checksums_values.admin_variables.checksum, GloVars.checksums_values.admin_variables.checksum) == 0) { - checksums_values.admin_variables.diff_check = 0; - proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Checksum for admin_variables from peer %s:%d matches with local checksum %s, reset diff_check to 0.\n", hostname, port, GloVars.checksums_values.admin_variables.checksum); - } - continue; - } - if (strcmp(row[0],"mysql_query_rules")==0) { - ProxySQL_Checksum_Value_2& checksum = checksums_values.mysql_query_rules; - ProxySQL_Checksum_Value& global_checksum = GloVars.checksums_values.mysql_query_rules; - checksums_values.mysql_query_rules.version = atoll(row[1]); - checksums_values.mysql_query_rules.epoch = atoll(row[2]); - checksums_values.mysql_query_rules.last_updated = now; - if (strcmp(checksums_values.mysql_query_rules.checksum, row[3])) { - strcpy(checksums_values.mysql_query_rules.checksum, row[3]); - checksums_values.mysql_query_rules.last_changed = now; - checksums_values.mysql_query_rules.diff_check = 1; - const char* no_sync_message = NULL; - - if (diff_mqr) { - no_sync_message = "Not syncing yet ...\n"; - } else { - no_sync_message = "Not syncing due to 'admin-cluster_mysql_query_rules_diffs_before_sync=0'.\n"; - } + bool (*enabled_check)(); // Function to check if module is enabled (nullptr for always enabled) + }; - proxy_info( - "Cluster: detected a new checksum for %s from peer %s:%d, version %llu, epoch %llu, checksum %s . %s", - row[0], hostname, port, checksum.version, checksum.epoch, checksum.checksum, no_sync_message - ); + // Initialize all supported modules with their respective checksum field pointers + ChecksumModuleInfo modules[] = { + {"admin_variables", &checksums_values.admin_variables, &GloVars.checksums_values.admin_variables, &ProxySQL_Cluster::cluster_admin_variables_diffs_before_sync, + "admin", "LOAD ADMIN VARIABLES TO RUNTIME", + static_cast(p_cluster_counter::sync_conflict_admin_variables_share_epoch), + static_cast(p_cluster_counter::sync_delayed_admin_variables_version_one), nullptr}, + {"mysql_query_rules", &checksums_values.mysql_query_rules, &GloVars.checksums_values.mysql_query_rules, &ProxySQL_Cluster::cluster_mysql_query_rules_diffs_before_sync, + nullptr, nullptr, 0, 0, nullptr}, + {"mysql_servers", &checksums_values.mysql_servers, &GloVars.checksums_values.mysql_servers, &ProxySQL_Cluster::cluster_mysql_servers_diffs_before_sync, + nullptr, nullptr, 0, 0, nullptr}, + {"mysql_servers_v2", &checksums_values.mysql_servers_v2, &GloVars.checksums_values.mysql_servers_v2, &ProxySQL_Cluster::cluster_mysql_servers_diffs_before_sync, + nullptr, nullptr, 0, 0, nullptr}, + {"mysql_users", &checksums_values.mysql_users, &GloVars.checksums_values.mysql_users, &ProxySQL_Cluster::cluster_mysql_users_diffs_before_sync, + nullptr, nullptr, 0, 0, nullptr}, + {"mysql_variables", &checksums_values.mysql_variables, &GloVars.checksums_values.mysql_variables, &ProxySQL_Cluster::cluster_mysql_variables_diffs_before_sync, + "mysql", "LOAD MYSQL VARIABLES TO RUNTIME", + static_cast(p_cluster_counter::sync_conflict_mysql_variables_share_epoch), + static_cast(p_cluster_counter::sync_delayed_mysql_variables_version_one), nullptr}, + {"proxysql_servers", &checksums_values.proxysql_servers, &GloVars.checksums_values.proxysql_servers, &ProxySQL_Cluster::cluster_proxysql_servers_diffs_before_sync, + nullptr, nullptr, 0, 0, nullptr}, + {"ldap_variables", &checksums_values.ldap_variables, &GloVars.checksums_values.ldap_variables, &ProxySQL_Cluster::cluster_ldap_variables_diffs_before_sync, + "ldap", "LOAD LDAP VARIABLES TO RUNTIME", + static_cast(p_cluster_counter::sync_conflict_ldap_variables_share_epoch), + static_cast(p_cluster_counter::sync_delayed_ldap_variables_version_one), + []() { return GloMyLdapAuth != nullptr; }}, + {"pgsql_query_rules", &checksums_values.pgsql_query_rules, &GloVars.checksums_values.pgsql_query_rules, &ProxySQL_Cluster::cluster_pgsql_query_rules_diffs_before_sync, + nullptr, nullptr, 0, 0, nullptr}, + {"pgsql_servers", &checksums_values.pgsql_servers, &GloVars.checksums_values.pgsql_servers, &ProxySQL_Cluster::cluster_pgsql_servers_diffs_before_sync, + nullptr, nullptr, 0, 0, nullptr}, + {"pgsql_servers_v2", &checksums_values.pgsql_servers_v2, &GloVars.checksums_values.pgsql_servers_v2, &ProxySQL_Cluster::cluster_pgsql_servers_diffs_before_sync, + nullptr, nullptr, 0, 0, nullptr}, + {"pgsql_users", &checksums_values.pgsql_users, &GloVars.checksums_values.pgsql_users, &ProxySQL_Cluster::cluster_pgsql_users_diffs_before_sync, + nullptr, nullptr, 0, 0, nullptr}, + {"pgsql_variables", &checksums_values.pgsql_variables, &GloVars.checksums_values.pgsql_variables, &ProxySQL_Cluster::cluster_pgsql_variables_diffs_before_sync, + "pgsql", "LOAD PGSQL VARIABLES TO RUNTIME", + static_cast(p_cluster_counter::sync_conflict_pgsql_variables_share_epoch), + static_cast(p_cluster_counter::sync_delayed_pgsql_variables_version_one), nullptr} + }; - if (strcmp(checksum.checksum, global_checksum.checksum) == 0) { - proxy_info( - "Cluster: checksum for %s from peer %s:%d matches with local checksum %s , we won't sync.\n", - row[0], hostname, port, global_checksum.checksum - ); + while ( _r && (row = mysql_fetch_row(_r))) { + // Data-driven approach: find the matching module and process it + bool module_found = false; + + // Search for the module in our data structure and check if it's enabled + for (const auto& module : modules) { + if (strcmp(row[0], module.module_name) == 0) { + // Skip module if not enabled (for modules with optional dependencies like LDAP) + if (module.enabled_check && !module.enabled_check()) { + module_found = true; + break; } - } else { - checksums_values.mysql_query_rules.diff_check++; - proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Checksum for mysql_query_rules from peer %s:%d, version %llu, epoch %llu, checksum %s is different from local checksum %s. Incremented diff_check %d ...\n", hostname, port, checksums_values.mysql_query_rules.version, checksums_values.mysql_query_rules.epoch, - checksums_values.mysql_query_rules.checksum, GloVars.checksums_values.mysql_query_rules.checksum, checksums_values.mysql_query_rules.diff_check); - } - if (strcmp(checksums_values.mysql_query_rules.checksum, GloVars.checksums_values.mysql_query_rules.checksum) == 0) { - checksums_values.mysql_query_rules.diff_check = 0; - proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Checksum for mysql_query_rules from peer %s:%d matches with local checksum %s, reset diff_check to 0.\n", hostname, port, GloVars.checksums_values.mysql_query_rules.checksum); + module_found = true; + break; } - continue; } - if (strcmp(row[0],"mysql_servers")==0) { - ProxySQL_Checksum_Value_2& checksum = checksums_values.mysql_servers; - ProxySQL_Checksum_Value& global_checksum = GloVars.checksums_values.mysql_servers; - checksums_values.mysql_servers.version = atoll(row[1]); - checksums_values.mysql_servers.epoch = atoll(row[2]); - checksums_values.mysql_servers.last_updated = now; - if (strcmp(checksums_values.mysql_servers.checksum, row[3])) { - strcpy(checksums_values.mysql_servers.checksum, row[3]); - checksums_values.mysql_servers.last_changed = now; - checksums_values.mysql_servers.diff_check = 1; - const char* no_sync_message = NULL; - - if (diff_ms) { - no_sync_message = "Not syncing yet ...\n"; - } else { - no_sync_message = "Not syncing due to 'admin-cluster_mysql_servers_diffs_before_sync=0'.\n"; - } - - proxy_info( - "Cluster: detected a new checksum for %s from peer %s:%d, version %llu, epoch %llu, checksum %s . %s", - row[0], hostname, port, checksum.version, checksum.epoch, checksum.checksum, no_sync_message - ); - if (strcmp(checksum.checksum, global_checksum.checksum) == 0) { - proxy_info( - "Cluster: checksum for %s from peer %s:%d matches with local checksum %s , we won't sync.\n", - row[0], hostname, port, global_checksum.checksum + if (module_found) { + // Find the module and get its diff threshold + for (const auto& module : modules) { + if (strcmp(row[0], module.module_name) == 0) { + // Get diff threshold using member pointer with atomic load + unsigned int diff_threshold = (unsigned int)(GloProxyCluster->*(module.diff_member)).load(); + + // Generate generalized sync message + char sync_msg[256]; + snprintf(sync_msg, sizeof(sync_msg), + "Not syncing due to 'admin-cluster_%s_diffs_before_sync=0'.\n", + module.module_name); + + process_component_checksum( + row, + *module.local_checksum, + *module.global_checksum, + now, diff_threshold, + sync_msg, + hostname, port ); + break; } - } else { - checksums_values.mysql_servers.diff_check++; - proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Checksum for mysql_servers from peer %s:%d, version %llu, epoch %llu, checksum %s is different from local checksum %s. Incremented diff_check %d ...\n", hostname, port, checksums_values.mysql_servers.version, checksums_values.mysql_servers.epoch, - checksums_values.mysql_servers.checksum, GloVars.checksums_values.mysql_servers.checksum, checksums_values.mysql_servers.diff_check); - } - if (strcmp(checksums_values.mysql_servers.checksum, GloVars.checksums_values.mysql_servers.checksum) == 0) { - checksums_values.mysql_servers.diff_check = 0; - proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Checksum for mysql_servers from peer %s:%d matches with local checksum %s, reset diff_check to 0.\n", hostname, port, GloVars.checksums_values.mysql_servers.checksum); } - continue; } - if (strcmp(row[0], "mysql_servers_v2")==0) { - ProxySQL_Checksum_Value_2& checksum = checksums_values.mysql_servers_v2; - ProxySQL_Checksum_Value& global_checksum = GloVars.checksums_values.mysql_servers_v2; - checksums_values.mysql_servers_v2.version = atoll(row[1]); - checksums_values.mysql_servers_v2.epoch = atoll(row[2]); - checksums_values.mysql_servers_v2.last_updated = now; - if (strcmp(checksums_values.mysql_servers_v2.checksum, row[3])) { - strcpy(checksums_values.mysql_servers_v2.checksum, row[3]); - checksums_values.mysql_servers_v2.last_changed = now; - checksums_values.mysql_servers_v2.diff_check = 1; - const char* no_sync_message = NULL; - - if (diff_ms) { - no_sync_message = "Not syncing yet ...\n"; - } else { - no_sync_message = "Not syncing due to 'admin-cluster_mysql_servers_diffs_before_sync=0'.\n"; + } + if (_r == NULL) { + // Update diff_check counters for all modules using data-driven approach + size_t module_count = sizeof(modules) / sizeof(modules[0]); + for (size_t i = 0; i < module_count; i++) { + ProxySQL_Checksum_Value_2* local_v = modules[i].local_checksum; + ProxySQL_Checksum_Value* global_v = modules[i].global_checksum; + + if (local_v && global_v) { + local_v->last_updated = now; + if (strcmp(local_v->checksum, global_v->checksum) == 0) { + local_v->diff_check = 0; } - - proxy_info( - "Cluster: detected a new checksum for %s from peer %s:%d, version %llu, epoch %llu, checksum %s . %s", - row[0], hostname, port, checksum.version, checksum.epoch, checksum.checksum, no_sync_message - ); - - if (strcmp(checksum.checksum, global_checksum.checksum) == 0) { - proxy_info( - "Cluster: checksum for %s from peer %s:%d matches with local checksum %s , we won't sync.\n", - row[0], hostname, port, global_checksum.checksum - ); + if (local_v->diff_check) { + local_v->diff_check++; } - } else { - checksums_values.mysql_servers_v2.diff_check++; - proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Checksum for mysql_servers_v2 from peer %s:%d, version %llu, epoch %llu, checksum %s is different from local checksum %s. Incremented diff_check %d ...\n", hostname, port, checksums_values.mysql_servers_v2.version, checksums_values.mysql_servers_v2.epoch, - checksums_values.mysql_servers_v2.checksum, GloVars.checksums_values.mysql_servers_v2.checksum, checksums_values.mysql_servers_v2.diff_check); } - if (strcmp(checksums_values.mysql_servers_v2.checksum, GloVars.checksums_values.mysql_servers_v2.checksum) == 0) { - checksums_values.mysql_servers_v2.diff_check = 0; - proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Checksum for mysql_servers_v2 from peer %s:%d matches with local checksum %s, reset diff_check to 0.\n", hostname, port, GloVars.checksums_values.mysql_servers.checksum); - } - continue; } - if (strcmp(row[0],"mysql_users")==0) { - ProxySQL_Checksum_Value_2& checksum = checksums_values.mysql_users; - ProxySQL_Checksum_Value& global_checksum = GloVars.checksums_values.mysql_users; - checksums_values.mysql_users.version = atoll(row[1]); - checksums_values.mysql_users.epoch = atoll(row[2]); - checksums_values.mysql_users.last_updated = now; - if (strcmp(checksums_values.mysql_users.checksum, row[3])) { - strcpy(checksums_values.mysql_users.checksum, row[3]); - checksums_values.mysql_users.last_changed = now; - checksums_values.mysql_users.diff_check = 1; - const char* no_sync_message = NULL; - - if (diff_mu) { - no_sync_message = "Not syncing yet ...\n"; - } else { - no_sync_message = "Not syncing due to 'admin-cluster_mysql_users_diffs_before_sync=0'.\n"; - } + } + pthread_mutex_unlock(&GloVars.checksum_mutex); + // we now do a series of checks, and we take action + // note that this is done outside the critical section + // as mutex on GloVars.checksum_mutex is already released - proxy_info( - "Cluster: detected a new checksum for %s from peer %s:%d, version %llu, epoch %llu, checksum %s . %s", - row[0], hostname, port, checksum.version, checksum.epoch, checksum.checksum, no_sync_message - ); + // Set of modules that need special sync decision processing (admin_variables, mysql_variables, ldap_variables, pgsql_variables) + const std::unordered_set sync_enabled_modules = { + "admin_variables", + "mysql_variables", + "ldap_variables", + "pgsql_variables" + }; - if (strcmp(checksum.checksum, global_checksum.checksum) == 0) { - proxy_info( - "Cluster: checksum for %s from peer %s:%d matches with local checksum %s , we won't sync.\n", - row[0], hostname, port, global_checksum.checksum - ); - } - } else { - checksums_values.mysql_users.diff_check++; - proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Checksum for mysql_users from peer %s:%d, version %llu, epoch %llu, checksum %s is different from local checksum %s. Incremented diff_check %d ...\n", hostname, port, checksums_values.mysql_users.version, checksums_values.mysql_users.epoch, - checksums_values.mysql_users.checksum, GloVars.checksums_values.mysql_users.checksum, checksums_values.mysql_users.diff_check); - } - if (strcmp(checksums_values.mysql_users.checksum, GloVars.checksums_values.mysql_users.checksum) == 0) { - checksums_values.mysql_users.diff_check = 0; - proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Checksum for mysql_users from peer %s:%d matches with local checksum %s, reset diff_check to 0.\n", hostname, port, GloVars.checksums_values.mysql_users.checksum); - } + // Process sync decisions for modules that need special sync processing + for (const auto& module : modules) { + // Only process modules that are in the sync_enabled_modules set + if (sync_enabled_modules.find(module.module_name) == sync_enabled_modules.end()) { continue; } - if (strcmp(row[0],"mysql_variables")==0) { - ProxySQL_Checksum_Value_2& checksum = checksums_values.mysql_variables; - ProxySQL_Checksum_Value& global_checksum = GloVars.checksums_values.mysql_variables; - checksums_values.mysql_variables.version = atoll(row[1]); - checksums_values.mysql_variables.epoch = atoll(row[2]); - checksums_values.mysql_variables.last_updated = now; - if (strcmp(checksums_values.mysql_variables.checksum, row[3])) { - strcpy(checksums_values.mysql_variables.checksum, row[3]); - checksums_values.mysql_variables.last_changed = now; - checksums_values.mysql_variables.diff_check = 1; - const char* no_sync_message = NULL; - - if (diff_mv) { - no_sync_message = "Not syncing yet ...\n"; - } else { - no_sync_message = "Not syncing due to 'admin-cluster_mysql_variables_diffs_before_sync=0'.\n"; - } - - proxy_info( - "Cluster: detected a new checksum for %s from peer %s:%d, version %llu, epoch %llu, checksum %s . %s", - row[0], hostname, port, checksum.version, checksum.epoch, checksum.checksum, no_sync_message - ); - if (strcmp(checksum.checksum, global_checksum.checksum) == 0) { - proxy_info( - "Cluster: checksum for %s from peer %s:%d matches with local checksum %s , we won't sync.\n", - row[0], hostname, port, global_checksum.checksum - ); - } - } else { - checksums_values.mysql_variables.diff_check++; - proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Checksum for mysql_variables from peer %s:%d, version %llu, epoch %llu, checksum %s is different from local checksum %s. Incremented diff_check %d ...\n", hostname, port, checksums_values.mysql_variables.version, checksums_values.mysql_variables.epoch, - checksums_values.mysql_variables.checksum, GloVars.checksums_values.mysql_variables.checksum, checksums_values.mysql_variables.diff_check); - } - if (strcmp(checksums_values.mysql_variables.checksum, GloVars.checksums_values.mysql_variables.checksum) == 0) { - checksums_values.mysql_variables.diff_check = 0; - proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Checksum for mysql_variables from peer %s:%d matches with local checksum %s, reset diff_check to 0.\n", hostname, port, GloVars.checksums_values.mysql_variables.checksum); - } + // Skip module if not enabled (for modules with optional dependencies like LDAP) + if (module.enabled_check && !module.enabled_check()) { continue; } - if (strcmp(row[0],"proxysql_servers")==0) { - ProxySQL_Checksum_Value_2& checksum = checksums_values.proxysql_servers; - ProxySQL_Checksum_Value& global_checksum = GloVars.checksums_values.proxysql_servers; - checksums_values.proxysql_servers.version = atoll(row[1]); - checksums_values.proxysql_servers.epoch = atoll(row[2]); - checksums_values.proxysql_servers.last_updated = now; - if (strcmp(checksums_values.proxysql_servers.checksum, row[3])) { - strcpy(checksums_values.proxysql_servers.checksum, row[3]); - checksums_values.proxysql_servers.last_changed = now; - checksums_values.proxysql_servers.diff_check = 1; - const char* no_sync_message = NULL; - - if (diff_ps) { - no_sync_message = "Not syncing yet ...\n"; - } else { - no_sync_message = "Not syncing due to 'admin-cluster_proxysql_servers_diffs_before_sync=0'.\n"; - } - - proxy_info( - "Cluster: detected a new checksum for %s from peer %s:%d, version %llu, epoch %llu, checksum %s . %s", - row[0], hostname, port, checksum.version, checksum.epoch, checksum.checksum, no_sync_message - ); - if (strcmp(checksum.checksum, global_checksum.checksum) == 0) { - proxy_info( - "Cluster: checksum for %s from peer %s:%d matches with local checksum %s , we won't sync.\n", - row[0], hostname, port, global_checksum.checksum - ); - } - } else { - checksums_values.proxysql_servers.diff_check++; - proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Checksum for proxysql_servers from peer %s:%d, version %llu, epoch %llu, checksum %s is different from local checksum %s. Incremented diff_check %d ...\n", hostname, port, checksums_values.proxysql_servers.version, checksums_values.proxysql_servers.epoch, - checksums_values.proxysql_servers.checksum, GloVars.checksums_values.proxysql_servers.checksum, checksums_values.proxysql_servers.diff_check); - } - if (strcmp(checksums_values.proxysql_servers.checksum, GloVars.checksums_values.proxysql_servers.checksum) == 0) { - checksums_values.proxysql_servers.diff_check = 0; - proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Checksum for proxysql_servers from peer %s:%d matches with local checksum %s, reset diff_check to 0.\n", hostname, port, GloVars.checksums_values.proxysql_servers.checksum); - } + // Skip modules that don't have sync configuration (those with nullptr sync_command) + if (!module.sync_command) { continue; } - if (GloMyLdapAuth && strcmp(row[0],"ldap_variables")==0) { - ProxySQL_Checksum_Value_2& checksum = checksums_values.ldap_variables; - ProxySQL_Checksum_Value& global_checksum = GloVars.checksums_values.ldap_variables; - checksums_values.ldap_variables.version = atoll(row[1]); - checksums_values.ldap_variables.epoch = atoll(row[2]); - checksums_values.ldap_variables.last_updated = now; - if (strcmp(checksums_values.ldap_variables.checksum, row[3])) { - strcpy(checksums_values.ldap_variables.checksum, row[3]); - checksums_values.ldap_variables.last_changed = now; - checksums_values.ldap_variables.diff_check = 1; - const char* no_sync_message = NULL; - - if (diff_lv) { - no_sync_message = "Not syncing yet ...\n"; - } else { - no_sync_message = "Not syncing due to 'admin-cluster_ldap_variables_diffs_before_sync=0'.\n"; - } - proxy_info( - "Cluster: detected a new checksum for %s from peer %s:%d, version %llu, epoch %llu, checksum %s . %s", - row[0], hostname, port, checksum.version, checksum.epoch, checksum.checksum, no_sync_message - ); + // Get diff threshold using member pointer with atomic load + unsigned int diff_threshold = (unsigned int)(GloProxyCluster->*(module.diff_member)).load(); - if (strcmp(checksum.checksum, global_checksum.checksum) == 0) { - proxy_info( - "Cluster: checksum for %s from peer %s:%d matches with local checksum %s , we won't sync.\n", - row[0], hostname, port, global_checksum.checksum - ); + if (diff_threshold > 0) { + ProxySQL_Checksum_Value_2 *v = module.local_checksum; + unsigned long long own_version = __sync_fetch_and_add(&module.global_checksum->version, 0); + unsigned long long own_epoch = __sync_fetch_and_add(&module.global_checksum->epoch, 0); + char* own_checksum = __sync_fetch_and_add(&module.global_checksum->checksum, 0); + const string expected_checksum { v->checksum }; + + if (v->version > 1) { + if ( + (own_version == 1) // we just booted + || + (v->epoch > own_epoch) // epoch is newer + ) { + if (v->diff_check >= diff_threshold) { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with %s version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, module.module_name, v->version, v->epoch, v->diff_check, own_version, own_epoch); + proxy_info("Cluster: detected a peer %s:%d with %s version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, module.module_name, v->version, v->epoch, v->diff_check, own_version, own_epoch); + GloProxyCluster->pull_global_variables_from_peer(module.sync_command, expected_checksum, v->epoch); + } + } + if ((v->epoch == own_epoch) && v->diff_check && ((v->diff_check % (diff_threshold*10)) == 0)) { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with %s version %llu, epoch %llu, diff_check %u, checksum %s. Own version: %llu, epoch: %llu, checksum %s. Sync conflict, epoch times are EQUAL, can't determine which server holds the latest config, we won't sync. This message will be repeated every %u checks until %s is executed on candidate master.\n", hostname, port, module.module_name, v->version, v->epoch, v->diff_check, v->checksum, own_version, own_epoch, own_checksum, (diff_threshold * 10), module.load_runtime_command); + proxy_error("Cluster: detected a peer %s:%d with %s version %llu, epoch %llu, diff_check %u, checksum %s. Own version: %llu, epoch: %llu, checksum %s. Sync conflict, epoch times are EQUAL, can't determine which server holds the latest config, we won't sync. This message will be repeated every %u checks until %s is executed on candidate master.\n", hostname, port, module.module_name, v->version, v->epoch, v->diff_check, v->checksum, own_version, own_epoch, own_checksum, (diff_threshold*10), module.load_runtime_command); + GloProxyCluster->metrics.p_counter_array[module.sync_conflict_counter]->Increment(); } } else { - checksums_values.ldap_variables.diff_check++; - proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Checksum for ldap_variables from peer %s:%d, version %llu, epoch %llu, checksum %s is different from local checksum %s. Incremented diff_check %d ...\n", hostname, port, checksums_values.ldap_variables.version, checksums_values.ldap_variables.epoch, - checksums_values.ldap_variables.checksum, GloVars.checksums_values.ldap_variables.checksum, checksums_values.ldap_variables.diff_check); - } - if (strcmp(checksums_values.ldap_variables.checksum, GloVars.checksums_values.ldap_variables.checksum) == 0) { - checksums_values.ldap_variables.diff_check = 0; - proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Checksum for ldap_variables from peer %s:%d matches with local checksum %s, reset diff_check to 0.\n", hostname, port, GloVars.checksums_values.ldap_variables.checksum); - } - continue; - } - } - if (_r == NULL) { - ProxySQL_Checksum_Value_2 *v = NULL; - v = &checksums_values.admin_variables; - v->last_updated = now; - if (strcmp(v->checksum, GloVars.checksums_values.admin_variables.checksum) == 0) { - v->diff_check = 0; - } - if (v->diff_check) - v->diff_check++; - v = &checksums_values.mysql_query_rules; - v->last_updated = now; - if (strcmp(v->checksum, GloVars.checksums_values.mysql_query_rules.checksum) == 0) { - v->diff_check = 0; - } - if (v->diff_check) - v->diff_check++; - v = &checksums_values.mysql_servers; - v->last_updated = now; - if (strcmp(v->checksum, GloVars.checksums_values.mysql_servers.checksum) == 0) { - v->diff_check = 0; - } - if (v->diff_check) - v->diff_check++; - v = &checksums_values.mysql_servers_v2; - v->last_updated = now; - if (strcmp(v->checksum, GloVars.checksums_values.mysql_servers_v2.checksum) == 0) { - v->diff_check = 0; - } - if (v->diff_check) - v->diff_check++; - v = &checksums_values.mysql_users; - v->last_updated = now; - if (strcmp(v->checksum, GloVars.checksums_values.mysql_users.checksum) == 0) { - v->diff_check = 0; - } - if (v->diff_check) - v->diff_check++; - v = &checksums_values.mysql_variables; - v->last_updated = now; - if (strcmp(v->checksum, GloVars.checksums_values.mysql_variables.checksum) == 0) { - v->diff_check = 0; - } - if (v->diff_check) - v->diff_check++; - v = &checksums_values.proxysql_servers; - v->last_updated = now; - if (strcmp(v->checksum, GloVars.checksums_values.proxysql_servers.checksum) == 0) { - v->diff_check = 0; - } - if (v->diff_check) - v->diff_check++; - v = &checksums_values.ldap_variables; - v->last_updated = now; - if (strcmp(v->checksum, GloVars.checksums_values.ldap_variables.checksum) == 0) { - v->diff_check = 0; - } - if (v->diff_check) - v->diff_check++; - } - pthread_mutex_unlock(&GloVars.checksum_mutex); - // we now do a series of checks, and we take action - // note that this is done outside the critical section - // as mutex on GloVars.checksum_mutex is already released - ProxySQL_Checksum_Value_2 *v = NULL; - if (diff_av) { - v = &checksums_values.admin_variables; - unsigned long long own_version = __sync_fetch_and_add(&GloVars.checksums_values.admin_variables.version, 0); - unsigned long long own_epoch = __sync_fetch_and_add(&GloVars.checksums_values.admin_variables.epoch, 0); - char* own_checksum = __sync_fetch_and_add(&GloVars.checksums_values.admin_variables.checksum, 0); - const string expected_checksum { v->checksum }; - - if (v->version > 1) { - if ( - (own_version == 1) // we just booted - || - (v->epoch > own_epoch) // epoch is newer - ) { - if (v->diff_check >= diff_av) { - proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with admin_variables version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); - proxy_info("Cluster: detected a peer %s:%d with admin_variables version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); - GloProxyCluster->pull_global_variables_from_peer("admin", expected_checksum, v->epoch); + if (v->diff_check && (v->diff_check % (diff_threshold*10)) == 0) { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with %s version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. diff_check is increasing, but version 1 doesn't allow sync. This message will be repeated every %u checks until %s is executed on candidate master.\n", hostname, port, module.module_name, v->version, v->epoch, v->diff_check, own_version, own_epoch, (diff_threshold * 10), module.load_runtime_command); + proxy_warning("Cluster: detected a peer %s:%d with %s version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. diff_check is increasing, but version 1 doesn't allow sync. This message will be repeated every %u checks until %s is executed on candidate master.\n", hostname, port, module.module_name, v->version, v->epoch, v->diff_check, own_version, own_epoch, (diff_threshold*10), module.load_runtime_command); + GloProxyCluster->metrics.p_counter_array[module.sync_delayed_counter]->Increment(); } } - if ((v->epoch == own_epoch) && v->diff_check && ((v->diff_check % (diff_av*10)) == 0)) { - proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with admin_variables version %llu, epoch %llu, diff_check %u, checksum %s. Own version: %llu, epoch: %llu, checksum %s. Sync conflict, epoch times are EQUAL, can't determine which server holds the latest config, we won't sync. This message will be repeated every %u checks until LOAD ADMIN VARIABLES TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, v->checksum, own_version, own_epoch, own_checksum, (diff_av * 10)); - proxy_error("Cluster: detected a peer %s:%d with admin_variables version %llu, epoch %llu, diff_check %u, checksum %s. Own version: %llu, epoch: %llu, checksum %s. Sync conflict, epoch times are EQUAL, can't determine which server holds the latest config, we won't sync. This message will be repeated every %u checks until LOAD ADMIN VARIABLES TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, v->checksum, own_version, own_epoch, own_checksum, (diff_av*10)); - GloProxyCluster->metrics.p_counter_array[p_cluster_counter::sync_conflict_admin_variables_share_epoch]->Increment(); - } - } else { - if (v->diff_check && (v->diff_check % (diff_av*10)) == 0) { - proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with admin_variables version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. diff_check is increasing, but version 1 doesn't allow sync. This message will be repeated every %u checks until LOAD ADMIN VARIABLES TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch, (diff_av * 10)); - proxy_warning("Cluster: detected a peer %s:%d with admin_variables version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. diff_check is increasing, but version 1 doesn't allow sync. This message will be repeated every %u checks until LOAD ADMIN VARIABLES TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch, (diff_av*10)); - GloProxyCluster->metrics.p_counter_array[p_cluster_counter::sync_delayed_admin_variables_version_one]->Increment(); - } } } + + ProxySQL_Checksum_Value_2 *v = NULL; if (diff_mqr) { unsigned long long own_version = __sync_fetch_and_add(&GloVars.checksums_values.mysql_query_rules.version,0); unsigned long long own_epoch = __sync_fetch_and_add(&GloVars.checksums_values.mysql_query_rules.epoch,0); @@ -982,12 +868,52 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) { } } } - if (diff_mv) { - v = &checksums_values.mysql_variables; - unsigned long long own_version = __sync_fetch_and_add(&GloVars.checksums_values.mysql_variables.version, 0); - unsigned long long own_epoch = __sync_fetch_and_add(&GloVars.checksums_values.mysql_variables.epoch, 0); - char* own_checksum = __sync_fetch_and_add(&GloVars.checksums_values.mysql_variables.checksum, 0); - const string expected_checksum { v->checksum }; + // IMPORTANT-NOTE: This action should ALWAYS be performed the last, since the 'checksums_values' gets + // invalidated by 'pull_proxysql_servers_from_peer' and further memory accesses would be invalid. + if (diff_ps) { + v = &checksums_values.proxysql_servers; + unsigned long long own_version = __sync_fetch_and_add(&GloVars.checksums_values.proxysql_servers.version,0); + unsigned long long own_epoch = __sync_fetch_and_add(&GloVars.checksums_values.proxysql_servers.epoch,0); + char* own_checksum = __sync_fetch_and_add(&GloVars.checksums_values.proxysql_servers.checksum,0); + if (v->version > 1) { + // NOTE: Backup values: 'v' gets invalidated by 'pull_proxysql_servers_from_peer()' + unsigned long long v_epoch = v->epoch; + unsigned long long v_version = v->version; + unsigned int v_diff_check = v->diff_check; + const string v_exp_checksum { v->checksum }; + + if ( + (own_version == 1) // we just booted + || + (v->epoch > own_epoch) // epoch is newer + ) { + if (v->diff_check >= diff_ps) { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with proxysql_servers version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); + proxy_info("Cluster: detected a peer %s:%d with proxysql_servers version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); + GloProxyCluster->pull_proxysql_servers_from_peer(v_exp_checksum, v->epoch); + } + } + if ((v_epoch == own_epoch) && v_diff_check && ((v_diff_check % (diff_ps*10)) == 0)) { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with proxysql_servers version %llu, epoch %llu, diff_check %u, checksum %s. Own version: %llu, epoch: %llu, checksum %s. Sync conflict, epoch times are EQUAL, can't determine which server holds the latest config, we won't sync. This message will be repeated every %u checks until LOAD MYSQL SERVERS TO RUNTIME is executed on candidate master.\n", hostname, port, v_version, v_epoch, v_diff_check, v->checksum, own_version, own_epoch, own_checksum, (diff_ps * 10)); + proxy_error("Cluster: detected a peer %s:%d with proxysql_servers version %llu, epoch %llu, diff_check %u, checksum %s. Own version: %llu, epoch: %llu, checksum %s. Sync conflict, epoch times are EQUAL, can't determine which server holds the latest config, we won't sync. This message will be repeated every %u checks until LOAD MYSQL SERVERS TO RUNTIME is executed on candidate master.\n", hostname, port, v_version, v_epoch, v_diff_check, v->checksum, own_version, own_epoch, own_checksum, (diff_ps*10)); + GloProxyCluster->metrics.p_counter_array[p_cluster_counter::sync_conflict_proxysql_servers_share_epoch]->Increment(); + } + } else { + if (v->diff_check && (v->diff_check % (diff_ps*10)) == 0) { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with proxysql_servers version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. diff_check is increasing, but version 1 doesn't allow sync. This message will be repeated every %u checks until LOAD PROXYSQL SERVERS TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch, (diff_ps * 10)); + proxy_warning("Cluster: detected a peer %s:%d with proxysql_servers version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. diff_check is increasing, but version 1 doesn't allow sync. This message will be repeated every %u checks until LOAD PROXYSQL SERVERS TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch, (diff_ps*10)); + GloProxyCluster->metrics.p_counter_array[p_cluster_counter::sync_delayed_proxysql_servers_version_one]->Increment(); + } + } + } + + // PostgreSQL Query Rules Sync + if (diff_pqr) { + v = &checksums_values.pgsql_query_rules; + unsigned long long own_version = __sync_fetch_and_add(&GloVars.checksums_values.pgsql_query_rules.version,0); + unsigned long long own_epoch = __sync_fetch_and_add(&GloVars.checksums_values.pgsql_query_rules.epoch,0); + char* own_checksum = __sync_fetch_and_add(&GloVars.checksums_values.pgsql_query_rules.checksum,0); + const std::string v_exp_checksum { v->checksum }; if (v->version > 1) { if ( @@ -995,31 +921,31 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) { || (v->epoch > own_epoch) // epoch is newer ) { - if (v->diff_check >= diff_mv) { - proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with mysql_variables version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); - proxy_info("Cluster: detected a peer %s:%d with mysql_variables version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); - GloProxyCluster->pull_global_variables_from_peer("mysql", expected_checksum, v->epoch); + if (v->diff_check >= diff_pqr) { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with pgsql_query_rules version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); + proxy_info("Cluster: detected a peer %s:%d with pgsql_query_rules version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); + GloProxyCluster->pull_pgsql_query_rules_from_peer(v_exp_checksum, v->epoch); } } - if ((v->epoch == own_epoch) && v->diff_check && ((v->diff_check % (diff_mv*10)) == 0)) { - proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with mysql_variables version %llu, epoch %llu, diff_check %u, checksum %s. Own version: %llu, epoch: %llu, checksum %s. Sync conflict, epoch times are EQUAL, can't determine which server holds the latest config, we won't sync. This message will be repeated every %u checks until LOAD MYSQL VARIABLES TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, v->checksum, own_version, own_epoch, own_checksum, (diff_mv * 10)); - proxy_error("Cluster: detected a peer %s:%d with mysql_variables version %llu, epoch %llu, diff_check %u, checksum %s. Own version: %llu, epoch: %llu, checksum %s. Sync conflict, epoch times are EQUAL, can't determine which server holds the latest config, we won't sync. This message will be repeated every %u checks until LOAD MYSQL VARIABLES TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, v->checksum, own_version, own_epoch, own_checksum, (diff_mv*10)); - GloProxyCluster->metrics.p_counter_array[p_cluster_counter::sync_conflict_mysql_variables_share_epoch]->Increment(); + if ((v->epoch == own_epoch) && v->diff_check && ((v->diff_check % (diff_pqr*10)) == 0)) { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with pgsql_query_rules version %llu, epoch %llu, diff_check %u, checksum %s. Own version: %llu, epoch: %llu, checksum %s. Sync conflict, epoch times are EQUAL, can't determine which server holds the latest config, we won't sync. This message will be repeated every %u checks until LOAD PGSQL QUERY RULES TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, v->checksum, own_version, own_epoch, own_checksum, (diff_pqr * 10)); + proxy_error("Cluster: detected a peer %s:%d with pgsql_query_rules version %llu, epoch %llu, diff_check %u, checksum %s. Own version: %llu, epoch: %llu, checksum %s. Sync conflict, epoch times are EQUAL, can't determine which server holds the latest config, we won't sync. This message will be repeated every %u checks until LOAD PGSQL QUERY RULES TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, v->checksum, own_version, own_epoch, own_checksum, (diff_pqr*10)); } } else { - if (v->diff_check && (v->diff_check % (diff_mv*10)) == 0) { - proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with mysql_variables version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. diff_check is increasing, but version 1 doesn't allow sync. This message will be repeated every %u checks until LOAD MYSQL VARIABLES TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch, (diff_mv * 10)); - proxy_warning("Cluster: detected a peer %s:%d with mysql_variables version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. diff_check is increasing, but version 1 doesn't allow sync. This message will be repeated every %u checks until LOAD MYSQL VARIABLES TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch, (diff_mv*10)); - GloProxyCluster->metrics.p_counter_array[p_cluster_counter::sync_delayed_mysql_variables_version_one]->Increment(); + if (v->diff_check && (v->diff_check % (diff_pqr*10)) == 0) { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected a peer %s:%d with pgsql_query_rules version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. diff_check is increasing, but version 1 doesn't allow sync. This message will be repeated every %u checks until LOAD PGSQL QUERY RULES TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch, (diff_pqr * 10)); + proxy_warning("Cluster: detected a peer %s:%d with pgsql_query_rules version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. diff_check is increasing, but version 1 doesn't allow sync. This message will be repeated every %u checks until LOAD PGSQL QUERY RULES TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch, (diff_pqr*10)); } } } - if (GloMyLdapAuth && diff_lv) { - v = &checksums_values.ldap_variables; - unsigned long long own_version = __sync_fetch_and_add(&GloVars.checksums_values.ldap_variables.version, 0); - unsigned long long own_epoch = __sync_fetch_and_add(&GloVars.checksums_values.ldap_variables.epoch, 0); - char* own_checksum = __sync_fetch_and_add(&GloVars.checksums_values.ldap_variables.checksum, 0); - const string expected_checksum { v->checksum }; + + // PostgreSQL Users Sync + if (diff_mu_pgsql) { + v = &checksums_values.pgsql_users; + unsigned long long own_version = __sync_fetch_and_add(&GloVars.checksums_values.pgsql_users.version,0); + unsigned long long own_epoch = __sync_fetch_and_add(&GloVars.checksums_values.pgsql_users.epoch,0); + char* own_checksum = __sync_fetch_and_add(&GloVars.checksums_values.pgsql_users.checksum,0); + const std::string v_exp_checksum { v->checksum }; if (v->version > 1) { if ( @@ -1027,60 +953,88 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) { || (v->epoch > own_epoch) // epoch is newer ) { - if (v->diff_check >= diff_lv) { - proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with ldap_variables version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); - proxy_info("Cluster: detected a peer %s:%d with ldap_variables version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); - GloProxyCluster->pull_global_variables_from_peer("ldap", expected_checksum, v->epoch); + if (v->diff_check >= diff_mu_pgsql) { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with pgsql_users version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); + proxy_info("Cluster: detected a peer %s:%d with pgsql_users version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); + GloProxyCluster->pull_pgsql_users_from_peer(v_exp_checksum, v->epoch); } } - if ((v->epoch == own_epoch) && v->diff_check && ((v->diff_check % (diff_lv*10)) == 0)) { - proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with ldap_variables version %llu, epoch %llu, diff_check %u, checksum %s. Own version: %llu, epoch: %llu, checksum %s. Sync conflict, epoch times are EQUAL, can't determine which server holds the latest config, we won't sync. This message will be repeated every %u checks until LOAD LDAP VARIABLES is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, v->checksum, own_version, own_epoch, own_checksum, (diff_lv * 10)); - proxy_error("Cluster: detected a peer %s:%d with ldap_variables version %llu, epoch %llu, diff_check %u, checksum %s. Own version: %llu, epoch: %llu, checksum %s. Sync conflict, epoch times are EQUAL, can't determine which server holds the latest config, we won't sync. This message will be repeated every %u checks until LOAD LDAP VARIABLES is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, v->checksum, own_version, own_epoch, own_checksum, (diff_lv*10)); - GloProxyCluster->metrics.p_counter_array[p_cluster_counter::sync_conflict_ldap_variables_share_epoch]->Increment(); + if ((v->epoch == own_epoch) && v->diff_check && ((v->diff_check % (diff_mu_pgsql*10)) == 0)) { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with pgsql_users version %llu, epoch %llu, diff_check %u, checksum %s. Own version: %llu, epoch: %llu, checksum %s. Sync conflict, epoch times are EQUAL, can't determine which server holds the latest config, we won't sync. This message will be repeated every %u checks until LOAD PGSQL USERS TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, v->checksum, own_version, own_epoch, own_checksum, (diff_mu_pgsql * 10)); + proxy_error("Cluster: detected a peer %s:%d with pgsql_users version %llu, epoch %llu, diff_check %u, checksum %s. Own version: %llu, epoch: %llu, checksum %s. Sync conflict, epoch times are EQUAL, can't determine which server holds the latest config, we won't sync. This message will be repeated every %u checks until LOAD PGSQL USERS TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, v->checksum, own_version, own_epoch, own_checksum, (diff_mu_pgsql*10)); } } else { - if (v->diff_check && (v->diff_check % (diff_lv*10)) == 0) { - proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with ldap_variables version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. diff_check is increasing, but version 1 doesn't allow sync. This message will be repeated every %u checks until LOAD LDAP VARIABLES TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch, (diff_lv * 10)); - proxy_warning("Cluster: detected a peer %s:%d with ldap_variables version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. diff_check is increasing, but version 1 doesn't allow sync. This message will be repeated every %u checks until LOAD LDAP VARIABLES TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch, (diff_lv*10)); - GloProxyCluster->metrics.p_counter_array[p_cluster_counter::sync_delayed_ldap_variables_version_one]->Increment(); + if (v->diff_check && (v->diff_check % (diff_mu_pgsql*10)) == 0) { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected a peer %s:%d with pgsql_users version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. diff_check is increasing, but version 1 doesn't allow sync. This message will be repeated every %u checks until LOAD PGSQL USERS TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch, (diff_mu_pgsql * 10)); + proxy_warning("Cluster: detected a peer %s:%d with pgsql_users version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. diff_check is increasing, but version 1 doesn't allow sync. This message will be repeated every %u checks until LOAD PGSQL USERS TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch, (diff_mu_pgsql*10)); } } } - // IMPORTANT-NOTE: This action should ALWAYS be performed the last, since the 'checksums_values' gets - // invalidated by 'pull_proxysql_servers_from_peer' and further memory accesses would be invalid. - if (diff_ps) { - v = &checksums_values.proxysql_servers; - unsigned long long own_version = __sync_fetch_and_add(&GloVars.checksums_values.proxysql_servers.version,0); - unsigned long long own_epoch = __sync_fetch_and_add(&GloVars.checksums_values.proxysql_servers.epoch,0); - char* own_checksum = __sync_fetch_and_add(&GloVars.checksums_values.proxysql_servers.checksum,0); + + // PostgreSQL Servers Sync + if (diff_ms_pgsql) { + v = &checksums_values.pgsql_servers_v2; + unsigned long long own_version = __sync_fetch_and_add(&GloVars.checksums_values.pgsql_servers_v2.version,0); + unsigned long long own_epoch = __sync_fetch_and_add(&GloVars.checksums_values.pgsql_servers_v2.epoch,0); + char* own_checksum = __sync_fetch_and_add(&GloVars.checksums_values.pgsql_servers_v2.checksum,0); + const std::string v_exp_checksum { v->checksum }; + if (v->version > 1) { - // NOTE: Backup values: 'v' gets invalidated by 'pull_proxysql_servers_from_peer()' - unsigned long long v_epoch = v->epoch; - unsigned long long v_version = v->version; - unsigned int v_diff_check = v->diff_check; - const string v_exp_checksum { v->checksum }; + if ( + (own_version == 1) // we just booted + || + (v->epoch > own_epoch) // epoch is newer + ) { + if (v->diff_check >= diff_ms_pgsql) { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with pgsql_servers_v2 version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); + proxy_info("Cluster: detected a peer %s:%d with pgsql_servers_v2 version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); + // Create checksum structures for PostgreSQL servers + pgsql_servers_v2_checksum_t pgsql_servers_v2_checksum{v_exp_checksum, static_cast(v->epoch)}; + runtime_pgsql_servers_checksum_t runtime_pgsql_servers_checksum{v_exp_checksum, static_cast(v->epoch)}; + GloProxyCluster->pull_pgsql_servers_v2_from_peer(pgsql_servers_v2_checksum, runtime_pgsql_servers_checksum, true); + } + } + if ((v->epoch == own_epoch) && v->diff_check && ((v->diff_check % (diff_ms_pgsql*10)) == 0)) { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with pgsql_servers_v2 version %llu, epoch %llu, diff_check %u, checksum %s. Own version: %llu, epoch: %llu, checksum %s. Sync conflict, epoch times are EQUAL, can't determine which server holds the latest config, we won't sync. This message will be repeated every %u checks until LOAD PGSQL SERVERS TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, v->checksum, own_version, own_epoch, own_checksum, (diff_ms_pgsql * 10)); + proxy_error("Cluster: detected a peer %s:%d with pgsql_servers_v2 version %llu, epoch %llu, diff_check %u, checksum %s. Own version: %llu, epoch: %llu, checksum %s. Sync conflict, epoch times are EQUAL, can't determine which server holds the latest config, we won't sync. This message will be repeated every %u checks until LOAD PGSQL SERVERS TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, v->checksum, own_version, own_epoch, own_checksum, (diff_ms_pgsql*10)); + } + } else { + if (v->diff_check && (v->diff_check % (diff_ms_pgsql*10)) == 0) { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected a peer %s:%d with pgsql_servers_v2 version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. diff_check is increasing, but version 1 doesn't allow sync. This message will be repeated every %u checks until LOAD PGSQL SERVERS TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch, (diff_ms_pgsql * 10)); + proxy_warning("Cluster: detected a peer %s:%d with pgsql_servers_v2 version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. diff_check is increasing, but version 1 doesn't allow sync. This message will be repeated every %u checks until LOAD PGSQL SERVERS TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch, (diff_ms_pgsql*10)); + } + } + } + // PostgreSQL Variables Sync + if (diff_mv_pgsql) { + v = &checksums_values.pgsql_variables; + unsigned long long own_version = __sync_fetch_and_add(&GloVars.checksums_values.pgsql_variables.version,0); + unsigned long long own_epoch = __sync_fetch_and_add(&GloVars.checksums_values.pgsql_variables.epoch,0); + char* own_checksum = __sync_fetch_and_add(&GloVars.checksums_values.pgsql_variables.checksum,0); + const std::string v_exp_checksum { v->checksum }; + + if (v->version > 1) { if ( (own_version == 1) // we just booted || (v->epoch > own_epoch) // epoch is newer ) { - if (v->diff_check >= diff_ps) { - proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with proxysql_servers version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); - proxy_info("Cluster: detected a peer %s:%d with proxysql_servers version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); - GloProxyCluster->pull_proxysql_servers_from_peer(v_exp_checksum, v->epoch); + if (v->diff_check >= diff_mv_pgsql) { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with pgsql_variables version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); + proxy_info("Cluster: detected a peer %s:%d with pgsql_variables version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); + //GloProxyCluster->pull_pgsql_variables_from_peer(v_exp_checksum, v->epoch); + //metrics.p_counter_array[pulled_pgsql_variables_success]->Increment(); } } - if ((v_epoch == own_epoch) && v_diff_check && ((v_diff_check % (diff_ps*10)) == 0)) { - proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with proxysql_servers version %llu, epoch %llu, diff_check %u, checksum %s. Own version: %llu, epoch: %llu, checksum %s. Sync conflict, epoch times are EQUAL, can't determine which server holds the latest config, we won't sync. This message will be repeated every %u checks until LOAD MYSQL SERVERS TO RUNTIME is executed on candidate master.\n", hostname, port, v_version, v_epoch, v_diff_check, v->checksum, own_version, own_epoch, own_checksum, (diff_ps * 10)); - proxy_error("Cluster: detected a peer %s:%d with proxysql_servers version %llu, epoch %llu, diff_check %u, checksum %s. Own version: %llu, epoch: %llu, checksum %s. Sync conflict, epoch times are EQUAL, can't determine which server holds the latest config, we won't sync. This message will be repeated every %u checks until LOAD MYSQL SERVERS TO RUNTIME is executed on candidate master.\n", hostname, port, v_version, v_epoch, v_diff_check, v->checksum, own_version, own_epoch, own_checksum, (diff_ps*10)); - GloProxyCluster->metrics.p_counter_array[p_cluster_counter::sync_conflict_proxysql_servers_share_epoch]->Increment(); + if ((v->epoch == own_epoch) && v->diff_check && ((v->diff_check % (diff_mv_pgsql*10)) == 0)) { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected a peer %s:%d with pgsql_variables version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. diff_check is increasing, but version 1 doesn't allow sync. This message will be repeated every %u checks until LOAD PGSQL VARIABLES TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch, (diff_mv_pgsql * 10)); + proxy_warning("Cluster: detected a peer %s:%d with pgsql_variables version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. diff_check is increasing, but version 1 doesn't allow sync. This message will be repeated every %u checks until LOAD PGSQL VARIABLES TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch, (diff_mv_pgsql*10)); } } else { - if (v->diff_check && (v->diff_check % (diff_ps*10)) == 0) { - proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with proxysql_servers version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. diff_check is increasing, but version 1 doesn't allow sync. This message will be repeated every %u checks until LOAD PROXYSQL SERVERS TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch, (diff_ps * 10)); - proxy_warning("Cluster: detected a peer %s:%d with proxysql_servers version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. diff_check is increasing, but version 1 doesn't allow sync. This message will be repeated every %u checks until LOAD PROXYSQL SERVERS TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch, (diff_ps*10)); - GloProxyCluster->metrics.p_counter_array[p_cluster_counter::sync_delayed_proxysql_servers_version_one]->Increment(); + if (v->diff_check && (v->diff_check % (diff_mv_pgsql*10)) == 0) { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected a peer %s:%d with pgsql_variables version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. diff_check is increasing, but version 1 doesn't allow sync. This message will be repeated every %u checks until LOAD PGSQL VARIABLES TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch, (diff_mv_pgsql * 10)); + proxy_warning("Cluster: detected a peer %s:%d with pgsql_variables version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. diff_check is increasing, but version 1 doesn't allow sync. This message will be repeated every %u checks until LOAD PGSQL VARIABLES TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch, (diff_mv_pgsql*10)); } } } @@ -1123,6 +1077,33 @@ uint64_t mysql_raw_checksum(MYSQL_RES* resultset) { return res_hash; } +/** + * @brief Pulls MySQL query rules configuration from a cluster peer node. + * + * This function fetches MySQL query rules from a peer ProxySQL instance when the peer's + * checksum differs from the local checksum and the difference exceeds the configured + * threshold (cluster_mysql_query_rules_diffs_before_sync). It retrieves both regular query + * rules and fast routing rules. + * + * The function performs the following steps: + * 1. Identifies the optimal peer to sync from using get_peer_to_sync_mysql_query_rules() + * 2. Establishes a MySQL connection to the peer's admin interface + * 3. Executes CLUSTER_QUERY_MYSQL_QUERY_RULES and CLUSTER_QUERY_MYSQL_QUERY_RULES_FAST_ROUTING + * 4. Computes checksums for the fetched data + * 5. Validates checksums match the expected values + * 6. Loads the query rules to runtime via load_mysql_query_rules_to_runtime() + * 7. Optionally saves configuration to disk if cluster_mysql_query_rules_save_to_disk is enabled + * + * @param expected_checksum The expected checksum of the query rules on the peer + * @param epoch The epoch timestamp of the query rules on the peer + * + * @note This function is thread-safe and requires the update_mysql_query_rules_mutex to be held + * @note The function will sleep(1) if the fetch operation fails to prevent busy loops + * @see get_peer_to_sync_mysql_query_rules() + * @see CLUSTER_QUERY_MYSQL_QUERY_RULES + * @see CLUSTER_QUERY_MYSQL_QUERY_RULES_FAST_ROUTING + * @see load_mysql_query_rules_to_runtime() + */ void ProxySQL_Cluster::pull_mysql_query_rules_from_peer(const string& expected_checksum, const time_t epoch) { char * hostname = NULL; char * ip_address = NULL; @@ -2676,17 +2657,599 @@ void ProxySQL_Cluster::pull_proxysql_servers_from_peer(const std::string& expect if (fetch_failed == true) sleep(1); } -void ProxySQL_Node_Entry::set_metrics(MYSQL_RES *_r, unsigned long long _response_time) { - MYSQL_ROW row; - metrics_idx_prev = metrics_idx; - metrics_idx++; - if (metrics_idx == PROXYSQL_NODE_METRICS_LEN) { - metrics_idx = 0; - } - ProxySQL_Node_Metrics *m = metrics[metrics_idx]; - m->reset(); - m->read_time_us = monotonic_time(); - m->response_time_us = _response_time; +/** + * @brief Pulls PostgreSQL users configuration from a cluster peer node. + * + * This function fetches PostgreSQL users from a peer ProxySQL instance when the peer's + * checksum differs from the local checksum and the difference exceeds the configured + * threshold (cluster_pgsql_users_diffs_before_sync). It retrieves PostgreSQL user credentials + * including usernames, passwords, and connection settings for PostgreSQL authentication. + * + * The function performs the following steps: + * 1. Identifies the optimal peer to sync from using get_peer_to_sync_pgsql_users() + * 2. Establishes a MySQL connection to the peer's admin interface + * 3. Executes CLUSTER_QUERY_PGSQL_USERS to fetch user configuration + * 4. Computes checksums for the fetched data using get_mysql_users_checksum() + * 5. Validates checksums match the expected values + * 6. Loads the users to runtime via init_pgsql_users() + * 7. Optionally saves configuration to disk if cluster_pgsql_users_save_to_disk is enabled + * + * This function provides PostgreSQL-specific cluster synchronization for user credentials, + * ensuring consistent PostgreSQL authentication across all cluster nodes. + * + * @param expected_checksum The expected checksum of the PostgreSQL users on the peer + * @param epoch The epoch timestamp of the PostgreSQL users on the peer + * + * @note This function is thread-safe and reuses the update_mysql_users_mutex + * @note The function will sleep(1) if the fetch operation fails to prevent busy loops + * @note Reuses get_mysql_users_checksum() from MySQL users for consistency + * @see get_peer_to_sync_pgsql_users() + * @see CLUSTER_QUERY_PGSQL_USERS + * @see init_pgsql_users() + * @see get_mysql_users_checksum() + */ +void ProxySQL_Cluster::pull_pgsql_users_from_peer(const std::string& expected_checksum, const time_t epoch) { + char * hostname = NULL; + char * ip_address = NULL; + uint16_t port = 0; + bool fetch_failed = false; + pthread_mutex_lock(&GloProxyCluster->update_mysql_users_mutex); // Reuse mysql_users mutex for pgsql_users + nodes.get_peer_to_sync_pgsql_users(&hostname, &port, &ip_address); + if (hostname) { + cluster_creds_t creds {}; + + MYSQL *conn = mysql_init(NULL); + if (conn==NULL) { + proxy_error("Unable to run mysql_init()\n"); + goto __exit_pull_pgsql_users_from_peer; + } + + creds = GloProxyCluster->get_credentials(); + if (creds.user.size()) { // do not monitor if the username is empty + // READ/WRITE timeouts were enforced as an attempt to prevent deadlocks in the original + // implementation. They were proven unnecessary, leaving only 'CONNECT_TIMEOUT'. + unsigned int timeout = 1; + mysql_options(conn, MYSQL_OPT_CONNECT_TIMEOUT, &timeout); + { + unsigned char val = 1; mysql_options(conn, MYSQL_OPT_SSL_ENFORCE, &val); + mysql_options(conn, MARIADB_OPT_SSL_KEYLOG_CALLBACK, (void*)proxysql_keylog_write_line_callback); + } + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Fetching PostgreSQL Users from peer %s:%d started. Expected checksum: %s\n", hostname, port, expected_checksum.c_str()); + proxy_info("Cluster: Fetching PostgreSQL Users from peer %s:%d started. Expected checksum: %s\n", hostname, port, expected_checksum.c_str()); + + MYSQL* rc_conn = mysql_real_connect(conn, ip_address ? ip_address : hostname, creds.user.c_str(), creds.pass.c_str(), NULL, port, NULL, 0); + if (rc_conn == nullptr) { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Fetching PostgreSQL Users from peer %s:%d failed: %s\n", hostname, port, mysql_error(conn)); + proxy_info("Cluster: Fetching PostgreSQL Users from peer %s:%d failed: %s\n", hostname, port, mysql_error(conn)); + metrics.p_counter_array[p_cluster_counter::pulled_mysql_users_failure]->Increment(); // Reuse mysql_users counter + fetch_failed = true; + goto __exit_pull_pgsql_users_from_peer; + } + + MySQL_Monitor::update_dns_cache_from_mysql_conn(conn); + + int rc_query = mysql_query(conn, CLUSTER_QUERY_PGSQL_USERS); + if (rc_query == 0) { + MYSQL_RES* pgsql_users_result = mysql_store_result(conn); + + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Fetching PostgreSQL Users from peer %s:%d completed\n", hostname, port); + proxy_info("Cluster: Fetching PostgreSQL Users from peer %s:%d completed\n", hostname, port); + + unique_ptr pgsql_users_resultset { nullptr }; + const uint64_t users_raw_checksum = get_mysql_users_checksum(pgsql_users_result, nullptr, pgsql_users_resultset); + const string computed_checksum { get_checksum_from_hash(users_raw_checksum) }; + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Computed checksum for PostgreSQL Users from peer %s:%d : %s\n", hostname, port, computed_checksum.c_str()); + proxy_info("Cluster: Computed checksum for PostgreSQL Users from peer %s:%d : %s\n", hostname, port, computed_checksum.c_str()); + + if (expected_checksum == computed_checksum) { + update_mysql_users(pgsql_users_result); // Reuse update_mysql_users for pgsql_users + mysql_free_result(pgsql_users_result); + + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Loading to runtime PostgreSQL Users from peer %s:%d\n", hostname, port); + proxy_info("Cluster: Loading to runtime PostgreSQL Users from peer %s:%d\n", hostname, port); + + GloAdmin->init_pgsql_users(std::move(pgsql_users_resultset), expected_checksum, epoch); + if (GloProxyCluster->cluster_pgsql_users_save_to_disk == true) { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Saving to disk PostgreSQL Users from peer %s:%d\n", hostname, port); + proxy_info("Cluster: Saving to disk PostgreSQL Users from peer %s:%d\n", hostname, port); + GloAdmin->flush_pgsql_users__from_memory_to_disk(); + } else { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "NOT saving to disk PostgreSQL Users from peer %s:%d\n", hostname, port); + proxy_info("Cluster: NOT saving to disk PostgreSQL Users from peer %s:%d\n", hostname, port); + } + + metrics.p_counter_array[p_cluster_counter::pulled_mysql_users_success]->Increment(); // Reuse mysql_users counter + } else { + if (pgsql_users_result) { + mysql_free_result(pgsql_users_result); + } + + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Fetching PostgreSQL Users from peer %s:%d failed: Checksum changed from %s to %s\n", + hostname, port, expected_checksum.c_str(), computed_checksum.c_str()); + proxy_info( + "Cluster: Fetching PostgreSQL Users from peer %s:%d failed: Checksum changed from %s to %s\n", + hostname, port, expected_checksum.c_str(), computed_checksum.c_str() + ); + metrics.p_counter_array[p_cluster_counter::pulled_mysql_users_failure]->Increment(); // Reuse mysql_users counter + fetch_failed = true; + } + } else { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Fetching PostgreSQL Users from peer %s:%d failed: %s\n", hostname, port, mysql_error(conn)); + proxy_info("Cluster: Fetching PostgreSQL Users from peer %s:%d failed: %s\n", hostname, port, mysql_error(conn)); + metrics.p_counter_array[p_cluster_counter::pulled_mysql_users_failure]->Increment(); // Reuse mysql_users counter + fetch_failed = true; + } + } +__exit_pull_pgsql_users_from_peer: + if (conn) { + if (conn->net.pvio) { + mysql_close(conn); + } + } + free(hostname); + + if (ip_address) + free(ip_address); + } + pthread_mutex_unlock(&GloProxyCluster->update_mysql_users_mutex); + if (fetch_failed == true) sleep(1); +} + +/** + * @brief Pulls PostgreSQL query rules configuration from a cluster peer node. + * + * This function fetches PostgreSQL query rules from a peer ProxySQL instance when the peer's + * checksum differs from the local checksum and the difference exceeds the configured + * threshold (cluster_pgsql_query_rules_diffs_before_sync). It retrieves both regular query + * rules and fast routing rules, providing PostgreSQL-specific cluster synchronization. + * + * The function performs the following steps: + * 1. Identifies the optimal peer to sync from using get_peer_to_sync_pgsql_query_rules() + * 2. Establishes a MySQL connection to the peer's admin interface + * 3. Executes CLUSTER_QUERY_PGSQL_QUERY_RULES and CLUSTER_QUERY_PGSQL_QUERY_RULES_FAST_ROUTING + * 4. Computes checksums for the fetched data using mysql_raw_checksum() and SpookyHash + * 5. Validates checksums match the expected values + * 6. Loads the query rules to runtime via load_pgsql_query_rules_to_runtime() + * 7. Optionally saves configuration to disk if cluster_pgsql_query_rules_save_to_disk is enabled + * + * This function implements the same synchronization pattern as MySQL query rules but + * for PostgreSQL-specific tables and query rule processing. + * + * @param expected_checksum The expected checksum of the PostgreSQL query rules on the peer + * @param epoch The epoch timestamp of the PostgreSQL query rules on the peer + * + * @note This function is thread-safe and reuses the update_mysql_query_rules_mutex + * @note The function will sleep(1) if the fetch operation fails to prevent busy loops + * @note Uses the same checksum computation algorithm as MySQL query rules for consistency + * @see get_peer_to_sync_pgsql_query_rules() + * @see CLUSTER_QUERY_PGSQL_QUERY_RULES + * @see CLUSTER_QUERY_PGSQL_QUERY_RULES_FAST_ROUTING + * @see load_pgsql_query_rules_to_runtime() + * @see SpookyHash + * @see mysql_raw_checksum() + */ +void ProxySQL_Cluster::pull_pgsql_query_rules_from_peer(const std::string& expected_checksum, const time_t epoch) { + char * hostname = NULL; + char * ip_address = NULL; + uint16_t port = 0; + bool fetch_failed = false; + pthread_mutex_lock(&GloProxyCluster->update_mysql_query_rules_mutex); // Reuse mysql_query_rules mutex for pgsql_query_rules + nodes.get_peer_to_sync_pgsql_query_rules(&hostname, &port, &ip_address); + if (hostname) { + cluster_creds_t creds {}; + + MYSQL *conn = mysql_init(NULL); + if (conn==NULL) { + proxy_error("Unable to run mysql_init()\n"); + goto __exit_pull_pgsql_query_rules_from_peer; + } + + creds = GloProxyCluster->get_credentials(); + if (creds.user.size()) { // do not monitor if the username is empty + // READ/WRITE timeouts were enforced as an attempt to prevent deadlocks in the original + // implementation. They were proven unnecessary, leaving only 'CONNECT_TIMEOUT'. + unsigned int timeout = 1; + mysql_options(conn, MYSQL_OPT_CONNECT_TIMEOUT, &timeout); + { + unsigned char val = 1; mysql_options(conn, MYSQL_OPT_SSL_ENFORCE, &val); + mysql_options(conn, MARIADB_OPT_SSL_KEYLOG_CALLBACK, (void*)proxysql_keylog_write_line_callback); + } + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Fetching PostgreSQL Query Rules from peer %s:%d started. Expected checksum: %s\n", hostname, port, expected_checksum.c_str()); + proxy_info("Cluster: Fetching PostgreSQL Query Rules from peer %s:%d started. Expected checksum: %s\n", hostname, port, expected_checksum.c_str()); + + MYSQL* rc_conn = mysql_real_connect(conn, ip_address ? ip_address : hostname, creds.user.c_str(), creds.pass.c_str(), NULL, port, NULL, 0); + if (rc_conn == nullptr) { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Fetching PostgreSQL Query Rules from peer %s:%d failed: %s\n", hostname, port, mysql_error(conn)); + proxy_info("Cluster: Fetching PostgreSQL Query Rules from peer %s:%d failed: %s\n", hostname, port, mysql_error(conn)); + metrics.p_counter_array[p_cluster_counter::pulled_mysql_query_rules_failure]->Increment(); // Reuse mysql_query_rules counter + fetch_failed = true; + goto __exit_pull_pgsql_query_rules_from_peer; + } + + MySQL_Monitor::update_dns_cache_from_mysql_conn(conn); + + int rc_query = mysql_query(conn, CLUSTER_QUERY_PGSQL_QUERY_RULES); + if (rc_query == 0) { + MYSQL_RES* query_rules_result = mysql_store_result(conn); + MYSQL_RES* fast_routing_result = nullptr; + + // Fetch fast routing rules + int rc_query_fast = mysql_query(conn, CLUSTER_QUERY_PGSQL_QUERY_RULES_FAST_ROUTING); + if (rc_query_fast == 0) { + fast_routing_result = mysql_store_result(conn); + } + + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Fetching PostgreSQL Query Rules from peer %s:%d completed\n", hostname, port); + proxy_info("Cluster: Fetching PostgreSQL Query Rules from peer %s:%d completed\n", hostname, port); + + // Compute checksum from the MySQL resultset + const uint64_t query_rules_raw_checksum = mysql_raw_checksum(query_rules_result); + const uint64_t fast_routing_raw_checksum = fast_routing_result ? mysql_raw_checksum(fast_routing_result) : 0; + + // Combine both checksums using the same pattern as MySQL query rules + SpookyHash myhash {}; + myhash.Init(19, 3); + myhash.Update(&query_rules_raw_checksum, sizeof(query_rules_raw_checksum)); + myhash.Update(&fast_routing_raw_checksum, sizeof(fast_routing_raw_checksum)); + + uint64_t rules_raw_checksum = 0, hash2 = 0; + myhash.Final(&rules_raw_checksum, &hash2); + + const string computed_checksum { get_checksum_from_hash(rules_raw_checksum) }; + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Computed checksum for PostgreSQL Query Rules from peer %s:%d : %s\n", hostname, port, computed_checksum.c_str()); + proxy_info("Cluster: Computed checksum for PostgreSQL Query Rules from peer %s:%d : %s\n", hostname, port, computed_checksum.c_str()); + + if (expected_checksum == computed_checksum) { + mysql_free_result(query_rules_result); + if (fast_routing_result) { + mysql_free_result(fast_routing_result); + } + + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Loading to runtime PostgreSQL Query Rules from peer %s:%d\n", hostname, port); + proxy_info("Cluster: Loading to runtime PostgreSQL Query Rules from peer %s:%d\n", hostname, port); + + // For cluster sync, we pass nullptr to let load_pgsql_query_rules_to_runtime fetch from database + // This is the same pattern used for other cluster sync operations + GloAdmin->load_pgsql_query_rules_to_runtime(nullptr, nullptr, expected_checksum, epoch); + if (GloProxyCluster->cluster_pgsql_query_rules_save_to_disk == true) { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Saving to disk PostgreSQL Query Rules from peer %s:%d\n", hostname, port); + proxy_info("Cluster: Saving to disk PostgreSQL Query Rules from peer %s:%d\n", hostname, port); + // TODO: Add flush_pgsql_query_rules__from_memory_to_disk() when implemented + } else { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "NOT saving to disk PostgreSQL Query Rules from peer %s:%d\n", hostname, port); + proxy_info("Cluster: NOT saving to disk PostgreSQL Query Rules from peer %s:%d\n", hostname, port); + } + + metrics.p_counter_array[p_cluster_counter::pulled_mysql_query_rules_success]->Increment(); // Reuse mysql_query_rules counter + } else { + if (query_rules_result) { + mysql_free_result(query_rules_result); + } + if (fast_routing_result) { + mysql_free_result(fast_routing_result); + } + + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Fetching PostgreSQL Query Rules from peer %s:%d failed: Checksum changed from %s to %s\n", + hostname, port, expected_checksum.c_str(), computed_checksum.c_str()); + proxy_info( + "Cluster: Fetching PostgreSQL Query Rules from peer %s:%d failed: Checksum changed from %s to %s\n", + hostname, port, expected_checksum.c_str(), computed_checksum.c_str() + ); + metrics.p_counter_array[p_cluster_counter::pulled_mysql_query_rules_failure]->Increment(); // Reuse mysql_query_rules counter + fetch_failed = true; + } + } else { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Fetching PostgreSQL Query Rules from peer %s:%d failed: %s\n", hostname, port, mysql_error(conn)); + proxy_info("Cluster: Fetching PostgreSQL Query Rules from peer %s:%d failed: %s\n", hostname, port, mysql_error(conn)); + metrics.p_counter_array[p_cluster_counter::pulled_mysql_query_rules_failure]->Increment(); // Reuse mysql_query_rules counter + fetch_failed = true; + } + } +__exit_pull_pgsql_query_rules_from_peer: + if (conn) { + if (conn->net.pvio) { + mysql_close(conn); + } + } + free(hostname); + + if (ip_address) + free(ip_address); + } + pthread_mutex_unlock(&GloProxyCluster->update_mysql_query_rules_mutex); + if (fetch_failed == true) sleep(1); +} + +/** + * @brief Pulls runtime PostgreSQL servers configuration from a cluster peer node. + * + * This function fetches runtime PostgreSQL servers status from a peer ProxySQL instance when the peer's + * checksum differs from the local checksum and the difference exceeds the configured + * threshold. It retrieves the current operational status, health metrics, and runtime statistics + * for PostgreSQL servers in the cluster. + * + * The function performs the following steps: + * 1. Identifies the optimal peer to sync from using get_peer_to_sync_runtime_pgsql_servers() + * 2. Establishes a MySQL connection to the peer's admin interface + * 3. Executes CLUSTER_QUERY_RUNTIME_PGSQL_SERVERS to fetch runtime server status + * 4. Computes checksum for the fetched data using mysql_raw_checksum() + * 5. Validates checksum matches the expected value from peer_runtime_pgsql_server.value + * 6. Loads the runtime PostgreSQL servers status (TODO: integrate with load_pgsql_servers_to_runtime) + * 7. Optionally saves configuration to disk if save settings are enabled + * + * Runtime data includes: + * - Server status (ONLINE, OFFLINE, SHUNNED, etc.) + * - Current connections and load metrics + * - Health check results and response times + * - Error counts and statistics + * + * @param peer_runtime_pgsql_server The checksum structure containing expected checksum value and epoch timestamp for runtime PostgreSQL servers + * + * @note This function is thread-safe and requires the update_runtime_mysql_servers_mutex to be held (reused for pgsql_servers) + * @note The function will sleep(1) if the fetch operation fails to prevent busy loops + * @note The function reuses MySQL servers counters for metrics tracking + * @note Runtime loading integration is pending and marked as TODO + * @see get_peer_to_sync_runtime_pgsql_servers() + * @see CLUSTER_QUERY_RUNTIME_PGSQL_SERVERS + * @see mysql_raw_checksum() + * @see get_checksum_from_hash() + */ +void ProxySQL_Cluster::pull_runtime_pgsql_servers_from_peer(const runtime_pgsql_servers_checksum_t& peer_runtime_pgsql_server) { + char * hostname = NULL; + char * ip_address = NULL; + uint16_t port = 0; + bool fetch_failed = false; + pthread_mutex_lock(&GloProxyCluster->update_runtime_mysql_servers_mutex); // Reuse mysql_servers mutex for pgsql_servers + nodes.get_peer_to_sync_runtime_pgsql_servers(&hostname, &port, nullptr, &ip_address); + if (hostname) { + cluster_creds_t creds {}; + + MYSQL *conn = mysql_init(NULL); + if (conn==NULL) { + proxy_error("Unable to run mysql_init()\n"); + goto __exit_pull_runtime_pgsql_servers_from_peer; + } + + creds = GloProxyCluster->get_credentials(); + if (creds.user.size()) { // do not monitor if the username is empty + // READ/WRITE timeouts were enforced as an attempt to prevent deadlocks in the original + // implementation. They were proven unnecessary, leaving only 'CONNECT_TIMEOUT'. + unsigned int timeout = 1; + mysql_options(conn, MYSQL_OPT_CONNECT_TIMEOUT, &timeout); + { + unsigned char val = 1; mysql_options(conn, MYSQL_OPT_SSL_ENFORCE, &val); + mysql_options(conn, MARIADB_OPT_SSL_KEYLOG_CALLBACK, (void*)proxysql_keylog_write_line_callback); + } + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Fetching Runtime PostgreSQL Servers from peer %s:%d started.\n", hostname, port); + proxy_info("Cluster: Fetching Runtime PostgreSQL Servers from peer %s:%d started.\n", hostname, port); + + MYSQL* rc_conn = mysql_real_connect(conn, ip_address ? ip_address : hostname, creds.user.c_str(), creds.pass.c_str(), NULL, port, NULL, 0); + if (rc_conn == nullptr) { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Fetching Runtime PostgreSQL Servers from peer %s:%d failed: %s\n", hostname, port, mysql_error(conn)); + proxy_info("Cluster: Fetching Runtime PostgreSQL Servers from peer %s:%d failed: %s\n", hostname, port, mysql_error(conn)); + metrics.p_counter_array[p_cluster_counter::pulled_mysql_servers_failure]->Increment(); // Reuse mysql_servers counter + fetch_failed = true; + goto __exit_pull_runtime_pgsql_servers_from_peer; + } + + MySQL_Monitor::update_dns_cache_from_mysql_conn(conn); + + const fetch_query f_queries[] = { + { + CLUSTER_QUERY_RUNTIME_PGSQL_SERVERS, + p_cluster_counter::pulled_mysql_servers_success, + p_cluster_counter::pulled_mysql_servers_failure, + { + "Cluster: Fetching Runtime PostgreSQL Servers from peer " + string(hostname) + ":" + std::to_string(port) + " completed.", + "Cluster: Loading to runtime PostgreSQL Servers from peer " + string(hostname) + ":" + std::to_string(port) + ".", + "Cluster: NOT saving to disk PostgreSQL Servers from peer " + string(hostname) + ":" + std::to_string(port) + "." + } + } + }; + + string computed_checksum; + int rc_query = -1; + MYSQL_RES* result = nullptr; + string tmp_expected_checksum = peer_runtime_pgsql_server.value; + + for (const auto& f_query : f_queries) { + if (tmp_expected_checksum.empty()) { break; } + + proxy_info("%s\n", f_query.msgs[0].c_str()); + rc_query = fetch_and_store(conn, f_query, &result); + + if (rc_query != 0) { + computed_checksum.clear(); + break; + } + + const uint64_t hash_val = mysql_raw_checksum(result); + computed_checksum = get_checksum_from_hash(hash_val); + + if (computed_checksum == tmp_expected_checksum) { + proxy_info("%s\n", f_query.msgs[1].c_str()); + // TODO: Call load_pgsql_servers_to_runtime when integrated with cluster sync + // GloAdmin->load_pgsql_servers_to_runtime(result, {}, computed_checksum, peer_runtime_pgsql_server.epoch); + metrics.p_counter_array[f_query.success_counter]->Increment(); + } else { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Checksum mismatch while syncing Runtime PostgreSQL Servers. Expected: %s, Computed: %s\n", + tmp_expected_checksum.c_str(), computed_checksum.c_str()); + proxy_info("Cluster: Checksum mismatch while syncing Runtime PostgreSQL Servers. Expected: %s, Computed: %s\n", + tmp_expected_checksum.c_str(), computed_checksum.c_str()); + metrics.p_counter_array[f_query.failure_counter]->Increment(); + fetch_failed = true; + } + + if (result) { + mysql_free_result(result); + result = nullptr; + } + } + } +__exit_pull_runtime_pgsql_servers_from_peer: + if (conn) { + if (conn->net.pvio) { + mysql_close(conn); + } + } + free(hostname); + + if (ip_address) + free(ip_address); + } + pthread_mutex_unlock(&GloProxyCluster->update_runtime_mysql_servers_mutex); + if (fetch_failed == true) sleep(1); +} + +/** + * @brief Pulls PostgreSQL servers v2 configuration from a cluster peer node. + * + * This function fetches PostgreSQL servers configuration from a peer ProxySQL instance when the peer's + * checksum differs from the local checksum and the difference exceeds the configured + * threshold (cluster_pgsql_servers_diffs_before_sync). It retrieves PostgreSQL server definitions + * including hostgroup mappings, connection parameters, and server status. + * + * The function performs the following steps: + * 1. Identifies the optimal peer to sync from using get_peer_to_sync_pgsql_servers_v2() + * 2. Establishes a MySQL connection to the peer's admin interface + * 3. Executes CLUSTER_QUERY_PGSQL_SERVERS_V2 to fetch server configuration + * 4. Computes checksum for the fetched data using mysql_raw_checksum() + * 5. Validates checksum matches the expected value from peer_pgsql_server_v2.value + * 6. Loads the PostgreSQL servers configuration (TODO: integrate with load_pgsql_servers_to_runtime) + * 7. Optionally saves configuration to disk if cluster_pgsql_servers_save_to_disk is enabled + * + * @param peer_pgsql_server_v2 The checksum structure containing expected checksum value and epoch timestamp for pgsql_servers_v2 + * @param peer_runtime_pgsql_server The checksum structure containing runtime PostgreSQL servers checksum (currently unused but reserved for future use) + * @param fetch_runtime_pgsql_servers Boolean flag indicating whether to fetch runtime PostgreSQL servers data (currently unused but reserved for future implementation) + * + * @note This function is thread-safe and requires the update_mysql_servers_v2_mutex to be held (reused for pgsql_servers_v2) + * @note The function will sleep(1) if the fetch operation fails to prevent busy loops + * @note The function reuses MySQL servers counters for metrics tracking + * @note Runtime loading integration is pending and marked as TODO + * @see get_peer_to_sync_pgsql_servers_v2() + * @see CLUSTER_QUERY_PGSQL_SERVERS_V2 + * @see mysql_raw_checksum() + * @see get_checksum_from_hash() + */ +void ProxySQL_Cluster::pull_pgsql_servers_v2_from_peer(const pgsql_servers_v2_checksum_t& peer_pgsql_server_v2, + const runtime_pgsql_servers_checksum_t& peer_runtime_pgsql_server, bool fetch_runtime_pgsql_servers) { + char * hostname = NULL; + char * ip_address = NULL; + uint16_t port = 0; + bool fetch_failed = false; + pthread_mutex_lock(&GloProxyCluster->update_mysql_servers_v2_mutex); // Reuse mysql_servers_v2 mutex for pgsql_servers_v2 + nodes.get_peer_to_sync_pgsql_servers_v2(&hostname, &port, nullptr, nullptr, &ip_address); + if (hostname) { + cluster_creds_t creds {}; + + MYSQL *conn = mysql_init(NULL); + if (conn==NULL) { + proxy_error("Unable to run mysql_init()\n"); + goto __exit_pull_pgsql_servers_v2_from_peer; + } + + creds = GloProxyCluster->get_credentials(); + if (creds.user.size()) { // do not monitor if the username is empty + // READ/WRITE timeouts were enforced as an attempt to prevent deadlocks in the original + // implementation. They were proven unnecessary, leaving only 'CONNECT_TIMEOUT'. + unsigned int timeout = 1; + mysql_options(conn, MYSQL_OPT_CONNECT_TIMEOUT, &timeout); + { + unsigned char val = 1; mysql_options(conn, MYSQL_OPT_SSL_ENFORCE, &val); + mysql_options(conn, MARIADB_OPT_SSL_KEYLOG_CALLBACK, (void*)proxysql_keylog_write_line_callback); + } + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Fetching PostgreSQL Servers v2 from peer %s:%d started.\n", hostname, port); + proxy_info("Cluster: Fetching PostgreSQL Servers v2 from peer %s:%d started.\n", hostname, port); + + MYSQL* rc_conn = mysql_real_connect(conn, ip_address ? ip_address : hostname, creds.user.c_str(), creds.pass.c_str(), NULL, port, NULL, 0); + if (rc_conn == nullptr) { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Fetching PostgreSQL Servers v2 from peer %s:%d failed: %s\n", hostname, port, mysql_error(conn)); + proxy_info("Cluster: Fetching PostgreSQL Servers v2 from peer %s:%d failed: %s\n", hostname, port, mysql_error(conn)); + metrics.p_counter_array[p_cluster_counter::pulled_mysql_servers_failure]->Increment(); // Reuse mysql_servers counter + fetch_failed = true; + goto __exit_pull_pgsql_servers_v2_from_peer; + } + + MySQL_Monitor::update_dns_cache_from_mysql_conn(conn); + + const fetch_query f_queries[] = { + { + CLUSTER_QUERY_PGSQL_SERVERS_V2, + p_cluster_counter::pulled_mysql_servers_success, + p_cluster_counter::pulled_mysql_servers_failure, + { + "Cluster: Fetching PostgreSQL Servers v2 from peer " + string(hostname) + ":" + std::to_string(port) + " completed.", + "Cluster: Loading to runtime PostgreSQL Servers from peer " + string(hostname) + ":" + std::to_string(port) + ".", + "Cluster: NOT saving to disk PostgreSQL Servers from peer " + string(hostname) + ":" + std::to_string(port) + "." + } + } + }; + + string computed_checksum; + int rc_query = -1; + MYSQL_RES* result = nullptr; + string tmp_expected_checksum = peer_pgsql_server_v2.value; + + for (const auto& f_query : f_queries) { + if (tmp_expected_checksum.empty()) { break; } + + proxy_info("%s\n", f_query.msgs[0].c_str()); + rc_query = fetch_and_store(conn, f_query, &result); + + if (rc_query != 0) { + computed_checksum.clear(); + break; + } + + const uint64_t hash_val = mysql_raw_checksum(result); + computed_checksum = get_checksum_from_hash(hash_val); + + if (computed_checksum == tmp_expected_checksum) { + proxy_info("%s\n", f_query.msgs[1].c_str()); + // TODO: Call load_pgsql_servers_to_runtime when integrated with cluster sync + // GloAdmin->load_pgsql_servers_to_runtime(result, {}, computed_checksum, peer_pgsql_server_v2.epoch); + metrics.p_counter_array[f_query.success_counter]->Increment(); + } else { + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Checksum mismatch while syncing PostgreSQL Servers v2. Expected: %s, Computed: %s\n", + tmp_expected_checksum.c_str(), computed_checksum.c_str()); + proxy_info("Cluster: Checksum mismatch while syncing PostgreSQL Servers v2. Expected: %s, Computed: %s\n", + tmp_expected_checksum.c_str(), computed_checksum.c_str()); + metrics.p_counter_array[f_query.failure_counter]->Increment(); + fetch_failed = true; + } + + if (result) { + mysql_free_result(result); + result = nullptr; + } + } + } +__exit_pull_pgsql_servers_v2_from_peer: + if (conn) { + if (conn->net.pvio) { + mysql_close(conn); + } + } + free(hostname); + + if (ip_address) + free(ip_address); + } + pthread_mutex_unlock(&GloProxyCluster->update_mysql_servers_v2_mutex); + if (fetch_failed == true) sleep(1); +} + +void ProxySQL_Node_Entry::set_metrics(MYSQL_RES *_r, unsigned long long _response_time) { + MYSQL_ROW row; + metrics_idx_prev = metrics_idx; + metrics_idx++; + if (metrics_idx == PROXYSQL_NODE_METRICS_LEN) { + metrics_idx = 0; + } + ProxySQL_Node_Metrics *m = metrics[metrics_idx]; + m->reset(); + m->read_time_us = monotonic_time(); + m->response_time_us = _response_time; while ((row = mysql_fetch_row(_r))) { char c = row[0][0]; switch (c) { @@ -3025,7 +3588,7 @@ void ProxySQL_Cluster_Nodes::get_peer_to_sync_mysql_query_rules(char **host, uin uint16_t p = 0; // pthread_mutex_lock(&mutex); //unsigned long long curtime = monotonic_time(); - unsigned int diff_mqr = (unsigned int)__sync_fetch_and_add(&GloProxyCluster->cluster_mysql_query_rules_diffs_before_sync,0); + unsigned int diff_mqr = (unsigned int)GloProxyCluster->cluster_mysql_query_rules_diffs_before_sync; for( std::unordered_map::iterator it = umap_proxy_nodes.begin(); it != umap_proxy_nodes.end(); ) { ProxySQL_Node_Entry * node = it->second; ProxySQL_Checksum_Value_2 * v = &node->checksums_values.mysql_query_rules; @@ -3086,7 +3649,7 @@ void ProxySQL_Cluster_Nodes::get_peer_to_sync_runtime_mysql_servers(char **host, char *pc = NULL; // pthread_mutex_lock(&mutex); //unsigned long long curtime = monotonic_time(); - unsigned int diff_ms = (unsigned int)__sync_fetch_and_add(&GloProxyCluster->cluster_mysql_servers_diffs_before_sync,0); + unsigned int diff_ms = (unsigned int)GloProxyCluster->cluster_mysql_servers_diffs_before_sync; for( std::unordered_map::iterator it = umap_proxy_nodes.begin(); it != umap_proxy_nodes.end(); ) { ProxySQL_Node_Entry * node = it->second; ProxySQL_Checksum_Value_2 * v = &node->checksums_values.mysql_servers; @@ -3156,7 +3719,7 @@ void ProxySQL_Cluster_Nodes::get_peer_to_sync_mysql_servers_v2(char** host, uint char* runtime_mysql_servers_checksum = NULL; //pthread_mutex_lock(&mutex); //unsigned long long curtime = monotonic_time(); - unsigned int diff_ms = (unsigned int)__sync_fetch_and_add(&GloProxyCluster->cluster_mysql_servers_diffs_before_sync, 0); + unsigned int diff_ms = (unsigned int)GloProxyCluster->cluster_mysql_servers_diffs_before_sync; for (std::unordered_map::iterator it = umap_proxy_nodes.begin(); it != umap_proxy_nodes.end(); ) { ProxySQL_Node_Entry* node = it->second; ProxySQL_Checksum_Value_2* v = &node->checksums_values.mysql_servers_v2; @@ -3232,7 +3795,7 @@ void ProxySQL_Cluster_Nodes::get_peer_to_sync_mysql_users(char **host, uint16_t uint16_t p = 0; // pthread_mutex_lock(&mutex); //unsigned long long curtime = monotonic_time(); - unsigned int diff_mu = (unsigned int)__sync_fetch_and_add(&GloProxyCluster->cluster_mysql_users_diffs_before_sync,0); + unsigned int diff_mu = (unsigned int)GloProxyCluster->cluster_mysql_users_diffs_before_sync; for( std::unordered_map::iterator it = umap_proxy_nodes.begin(); it != umap_proxy_nodes.end(); ) { ProxySQL_Node_Entry * node = it->second; ProxySQL_Checksum_Value_2 * v = &node->checksums_values.mysql_users; @@ -3281,21 +3844,73 @@ void ProxySQL_Cluster_Nodes::get_peer_to_sync_mysql_users(char **host, uint16_t } } -void ProxySQL_Cluster_Nodes::get_peer_to_sync_mysql_variables(char **host, uint16_t *port, char** ip_address) { +/** + * @brief Unified function to find optimal peer for syncing variables modules + * + * Data-driven implementation that replaces separate functions for mysql_variables, + * admin_variables, and ldap_variables. Uses module configuration to determine: + * - Which cluster variable to check for diff threshold + * - Which checksum field to examine in each node + * - Module name for debug logging + * + * @param module_name The name of the module ("mysql_variables", "admin_variables", "ldap_variables", "pgsql_variables") + * @param host Pointer to store the selected peer's hostname + * @param port Pointer to store the selected peer's port + * @param ip_address Pointer to store the selected peer's IP address + */ +void ProxySQL_Cluster_Nodes::get_peer_to_sync_variables_module(const char* module_name, char **host, uint16_t *port, char** ip_address) { + // Data-driven mapping of module names to their cluster configurations + struct VariablesModuleConfig { + const char* name; + std::atomic ProxySQL_Cluster::*diff_member; + std::function checksum_getter; + }; + + // Initialize all supported variables modules with their configuration + const VariablesModuleConfig modules[] = { + {"mysql_variables", &ProxySQL_Cluster::cluster_mysql_variables_diffs_before_sync, + [](ProxySQL_Node_Entry* node) { return &node->checksums_values.mysql_variables; }}, + {"admin_variables", &ProxySQL_Cluster::cluster_admin_variables_diffs_before_sync, + [](ProxySQL_Node_Entry* node) { return &node->checksums_values.admin_variables; }}, + {"ldap_variables", &ProxySQL_Cluster::cluster_ldap_variables_diffs_before_sync, + [](ProxySQL_Node_Entry* node) { return &node->checksums_values.ldap_variables; }}, + {"pgsql_variables", &ProxySQL_Cluster::cluster_pgsql_variables_diffs_before_sync, + [](ProxySQL_Node_Entry* node) { return &node->checksums_values.pgsql_variables; }} + }; + + // Find the matching module configuration + const VariablesModuleConfig* config = nullptr; + for (const auto& module : modules) { + if (strcmp(module_name, module.name) == 0) { + config = &module; + break; + } + } + + if (!config) { + proxy_error("Invalid module name supplied to get_peer_to_sync_variables_module: %s\n", module_name); + return; + } + unsigned long long version = 0; unsigned long long epoch = 0; unsigned long long max_epoch = 0; char *hostname = NULL; char* ip_addr = NULL; uint16_t p = 0; - unsigned int diff_mu = (unsigned int)__sync_fetch_and_add(&GloProxyCluster->cluster_mysql_variables_diffs_before_sync,0); + + // Get diff threshold using member pointer with atomic load + unsigned int diff_threshold = (unsigned int)(GloProxyCluster->*(config->diff_member)).load(); + for (std::unordered_map::iterator it = umap_proxy_nodes.begin(); it != umap_proxy_nodes.end();) { ProxySQL_Node_Entry * node = it->second; - ProxySQL_Checksum_Value_2 * v = &node->checksums_values.mysql_variables; + // Use function pointer to access the correct checksum field + ProxySQL_Checksum_Value_2 * v = config->checksum_getter(node); + if (v->version > 1) { if ( v->epoch > epoch ) { max_epoch = v->epoch; - if (v->diff_check >= diff_mu) { + if (v->diff_check >= diff_threshold) { epoch = v->epoch; version = v->version; if (hostname) { @@ -3312,11 +3927,13 @@ void ProxySQL_Cluster_Nodes::get_peer_to_sync_mysql_variables(char **host, uint1 } } } - it++; + + ++it; } + if (epoch) { if (max_epoch > epoch) { - proxy_warning("Cluster: detected a peer with mysql_variables epoch %llu, but not enough diff_check. We won't sync from epoch %llu: temporarily skipping sync\n", max_epoch, epoch); + proxy_warning("Cluster: detected a peer with %s epoch %llu, but not enough diff_check. We won't sync from epoch %llu: temporarily skipping sync\n", config->name, max_epoch, epoch); if (hostname) { free(hostname); hostname = NULL; @@ -3330,28 +3947,44 @@ void ProxySQL_Cluster_Nodes::get_peer_to_sync_mysql_variables(char **host, uint1 if (hostname) { *host = hostname; *port = p; - *ip_address = ip_addr; - proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with mysql_variables version %llu, epoch %llu\n", hostname, p, version, epoch); - proxy_info("Cluster: detected peer %s:%d with mysql_variables version %llu, epoch %llu\n", hostname, p, version, epoch); + if (ip_address) { + *ip_address = ip_addr; + } + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with %s version %llu, epoch %llu\n", hostname, p, config->name, version, epoch); + proxy_info("Cluster: detected peer %s:%d with %s version %llu, epoch %llu\n", hostname, p, config->name, version, epoch); } } +void ProxySQL_Cluster_Nodes::get_peer_to_sync_mysql_variables(char **host, uint16_t *port, char** ip_address) { + get_peer_to_sync_variables_module("mysql_variables", host, port, ip_address); +} + void ProxySQL_Cluster_Nodes::get_peer_to_sync_admin_variables(char **host, uint16_t *port, char** ip_address) { + get_peer_to_sync_variables_module("admin_variables", host, port, ip_address); +} + +void ProxySQL_Cluster_Nodes::get_peer_to_sync_ldap_variables(char **host, uint16_t *port, char** ip_address) { + get_peer_to_sync_variables_module("ldap_variables", host, port, ip_address); +} + +void ProxySQL_Cluster_Nodes::get_peer_to_sync_proxysql_servers(char **host, uint16_t *port, char** ip_address) { unsigned long long version = 0; unsigned long long epoch = 0; unsigned long long max_epoch = 0; char *hostname = NULL; char *ip_addr = NULL; uint16_t p = 0; - unsigned int diff_mu = (unsigned int)__sync_fetch_and_add(&GloProxyCluster->cluster_admin_variables_diffs_before_sync,0); - for (std::unordered_map::iterator it = umap_proxy_nodes.begin(); it != umap_proxy_nodes.end();) { +// pthread_mutex_lock(&mutex); + //unsigned long long curtime = monotonic_time(); + unsigned int diff_ps = (unsigned int)GloProxyCluster->cluster_proxysql_servers_diffs_before_sync; + for( std::unordered_map::iterator it = umap_proxy_nodes.begin(); it != umap_proxy_nodes.end(); ) { ProxySQL_Node_Entry * node = it->second; - ProxySQL_Checksum_Value_2 * v = &node->checksums_values.admin_variables; + ProxySQL_Checksum_Value_2 * v = &node->checksums_values.proxysql_servers; if (v->version > 1) { if ( v->epoch > epoch ) { max_epoch = v->epoch; - if (v->diff_check >= diff_mu) { + if (v->diff_check >= diff_ps) { epoch = v->epoch; version = v->version; if (hostname) { @@ -3370,9 +4003,10 @@ void ProxySQL_Cluster_Nodes::get_peer_to_sync_admin_variables(char **host, uint1 } it++; } +// pthread_mutex_unlock(&mutex); if (epoch) { if (max_epoch > epoch) { - proxy_warning("Cluster: detected a peer with admin_variables epoch %llu, but not enough diff_check. We won't sync from epoch %llu: temporarily skipping sync\n", max_epoch, epoch); + proxy_warning("Cluster: detected a peer with proxysql_servers epoch %llu , but not enough diff_check. We won't sync from epoch %llu: temporarily skipping sync\n", max_epoch, epoch); if (hostname) { free(hostname); hostname = NULL; @@ -3387,26 +4021,240 @@ void ProxySQL_Cluster_Nodes::get_peer_to_sync_admin_variables(char **host, uint1 *host = hostname; *port = p; *ip_address = ip_addr; - proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with admin_variables version %llu, epoch %llu\n", hostname, p, version, epoch); - proxy_info("Cluster: detected peer %s:%d with admin_variables version %llu, epoch %llu\n", hostname, p, version, epoch); + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with proxysql_servers version %llu, epoch %llu\n", hostname, p, version, epoch); + proxy_info("Cluster: detected peer %s:%d with proxysql_servers version %llu, epoch %llu\n", hostname, p, version, epoch); } } -void ProxySQL_Cluster_Nodes::get_peer_to_sync_ldap_variables(char **host, uint16_t *port, char** ip_address) { +/** + * @brief Helper function to safely update peer information with proper memory management. + * + * This function eliminates the common memory management pattern found in all peer + * selection functions. It safely frees existing allocations and creates new ones, + * handling error cases and preventing memory leaks. + * + * @param existing_hostname Pointer to existing hostname string (may be NULL) + * @param existing_ip_addr Pointer to existing IP address string (may be NULL) + * @param new_hostname New hostname to allocate (may be NULL) + * @param new_ip_addr New IP address to allocate (may be NULL) + * + * @return Returns true if allocation succeeded, false on memory allocation failure + * + * @note This function handles the common pattern where we need to replace existing + * hostname and ip_address allocations with new ones from a better peer + */ +static bool safe_update_peer_info(char** existing_hostname, char** existing_ip_addr, + const char* new_hostname, const char* new_ip_addr) { + // Free existing allocations + if (*existing_hostname) { + free(*existing_hostname); + *existing_hostname = NULL; + } + if (*existing_ip_addr) { + free(*existing_ip_addr); + *existing_ip_addr = NULL; + } + + // Allocate new values + if (new_hostname) { + *existing_hostname = strdup(new_hostname); + if (*existing_hostname == NULL) { + return false; // Memory allocation failed + } + } + if (new_ip_addr) { + *existing_ip_addr = strdup(new_ip_addr); + if (*existing_ip_addr == NULL) { + if (*existing_hostname) { + free(*existing_hostname); + *existing_hostname = NULL; + } + return false; // Memory allocation failed + } + } + + return true; +} + + +/** + * @brief Identifies the optimal cluster peer for PostgreSQL users synchronization. + * + * This function scans all available cluster nodes to find the best peer for synchronizing + * PostgreSQL users configuration. It selects a peer based on the following criteria: + * 1. The peer must have a valid pgsql_users checksum (version > 1) + * 2. The peer should have the latest epoch timestamp + * 3. The peer's diff_check count must exceed cluster_pgsql_users_diffs_before_sync threshold + * + * The algorithm prioritizes nodes with the most recent configuration changes while ensuring + * sufficient differences have accumulated to justify synchronization. This prevents excessive + * network traffic and unnecessary synchronization operations. + * + * @param host Pointer to store the hostname of the selected peer (caller must free) + * @param port Pointer to store the port number of the selected peer + * @param ip_address Pointer to store the IP address of the selected peer (caller must free) + * + * @note If no suitable peer is found, *host will be set to NULL + * @note If a peer has the maximum epoch but insufficient diff_check, a warning is logged and sync is skipped + * @note The function performs memory allocation for hostname and ip_address that must be freed by the caller + * @note This is a PostgreSQL counterpart to get_peer_to_sync_mysql_users() + * @see cluster_pgsql_users_diffs_before_sync + * @see ProxySQL_Checksum_Value_2::pgsql_users + */ +void ProxySQL_Cluster_Nodes::get_peer_to_sync_pgsql_users(char **host, uint16_t *port, char** ip_address) { unsigned long long version = 0; unsigned long long epoch = 0; unsigned long long max_epoch = 0; char *hostname = NULL; - char* ip_addr = NULL; + char *ip_addr = NULL; uint16_t p = 0; - unsigned int diff_mu = (unsigned int)__sync_fetch_and_add(&GloProxyCluster->cluster_ldap_variables_diffs_before_sync,0); - for (std::unordered_map::iterator it = umap_proxy_nodes.begin(); it != umap_proxy_nodes.end();) { + unsigned int diff_mu = (unsigned int)GloProxyCluster->cluster_pgsql_users_diffs_before_sync; + for( std::unordered_map::iterator it = umap_proxy_nodes.begin(); it != umap_proxy_nodes.end(); ) { ProxySQL_Node_Entry * node = it->second; - ProxySQL_Checksum_Value_2 * v = &node->checksums_values.ldap_variables; + ProxySQL_Checksum_Value_2 * v = &node->checksums_values.pgsql_users; if (v->version > 1) { if ( v->epoch > epoch ) { max_epoch = v->epoch; if (v->diff_check >= diff_mu) { + epoch = v->epoch; + version = v->version; + const char* ip = node->get_ipaddress(); + if (!safe_update_peer_info(&hostname, &ip_addr, node->get_hostname(), ip)) { + proxy_error("Memory allocation failed while updating pgsql_users peer info\n"); + return; + } + p = node->get_port(); + } + } + } + it++; + } + if (epoch) { + if (max_epoch > epoch) { + proxy_warning("Cluster: detected a peer with pgsql_users epoch %llu , but not enough diff_check. We won't sync from epoch %llu: temporarily skipping sync\n", max_epoch, epoch); + // Clean up allocated memory using helper function + safe_update_peer_info(&hostname, &ip_addr, NULL, NULL); + } + } + if (hostname) { + *host = hostname; + *port = p; + *ip_address = ip_addr; + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with pgsql_users version %llu, epoch %llu\n", hostname, p, version, epoch); + proxy_info("Cluster: detected peer %s:%d with pgsql_users version %llu, epoch %llu\n", hostname, p, version, epoch); + } +} + +/** + * @brief Identifies the optimal cluster peer for PostgreSQL query rules synchronization. + * + * This function scans all available cluster nodes to find the best peer for synchronizing + * PostgreSQL query rules configuration. It selects a peer based on the following criteria: + * 1. The peer must have a valid pgsql_query_rules checksum (version > 1) + * 2. The peer should have the latest epoch timestamp + * 3. The peer's diff_check count must exceed cluster_pgsql_query_rules_diffs_before_sync threshold + * + * The algorithm prioritizes nodes with the most recent query rules changes while ensuring + * sufficient differences have accumulated to justify synchronization. This prevents excessive + * network traffic and unnecessary synchronization operations for query rules that haven't + * changed significantly. + * + * @param host Pointer to store the hostname of the selected peer (caller must free) + * @param port Pointer to store the port number of the selected peer + * @param ip_address Pointer to store the IP address of the selected peer (caller must free) + * + * @note If no suitable peer is found, *host will be set to NULL + * @note If a peer has the maximum epoch but insufficient diff_check, a warning is logged and sync is skipped + * @note The function performs memory allocation for hostname and ip_address that must be freed by the caller + * @note This is a PostgreSQL counterpart to get_peer_to_sync_mysql_query_rules() + * @see cluster_pgsql_query_rules_diffs_before_sync + * @see ProxySQL_Checksum_Value_2::pgsql_query_rules + */ +void ProxySQL_Cluster_Nodes::get_peer_to_sync_pgsql_query_rules(char **host, uint16_t *port, char** ip_address) { + unsigned long long version = 0; + unsigned long long epoch = 0; + unsigned long long max_epoch = 0; + char *hostname = NULL; + char *ip_addr = NULL; + uint16_t p = 0; + unsigned int diff_mu = (unsigned int)GloProxyCluster->cluster_pgsql_query_rules_diffs_before_sync; + for( std::unordered_map::iterator it = umap_proxy_nodes.begin(); it != umap_proxy_nodes.end(); ) { + ProxySQL_Node_Entry * node = it->second; + ProxySQL_Checksum_Value_2 * v = &node->checksums_values.pgsql_query_rules; + if (v->version > 1) { + if ( v->epoch > epoch ) { + max_epoch = v->epoch; + if (v->diff_check >= diff_mu) { + epoch = v->epoch; + version = v->version; + const char* ip = node->get_ipaddress(); + if (!safe_update_peer_info(&hostname, &ip_addr, node->get_hostname(), ip)) { + proxy_error("Memory allocation failed while updating pgsql_query_rules peer info\n"); + return; + } + p = node->get_port(); + } + } + } + it++; + } + if (epoch) { + if (max_epoch > epoch) { + proxy_warning("Cluster: detected a peer with pgsql_query_rules epoch %llu , but not enough diff_check. We won't sync from epoch %llu: temporarily skipping sync\n", max_epoch, epoch); + // Clean up allocated memory using helper function + safe_update_peer_info(&hostname, &ip_addr, NULL, NULL); + } + } + if (hostname) { + *host = hostname; + *port = p; + *ip_address = ip_addr; + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with pgsql_query_rules version %llu, epoch %llu\n", hostname, p, version, epoch); + proxy_info("Cluster: detected peer %s:%d with pgsql_query_rules version %llu, epoch %llu\n", hostname, p, version, epoch); + } +} + +/** + * @brief Identifies the optimal cluster peer for runtime PostgreSQL servers synchronization. + * + * This function scans all available cluster nodes to find the best peer for synchronizing + * runtime PostgreSQL servers status and metrics. It selects a peer based on the following criteria: + * 1. The peer must have a valid pgsql_servers runtime checksum (version > 1) + * 2. The peer should have the latest epoch timestamp + * 3. The peer's diff_check count must exceed cluster_pgsql_servers_diffs_before_sync threshold + * + * The function focuses on runtime data synchronization, which includes server status, + * health metrics, connection counts, and other operational statistics. This enables + * cluster nodes to maintain consistent views of PostgreSQL server operational states. + * + * @param host Pointer to store the hostname of the selected peer (caller must free) + * @param port Pointer to store the port number of the selected peer + * @param peer_checksum Pointer to store the runtime checksum string of the selected peer (caller must free, optional) + * @param ip_address Pointer to store the IP address of the selected peer (caller must free) + * + * @note If no suitable peer is found, *host will be set to NULL + * @note If a peer has the maximum epoch but insufficient diff_check, a warning is logged and sync is skipped + * @note The function performs memory allocation for hostname, ip_address, and checksum that must be freed by the caller + * @note This is a PostgreSQL counterpart to get_peer_to_sync_runtime_mysql_servers() + * @see cluster_pgsql_servers_diffs_before_sync + * @see ProxySQL_Checksum_Value_2::pgsql_servers + */ +void ProxySQL_Cluster_Nodes::get_peer_to_sync_runtime_pgsql_servers(char **host, uint16_t *port, char **peer_checksum, char** ip_address) { + unsigned long long version = 0; + unsigned long long epoch = 0; + unsigned long long max_epoch = 0; + char *hostname = NULL; + char *ip_addr = NULL; + uint16_t p = 0; + char *checksum = NULL; + unsigned int diff_ms = (unsigned int)GloProxyCluster->cluster_pgsql_servers_diffs_before_sync; + for( std::unordered_map::iterator it = umap_proxy_nodes.begin(); it != umap_proxy_nodes.end(); ) { + ProxySQL_Node_Entry * node = it->second; + ProxySQL_Checksum_Value_2 * v = &node->checksums_values.pgsql_servers; + if (v->version > 1) { + if ( v->epoch > epoch ) { + max_epoch = v->epoch; + if (v->diff_check >= diff_ms) { epoch = v->epoch; version = v->version; if (hostname) { @@ -3415,11 +4263,15 @@ void ProxySQL_Cluster_Nodes::get_peer_to_sync_ldap_variables(char **host, uint16 if (ip_addr) { free(ip_addr); } + if (checksum) { + free(checksum); + } hostname=strdup(node->get_hostname()); const char* ip = node->get_ipaddress(); if (ip) ip_addr = strdup(ip); p = node->get_port(); + checksum = strdup(v->checksum); } } } @@ -3427,7 +4279,7 @@ void ProxySQL_Cluster_Nodes::get_peer_to_sync_ldap_variables(char **host, uint16 } if (epoch) { if (max_epoch > epoch) { - proxy_warning("Cluster: detected a peer with ldap_variables epoch %llu, but not enough diff_check. We won't sync from epoch %llu: temporarily skipping sync\n", max_epoch, epoch); + proxy_warning("Cluster: detected a peer with runtime_pgsql_servers epoch %llu , but not enough diff_check. We won't sync from epoch %llu: temporarily skipping sync\n", max_epoch, epoch); if (hostname) { free(hostname); hostname = NULL; @@ -3436,34 +4288,74 @@ void ProxySQL_Cluster_Nodes::get_peer_to_sync_ldap_variables(char **host, uint16 free(ip_addr); ip_addr = NULL; } + if (checksum) { + free(checksum); + checksum = NULL; + } } } if (hostname) { *host = hostname; *port = p; *ip_address = ip_addr; - proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with ldap_variables version %llu, epoch %llu\n", hostname, p, version, epoch); - proxy_info("Cluster: detected peer %s:%d with ldap_variables version %llu, epoch %llu\n", hostname, p, version, epoch); + if (peer_checksum) { + *peer_checksum = checksum; + } else { + if (checksum) + free(checksum); + } + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with runtime_pgsql_servers version %llu, epoch %llu\n", hostname, p, version, epoch); + proxy_info("Cluster: detected peer %s:%d with runtime_pgsql_servers version %llu, epoch %llu\n", hostname, p, version, epoch); } } -void ProxySQL_Cluster_Nodes::get_peer_to_sync_proxysql_servers(char **host, uint16_t *port, char** ip_address) { +/** + * @brief Identifies the optimal cluster peer for PostgreSQL servers v2 synchronization. + * + * This function scans all available cluster nodes to find the best peer for synchronizing + * PostgreSQL servers configuration. It selects a peer based on the following criteria: + * 1. The peer must have a valid pgsql_servers_v2 checksum (version > 1) + * 2. The peer should have the latest epoch timestamp + * 3. The peer's diff_check count must exceed cluster_pgsql_servers_diffs_before_sync threshold + * + * In addition to connection information, this function also provides checksums for both + * the static configuration (pgsql_servers_v2) and runtime status (pgsql_servers) if requested. + * This enables the caller to perform comprehensive synchronization of both configuration + * and runtime data. + * + * @param host Pointer to store the hostname of the selected peer (caller must free) + * @param port Pointer to store the port number of the selected peer + * @param peer_pgsql_servers_v2_checksum Pointer to store the pgsql_servers_v2 checksum string (caller must free, optional) + * @param peer_runtime_pgsql_servers_checksum Pointer to store the runtime pgsql_servers checksum string (caller must free, optional) + * @param ip_address Pointer to store the IP address of the selected peer (caller must free) + * + * @note If no suitable peer is found, *host will be set to NULL + * @note If a peer has the maximum epoch but insufficient diff_check, a warning is logged and sync is skipped + * @note The function performs memory allocation for all returned strings that must be freed by the caller + * @note Runtime checksum is only provided if the peer has valid runtime data (version > 1) + * @note This is a PostgreSQL counterpart to get_peer_to_sync_mysql_servers_v2() + * @see cluster_pgsql_servers_diffs_before_sync + * @see ProxySQL_Checksum_Value_2::pgsql_servers_v2 + * @see ProxySQL_Checksum_Value_2::pgsql_servers + */ +void ProxySQL_Cluster_Nodes::get_peer_to_sync_pgsql_servers_v2(char** host, uint16_t* port, char** peer_pgsql_servers_v2_checksum, + char** peer_runtime_pgsql_servers_checksum, char** ip_address) { unsigned long long version = 0; unsigned long long epoch = 0; unsigned long long max_epoch = 0; char *hostname = NULL; char *ip_addr = NULL; uint16_t p = 0; -// pthread_mutex_lock(&mutex); - //unsigned long long curtime = monotonic_time(); - unsigned int diff_ps = (unsigned int)__sync_fetch_and_add(&GloProxyCluster->cluster_proxysql_servers_diffs_before_sync,0); + char *checksum_v2 = NULL; + char *checksum_runtime = NULL; + unsigned int diff_ms = (unsigned int)GloProxyCluster->cluster_pgsql_servers_diffs_before_sync; for( std::unordered_map::iterator it = umap_proxy_nodes.begin(); it != umap_proxy_nodes.end(); ) { ProxySQL_Node_Entry * node = it->second; - ProxySQL_Checksum_Value_2 * v = &node->checksums_values.proxysql_servers; + ProxySQL_Checksum_Value_2 * v = &node->checksums_values.pgsql_servers_v2; if (v->version > 1) { if ( v->epoch > epoch ) { max_epoch = v->epoch; - if (v->diff_check >= diff_ps) { + if (v->diff_check >= diff_ms) { epoch = v->epoch; version = v->version; if (hostname) { @@ -3472,20 +4364,31 @@ void ProxySQL_Cluster_Nodes::get_peer_to_sync_proxysql_servers(char **host, uint if (ip_addr) { free(ip_addr); } + if (checksum_v2) { + free(checksum_v2); + } + if (checksum_runtime) { + free(checksum_runtime); + } hostname=strdup(node->get_hostname()); const char* ip = node->get_ipaddress(); if (ip) ip_addr = strdup(ip); p = node->get_port(); + checksum_v2 = strdup(v->checksum); + // Get runtime checksum as well + ProxySQL_Checksum_Value_2 * v_runtime = &node->checksums_values.pgsql_servers; + if (v_runtime->version > 1) { + checksum_runtime = strdup(v_runtime->checksum); + } } } } it++; } -// pthread_mutex_unlock(&mutex); if (epoch) { if (max_epoch > epoch) { - proxy_warning("Cluster: detected a peer with proxysql_servers epoch %llu , but not enough diff_check. We won't sync from epoch %llu: temporarily skipping sync\n", max_epoch, epoch); + proxy_warning("Cluster: detected a peer with pgsql_servers_v2 epoch %llu , but not enough diff_check. We won't sync from epoch %llu: temporarily skipping sync\n", max_epoch, epoch); if (hostname) { free(hostname); hostname = NULL; @@ -3494,14 +4397,34 @@ void ProxySQL_Cluster_Nodes::get_peer_to_sync_proxysql_servers(char **host, uint free(ip_addr); ip_addr = NULL; } + if (checksum_v2) { + free(checksum_v2); + checksum_v2 = NULL; + } + if (checksum_runtime) { + free(checksum_runtime); + checksum_runtime = NULL; + } } } if (hostname) { *host = hostname; *port = p; *ip_address = ip_addr; - proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with proxysql_servers version %llu, epoch %llu\n", hostname, p, version, epoch); - proxy_info("Cluster: detected peer %s:%d with proxysql_servers version %llu, epoch %llu\n", hostname, p, version, epoch); + if (peer_pgsql_servers_v2_checksum) { + *peer_pgsql_servers_v2_checksum = checksum_v2; + } else { + if (checksum_v2) + free(checksum_v2); + } + if (peer_runtime_pgsql_servers_checksum) { + *peer_runtime_pgsql_servers_checksum = checksum_runtime; + } else { + if (checksum_runtime) + free(checksum_runtime); + } + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Detected peer %s:%d with pgsql_servers_v2 version %llu, epoch %llu\n", hostname, p, version, epoch); + proxy_info("Cluster: detected peer %s:%d with pgsql_servers_v2 version %llu, epoch %llu\n", hostname, p, version, epoch); } } @@ -4265,6 +5188,15 @@ cluster_metrics_map = std::make_tuple( { "reason", "servers_share_epoch" } } ), + std::make_tuple ( + p_cluster_counter::sync_conflict_pgsql_variables_share_epoch, + "proxysql_cluster_syn_conflict_total", + "Number of times a 'module' has not been able to be synced.", + metric_tags { + { "module_name", "pgsql_variables" }, + { "reason", "servers_share_epoch" } + } + ), // ==================================================================== // sync_delayed due to version one @@ -4332,6 +5264,15 @@ cluster_metrics_map = std::make_tuple( { "reason", "version_one" } } ), + std::make_tuple ( + p_cluster_counter::sync_delayed_pgsql_variables_version_one, + "proxysql_cluster_syn_conflict_total", + "Number of times a 'module' has not been able to be synced.", + metric_tags { + { "module_name", "pgsql_variables" }, + { "reason", "version_one" } + } + ), // ==================================================================== }, cluster_gauge_vector {} diff --git a/test/tap/tests/test_cluster_sync_pgsql-t.cpp b/test/tap/tests/test_cluster_sync_pgsql-t.cpp new file mode 100644 index 0000000000..f09190076e --- /dev/null +++ b/test/tap/tests/test_cluster_sync_pgsql-t.cpp @@ -0,0 +1,231 @@ +/** + * @file test_cluster_sync_pgsql-t.cpp + * @brief Checks that ProxySQL PostgreSQL tables are properly syncing between cluster instances. + * @details Based on test_cluster_sync_mysql_servers-t.cpp, this test checks PostgreSQL cluster sync: + * - 'pgsql_servers_v2' sync between cluster nodes + * - 'pgsql_users' sync between cluster nodes + * - 'pgsql_query_rules' sync between cluster nodes + * - PostgreSQL modules checksums appear in runtime_checksums_values + * - Sync operation can be controlled via '%_diffs_before_sync' variables + * + * Test Cluster Isolation: + * ---------------------- + * For guaranteeing that this test doesn't invalidate the configuration of a running ProxySQL cluster and + * that after the test, the previous valid configuration is restored, the following actions are performed: + * + * 1. The Core nodes from the current cluster configuration are backup. + * 2. Primary (currently tested instance) is removed from the Core nodes. + * 3. A sync wait until all core nodes have performed the removal of primary is executed. + * 4. Now Primary is isolated from the previous cluster, tests can proceed. Primary is setup to hold itself + * in its 'proxysql_servers' as well as the target spawned replica. + * 5. After the tests recover the primary configuration and add it back to the Core nodes from Cluster: + * - Recover the previous 'pgsql_servers_v2' from disk, and load them to runtime, discarding any previous + * config performed during the test. + * - Insert the primary back into a Core node from cluster and wait for all nodes to sync including it. + * - Insert into the primary the previous backup Core nodes from Cluster and load to runtime. + */ + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "libconfig.h" + +#include "proxysql_utils.h" + +#include "mysql.h" +#ifndef SPOOKYV2 +#include "SpookyV2.h" +#define SPOOKYV2 +#endif +#include "tap.h" +#include "command_line.h" +#include "utils.h" + +using std::vector; +using std::string; + +int check_pgsql_servers_v2_sync( + const CommandLine& cl, MYSQL* proxy_admin, MYSQL* r_proxy_admin, + const vector>& insert_pgsql_servers_values +) { + // Configure 'pgsql_servers_v2' and check sync with NULL comments + const char* t_insert_pgsql_servers = + "INSERT INTO pgsql_servers_v2 (" + " hostgroup_id, hostname, port, status, weight, compression, max_connections," + " max_replication_lag, use_ssl, max_latency_ms, comment" + ") VALUES (%d, '%s', %d, '%s', %d, %d, %d, %d, %d, %d, '%s')"; + std::vector insert_pgsql_servers_queries {}; + + for (auto const& values : insert_pgsql_servers_values) { + std::string insert_pgsql_servers_query = ""; + string_format( + t_insert_pgsql_servers, + insert_pgsql_servers_query, + std::get<0>(values), // hostgroup_id + std::get<1>(values).c_str(), // hostname + std::get<2>(values), // port + std::get<3>(values).c_str(), // status + std::get<4>(values), // weight + std::get<5>(values), // compression + std::get<6>(values), // max_connections + std::get<7>(values), // max_replication_lag + std::get<8>(values), // use_ssl + std::get<9>(values), // max_latency_ms + std::get<10>(values), // comment + std::get<11>(values).c_str() + ); + insert_pgsql_servers_queries.push_back(insert_pgsql_servers_query); + } + + // Backup current table + MYSQL_QUERY(proxy_admin, "CREATE TABLE pgsql_servers_v2_sync_test AS SELECT * FROM pgsql_servers_v2"); + MYSQL_QUERY(proxy_admin, "DELETE FROM pgsql_servers_v2"); + + // Insert test data into primary + for (auto const& query : insert_pgsql_servers_queries) { + MYSQL_QUERY(proxy_admin, query.c_str()); + } + + // Load to runtime and verify sync + MYSQL_QUERY(proxy_admin, "LOAD PGSQL SERVERS TO RUNTIME"); + + // Wait for sync + sleep(5); + + // Check if data was synced to replica + for (auto const& values : insert_pgsql_servers_values) { + const char* t_select_pgsql_servers_inserted_entries = + "SELECT COUNT(*) FROM pgsql_servers_v2 WHERE hostgroup_id=%d AND hostname='%s'" + " AND port=%d AND status='%s' AND weight=%d AND" + " compression=%d AND max_connections=%d AND max_replication_lag=%d" + " AND use_ssl=%d AND max_latency_ms=%d AND comment='%s'"; + + std::string select_pgsql_servers_query = ""; + string_format( + t_select_pgsql_servers_inserted_entries, + select_pgsql_servers_query, + std::get<0>(values), + std::get<1>(values).c_str(), + std::get<2>(values), + std::get<3>(values).c_str(), + std::get<4>(values), + std::get<5>(values), + std::get<6>(values), + std::get<7>(values), + std::get<8>(values), + std::get<9>(values), + std::get<10>(values), + std::get<11>(values).c_str() + ); + + // Check on replica + MYSQL_RES* result = NULL; + MYSQL_QUERY(r_proxy_admin, select_pgsql_servers_query.c_str()); + result = mysql_store_result(r_proxy_admin); + int count = atoi(mysql_fetch_row(result)[0]); + mysql_free_result(result); + + if (count != 1) { + diag("PostgreSQL server sync failed for hostgroup %d, hostname %s", + std::get<0>(values), std::get<1>(values).c_str()); + return EXIT_FAILURE; + } + } + + // Restore original data + MYSQL_QUERY(proxy_admin, "DELETE FROM pgsql_servers_v2"); + MYSQL_QUERY(proxy_admin, "INSERT INTO pgsql_servers_v2 SELECT * FROM pgsql_servers_v2_sync_test"); + MYSQL_QUERY(proxy_admin, "DROP TABLE pgsql_servers_v2_sync_test"); + MYSQL_QUERY(proxy_admin, "LOAD PGSQL SERVERS TO RUNTIME"); + + return EXIT_SUCCESS; +} + +int check_pgsql_checksums_in_runtime_table(MYSQL* admin) { + const char* pgsql_checksums[] = { + "pgsql_query_rules", + "pgsql_servers", + "pgsql_servers_v2", + "pgsql_users", + "pgsql_variables" + }; + + for (const char* checksum_name : pgsql_checksums) { + const char* t_check_checksum = + "SELECT COUNT(*) FROM runtime_checksums_values WHERE name='%s'"; + + char query[256]; + snprintf(query, sizeof(query), t_check_checksum, checksum_name); + + MYSQL_QUERY(admin, query); + MYSQL_RES* result = mysql_store_result(admin); + int count = atoi(mysql_fetch_row(result)[0]); + mysql_free_result(result); + + if (count != 1) { + diag("PostgreSQL checksum '%s' not found in runtime_checksums_values", checksum_name); + return EXIT_FAILURE; + } + + ok(count == 1, "PostgreSQL checksum '%s' found in runtime_checksums_values", checksum_name); + } + + return EXIT_SUCCESS; +} + +int main(int argc, char** argv) { + CommandLine cl; + + if (cl.getEnv()) { + diag("Failed to get configuration from environment"); + return EXIT_FAILURE; + } + +plan(6); + + // Connect to admin interfaces + MYSQL* proxysql_admin = mysql_init(NULL); + if (!proxysql_admin) { + diag("mysql_init() failed"); + return exit_status(); + } + + if (!mysql_real_connect(proxysql_admin, cl.host, cl.admin_username, cl.admin_password, NULL, cl.admin_port, NULL, 0)) { + diag("Failed to connect to primary admin: %s", mysql_error(proxysql_admin)); + return exit_status(); + } + + // For this test, we'll just verify that PostgreSQL checksums are present + // In a full cluster test, we would connect to a replica and verify sync + + int res = check_pgsql_checksums_in_runtime_table(proxysql_admin); + ok(res == EXIT_SUCCESS, "PostgreSQL checksums are present in runtime_checksums_values"); + + // Test basic PostgreSQL configuration is supported + MYSQL_QUERY(proxysql_admin, "SELECT 1 FROM pgsql_servers LIMIT 1"); + ok(mysql_errno(proxysql_admin) == 0, "PostgreSQL servers table is accessible"); + + MYSQL_QUERY(proxysql_admin, "SELECT 1 FROM pgsql_users LIMIT 1"); + ok(mysql_errno(proxysql_admin) == 0, "PostgreSQL users table is accessible"); + + MYSQL_QUERY(proxysql_admin, "SELECT 1 FROM pgsql_query_rules LIMIT 1"); + ok(mysql_errno(proxysql_admin) == 0, "PostgreSQL query rules table is accessible"); + + // Check cluster variables exist + MYSQL_QUERY(proxysql_admin, "SHOW VARIABLES LIKE 'cluster_pgsql_%'"); + ok(mysql_errno(proxysql_admin) == 0, "PostgreSQL cluster variables are accessible"); + + mysql_close(proxysql_admin); + + return exit_status(); +} \ No newline at end of file