github/repmgr
1
0
Fork 0
mirror of https://github.com/EnterpriseDB/repmgr.git synced 2026-03-22 22:56:29 +00:00
Code Issues Packages Projects Releases 1 Wiki Activity

Handle various (unlikely) failure states

Browse Source
This commit is contained in:
Ian Barwick
2017-07-10 09:00:18 +09:00
parent 5bf7098139
commit 9e3d942917
2 changed files with 71 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
dbutils.c
View File

@@ -952,7 +952,7 @@ _get_primary_connection(PGconn *conn,
" SELECT node_id, conninfo, "
" CASE WHEN type = 'primary' THEN 1 ELSE 2 END AS type_priority"
" FROM repmgr.nodes "
" WHERE type != 'witness' "
" WHERE active IS TRUE "
"ORDER BY active DESC, type_priority, priority, node_id");
log_verbose(LOG_DEBUG, "get_primary_connection():\n%s", query.data);
@@ -1030,7 +1030,6 @@ get_primary_connection(PGconn *conn,
}
PGconn *
get_primary_connection_quiet(PGconn *conn,
int *primary_id, char *primary_conninfo_out)
@@ -1038,6 +1037,7 @@ get_primary_connection_quiet(PGconn *conn,
return _get_primary_connection(conn, primary_id, primary_conninfo_out, true);
}
/*
* Return the id of the active primary node, or NODE_NOT_FOUND if no
* record available.

90
repmgrd.c
View File

@@ -885,7 +885,9 @@ monitor_streaming_standby(void)
{
int degraded_monitoring_elapsed = calculate_elapsed(degraded_monitoring_start);
log_debug("monitoring node in degraded state for %i seconds", degraded_monitoring_elapsed);
log_debug("monitoring node %i in degraded state for %i seconds",
upstream_node_info.node_id,
degraded_monitoring_elapsed);
if (is_server_available(upstream_node_info.conninfo) == true)
{
@@ -908,7 +910,7 @@ monitor_streaming_standby(void)
else
{
if (primary_conn == NULL ||PQstatus(primary_conn) != CONNECTION_OK)
if (primary_conn == NULL || PQstatus(primary_conn) != CONNECTION_OK)
{
primary_conn = establish_primary_db_connection(upstream_conn, false);
}
@@ -1180,12 +1182,24 @@ do_primary_failover(void)
case FAILOVER_STATE_PROMOTION_FAILED:
log_debug("failover state is PROMOTION FAILED");
return false;
case FAILOVER_STATE_FOLLOW_FAIL:
/*
* for whatever reason we were unable to follow the new primary -
* continue monitoring in degraded state
*/
monitoring_state = MS_DEGRADED;
INSTR_TIME_SET_CURRENT(degraded_monitoring_start);
return false;
case FAILOVER_STATE_NO_NEW_PRIMARY:
case FAILOVER_STATE_WAITING_NEW_PRIMARY:
/* pass control back down to start_monitoring() */
// -> should kick off new election
return false;
case FAILOVER_STATE_NODE_NOTIFICATION_ERROR:
case FAILOVER_STATE_LOCAL_NODE_FAILURE:
case FAILOVER_STATE_UNKNOWN:
case FAILOVER_STATE_NONE:
@@ -1378,9 +1392,15 @@ promote_self(void)
sleep(config_file_options.promote_delay);
}
// XXX check success
record_status = get_node_record(local_conn, local_node_info.upstream_node_id, &failed_primary);
if (record_status != RECORD_FOUND)
{
log_error(_("unable to retrieve metadata record for failed upstream (ID: %i)"),
local_node_info.upstream_node_id);
return FAILOVER_STATE_PROMOTION_FAILED;
}
/* the presence of either of these commands has been established already */
if (config_file_options.service_promote_command[0] != '\0')
promote_command = config_file_options.service_promote_command;
@@ -1597,11 +1617,24 @@ follow_new_primary(int new_primary_id)
RecordStatus record_status;
bool new_primary_ok = false;
// XXX check success
record_status = get_node_record(local_conn, new_primary_id, &new_primary);
if (record_status != RECORD_FOUND)
{
log_error(_("unable to retrieve metadata record for upstream node (ID: %i)"),
new_primary_id);
return FAILOVER_STATE_FOLLOW_FAIL;
}
record_status = get_node_record(local_conn, local_node_info.upstream_node_id, &failed_primary);
if (record_status != RECORD_FOUND)
{
log_error(_("unable to retrieve metadata record for failed primary (ID: %i)"),
local_node_info.upstream_node_id);
return FAILOVER_STATE_FOLLOW_FAIL;
}
// XXX check if new_primary_id == failed_primary.node_id?
if (log_type == REPMGR_STDERR && *config_file_options.log_file)
@@ -1612,13 +1645,6 @@ follow_new_primary(int new_primary_id)
log_debug(_("standby follow command is:\n \"%s\""),
config_file_options.follow_command);
/*
* disconnect from local node, as follow operation will result in
* a server restart
*/
PQfinish(local_conn);
local_conn = NULL;
upstream_conn = establish_db_connection(new_primary.conninfo, false);
if (PQstatus(upstream_conn) == CONNECTION_OK)
@@ -1635,12 +1661,20 @@ follow_new_primary(int new_primary_id)
}
}
if (new_primary_ok == false)
{
return FAILOVER_STATE_FOLLOW_FAIL;
}
// XXX check new primary is reachable and is not in recovery here
/*
* disconnect from local node, as follow operation will result in
* a server restart
*/
PQfinish(local_conn);
local_conn = NULL;
/* execute the follow command */
r = system(config_file_options.follow_command);
if (r != 0)
@@ -1682,10 +1716,23 @@ follow_new_primary(int new_primary_id)
* directly from the primary to ensure they're the current version
*/
// XXX check success
record_status = get_node_record(upstream_conn, new_primary_id, &upstream_node_info);
if (record_status != RECORD_FOUND)
{
log_error(_("unable to retrieve metadata record found for node %i"),
new_primary_id);
return FAILOVER_STATE_FOLLOW_FAIL;
}
record_status = get_node_record(upstream_conn, local_node_info.node_id, &local_node_info);
if (record_status != RECORD_FOUND)
{
log_error(_("unable to retrieve metadata record found for node %i"),
local_node_info.node_id);
return FAILOVER_STATE_FOLLOW_FAIL;
}
local_conn = establish_db_connection(local_node_info.conninfo, false);
initPQExpBuffer(&event_details);
@@ -1836,8 +1883,6 @@ do_election(void)
electoral_term = set_voting_status_initiated(local_conn);
/* get all active nodes attached to primary, excluding self */
// XXX include barman node in results
get_active_sibling_node_records(local_conn,
local_node_info.node_id,
upstream_node_info.node_id,
@@ -1856,7 +1901,6 @@ do_election(void)
/* assume the worst case */
cell->node_info->is_visible = false;
// XXX handle witness-barman
cell->node_info->conn = establish_db_connection(cell->node_info->conninfo, false);
if (PQstatus(cell->node_info->conn) != CONNECTION_OK)
@@ -2201,8 +2245,12 @@ try_reconnect(const char *conninfo, NodeStatus *node_status)
{
log_notice(_("node has recovered, reconnecting"));
// XXX how to handle case where node is reachable
// but connection denied due to connection exhaustion
/*
* XXX we should also handle the case where node is pingable
* but connection denied due to connection exhaustion
* - fall back to degraded monitoring?
* - make that configurable
*/
conn = establish_db_connection(conninfo, false);
if (PQstatus(conn) == CONNECTION_OK)
{
@@ -2213,7 +2261,7 @@ try_reconnect(const char *conninfo, NodeStatus *node_status)
PQfinish(conn);
log_notice(_("unable to reconnect to node"));
}
log_info(_("sleeping %i seconds until next reconnection_attempt"),
log_info(_("sleeping %i seconds until next reconnection attempt"),
config_file_options.reconnect_interval);
sleep(config_file_options.reconnect_interval);
}