Handle various (unlikely) failure states

dbutils.c

@@ -952,7 +952,7 @@ _get_primary_connection(PGconn *conn,
 					  "  SELECT node_id, conninfo, "
 					  "         CASE WHEN type = 'primary' THEN 1 ELSE 2 END AS type_priority"
 					  "	   FROM repmgr.nodes "
-					  "   WHERE type != 'witness' "
+					  "   WHERE active IS TRUE "
 					  "ORDER BY active DESC, type_priority, priority, node_id");
 
 	log_verbose(LOG_DEBUG, "get_primary_connection():\n%s", query.data);
@@ -1030,7 +1030,6 @@ get_primary_connection(PGconn *conn,
 }
 
 
-
 PGconn *
 get_primary_connection_quiet(PGconn *conn,
 							int *primary_id, char *primary_conninfo_out)
@@ -1038,6 +1037,7 @@ get_primary_connection_quiet(PGconn *conn,
 	return _get_primary_connection(conn, primary_id, primary_conninfo_out, true);
 }
 
+
 /*
  * Return the id of the active primary node, or NODE_NOT_FOUND if no
  * record available.

repmgrd.c

@@ -885,7 +885,9 @@ monitor_streaming_standby(void)
 		{
 			int		degraded_monitoring_elapsed = calculate_elapsed(degraded_monitoring_start);
 
-			log_debug("monitoring node in degraded state for %i seconds", degraded_monitoring_elapsed);
+			log_debug("monitoring node %i in degraded state for %i seconds",
+					  upstream_node_info.node_id,
+					  degraded_monitoring_elapsed);
 
 			if (is_server_available(upstream_node_info.conninfo) == true)
 			{
@@ -908,7 +910,7 @@ monitor_streaming_standby(void)
 					else
 					{
 
-						if (primary_conn == NULL ||PQstatus(primary_conn) != CONNECTION_OK)
+						if (primary_conn == NULL || PQstatus(primary_conn) != CONNECTION_OK)
 						{
 							primary_conn = establish_primary_db_connection(upstream_conn, false);
 						}
@@ -1180,12 +1182,24 @@ do_primary_failover(void)
 		case FAILOVER_STATE_PROMOTION_FAILED:
 			log_debug("failover state is PROMOTION FAILED");
 			return false;
+
+		case FAILOVER_STATE_FOLLOW_FAIL:
+			/*
+			 * for whatever reason we were unable to follow the new primary -
+			 * continue monitoring in degraded state
+			 */
+			monitoring_state = MS_DEGRADED;
+			INSTR_TIME_SET_CURRENT(degraded_monitoring_start);
+
+			return false;
+
 		case FAILOVER_STATE_NO_NEW_PRIMARY:
 		case FAILOVER_STATE_WAITING_NEW_PRIMARY:
 			/* pass control back down to start_monitoring() */
 			// -> should kick off new election
 			return false;
 
+		case FAILOVER_STATE_NODE_NOTIFICATION_ERROR:
 		case FAILOVER_STATE_LOCAL_NODE_FAILURE:
 		case FAILOVER_STATE_UNKNOWN:
 		case FAILOVER_STATE_NONE:
@@ -1378,9 +1392,15 @@ promote_self(void)
 		sleep(config_file_options.promote_delay);
 	}
 
-	// XXX check success
 	record_status = get_node_record(local_conn, local_node_info.upstream_node_id, &failed_primary);
 
+	if (record_status != RECORD_FOUND)
+	{
+		log_error(_("unable to retrieve metadata record for failed upstream (ID: %i)"),
+				  local_node_info.upstream_node_id);
+		return FAILOVER_STATE_PROMOTION_FAILED;
+	}
+
 	/* the presence of either of these commands has been established already */
 	if (config_file_options.service_promote_command[0] != '\0')
 		promote_command = config_file_options.service_promote_command;
@@ -1597,11 +1617,24 @@ follow_new_primary(int new_primary_id)
 	RecordStatus record_status;
 	bool new_primary_ok = false;
 
-	// XXX check success
 	record_status = get_node_record(local_conn, new_primary_id, &new_primary);
 
+	if (record_status != RECORD_FOUND)
+	{
+		log_error(_("unable to retrieve metadata record for upstream node (ID: %i)"),
+				  new_primary_id);
+		return FAILOVER_STATE_FOLLOW_FAIL;
+	}
+
 	record_status = get_node_record(local_conn, local_node_info.upstream_node_id, &failed_primary);
 
+	if (record_status != RECORD_FOUND)
+	{
+		log_error(_("unable to retrieve metadata record for failed primary (ID: %i)"),
+					local_node_info.upstream_node_id);
+		return FAILOVER_STATE_FOLLOW_FAIL;
+	}
+
 	// XXX check if new_primary_id == failed_primary.node_id?
 
 	if (log_type == REPMGR_STDERR && *config_file_options.log_file)
@@ -1612,13 +1645,6 @@ follow_new_primary(int new_primary_id)
 	log_debug(_("standby follow command is:\n  \"%s\""),
 			  config_file_options.follow_command);
 
-	/*
-	 * disconnect from local node, as follow operation will result in
-	 * a server restart
-	 */
-	PQfinish(local_conn);
-	local_conn = NULL;
-
 	upstream_conn = establish_db_connection(new_primary.conninfo, false);
 
 	if (PQstatus(upstream_conn) == CONNECTION_OK)
@@ -1635,12 +1661,20 @@ follow_new_primary(int new_primary_id)
 		}
 	}
 
-
 	if (new_primary_ok == false)
 	{
 		return FAILOVER_STATE_FOLLOW_FAIL;
 	}
-	// XXX check new primary is reachable and is not in recovery here
+
+	/*
+	 * disconnect from local node, as follow operation will result in
+	 * a server restart
+	 */
+
+	PQfinish(local_conn);
+	local_conn = NULL;
+
+	/* execute the follow command */
 	r = system(config_file_options.follow_command);
 
 	if (r != 0)
@@ -1682,10 +1716,23 @@ follow_new_primary(int new_primary_id)
 	 * directly from the primary to ensure they're the current version
 	 */
 
-	// XXX check success
-
 	record_status = get_node_record(upstream_conn, new_primary_id, &upstream_node_info);
+
+	if (record_status != RECORD_FOUND)
+	{
+		log_error(_("unable to retrieve metadata record found for node %i"),
+				  new_primary_id);
+		return FAILOVER_STATE_FOLLOW_FAIL;
+	}
+
 	record_status = get_node_record(upstream_conn, local_node_info.node_id, &local_node_info);
+	if (record_status != RECORD_FOUND)
+	{
+		log_error(_("unable to retrieve metadata record found for node %i"),
+				  local_node_info.node_id);
+		return FAILOVER_STATE_FOLLOW_FAIL;
+	}
+
 
 	local_conn = establish_db_connection(local_node_info.conninfo, false);
 	initPQExpBuffer(&event_details);
@@ -1836,8 +1883,6 @@ do_election(void)
 	electoral_term = set_voting_status_initiated(local_conn);
 
 	/* get all active nodes attached to primary, excluding self */
-	// XXX include barman node in results
-
 	get_active_sibling_node_records(local_conn,
 									local_node_info.node_id,
 									upstream_node_info.node_id,
@@ -1856,7 +1901,6 @@ do_election(void)
 		/* assume the worst case */
 		cell->node_info->is_visible = false;
 
-		// XXX handle witness-barman
 		cell->node_info->conn = establish_db_connection(cell->node_info->conninfo, false);
 
 		if (PQstatus(cell->node_info->conn) != CONNECTION_OK)
@@ -2201,8 +2245,12 @@ try_reconnect(const char *conninfo, NodeStatus *node_status)
 		{
 			log_notice(_("node has recovered, reconnecting"));
 
-			// XXX how to handle case where node is reachable
+			/*
-			// but connection denied due to connection exhaustion
+			 * XXX we should also handle the case where node is pingable
+			 * but connection denied due to connection exhaustion
+			 *  - fall back to degraded monitoring?
+			 *  - make that configurable
+			 */
 			conn = establish_db_connection(conninfo, false);
 			if (PQstatus(conn) == CONNECTION_OK)
 			{
@@ -2213,7 +2261,7 @@ try_reconnect(const char *conninfo, NodeStatus *node_status)
 			PQfinish(conn);
 			log_notice(_("unable to reconnect to node"));
 		}
-		log_info(_("sleeping %i seconds until next reconnection_attempt"),
+		log_info(_("sleeping %i seconds until next reconnection attempt"),
 				 config_file_options.reconnect_interval);
 		sleep(config_file_options.reconnect_interval);
 	}