repmgrd: fix upstream role check

Only take action if it's confirmed as a standby.

HISTORY

@@ -1,3 +1,14 @@
+4.2.0   2018-??-??
+        repmgr: add parameter "shutdown_check_timeout" for use by "standby switchover";
+          GitHub #504 (Ian)
+        repmgr: add "--node-id" option to "repmgr cluster cleanup"; GitHub #493 (Ian)
+        repmgr: report unreachable nodes when running "repmgr cluster (matrix|crosscheck);
+          GitHub #246 (Ian)
+        repmgr: add configuration file parameter "repmgr_bindir"; GitHub #246 (Ian)
+        repmgr: fix "Missing replication slots" label in "node check"; GitHub #507 (Ian)
+        repmgrd: fix parsing of -d/--daemonize option (Ian)
+        repmgrd: support "pausing" of repmgrd (Ian)
+
 4.1.1   2018-09-05
         logging: explicitly log the text of failed queries as ERRORs to
           assist logfile analysis; GitHub #498

Makefile.in

@@ -13,8 +13,9 @@ DATA = \
   repmgr--unpackaged--4.0.sql \
   repmgr--4.0.sql \
   repmgr--4.0--4.1.sql \
-  repmgr--4.1.sql
+  repmgr--4.1.sql \
-
+  repmgr--4.1--4.2.sql \
+  repmgr--4.2.sql
 
 REGRESS = repmgr_extension
 
@@ -29,19 +30,24 @@ all: \
 PG_CPPFLAGS = -std=gnu89 -I$(includedir_internal) -I$(libpq_srcdir) -Wall -Wmissing-prototypes -Wmissing-declarations $(EXTRA_CFLAGS)
 SHLIB_LINK = $(libpq)
 
-HEADERS = $(wildcard *.h)
+
 
 OBJS = \
 	repmgr.o
 
 include Makefile.global
 
+ifeq ($(vpath_build),yes)
+	HEADERS = $(wildcard *.h)
+else
+	HEADERS_built = $(wildcard *.h)
+endif
 
 $(info Building against PostgreSQL $(MAJORVERSION))
 
 REPMGR_CLIENT_OBJS = repmgr-client.o \
 	repmgr-action-primary.o repmgr-action-standby.o repmgr-action-witness.o \
-	repmgr-action-bdr.o repmgr-action-cluster.o repmgr-action-node.o \
+	repmgr-action-bdr.o repmgr-action-cluster.o repmgr-action-node.o repmgr-action-daemon.o \
 	configfile.o log.o strutil.o controldata.o dirutil.o compat.o dbutils.o
 REPMGRD_OBJS = repmgrd.o repmgrd-physical.o repmgrd-bdr.o configfile.o log.o dbutils.o strutil.o controldata.o compat.o
 DATE=$(shell date "+%Y-%m-%d")
@@ -85,6 +91,7 @@ additional-clean:
 	rm -f repmgr-action-bdr.o
 	rm -f repmgr-action-node.o
 	rm -f repmgr-action-cluster.o
+	rm -f repmgr-action-daemon.o
 	rm -f repmgrd.o
 	rm -f repmgrd-physical.o
 	rm -f repmgrd-bdr.o

configfile.c

@@ -88,8 +88,7 @@ load_config(const char *config_file, bool verbose, bool terse, t_configuration_o
 
 			if (pwd != NULL)
 			{
-				appendPQExpBuffer(&fullpath,
+				appendPQExpBufferStr(&fullpath, pwd);
-								  "%s", pwd);
 			}
 			else
 			{
@@ -105,9 +104,7 @@ load_config(const char *config_file, bool verbose, bool terse, t_configuration_o
 					exit(ERR_BAD_CONFIG);
 				}
 
-				appendPQExpBuffer(&fullpath,
+				appendPQExpBufferStr(&fullpath, cwd);
-								  "%s",
-								  cwd);
 			}
 
 			appendPQExpBuffer(&fullpath,
@@ -288,6 +285,7 @@ _parse_config(t_configuration_options *options, ItemList *error_list, ItemList *
 	memset(options->data_directory, 0, sizeof(options->data_directory));
 	memset(options->config_directory, 0, sizeof(options->data_directory));
 	memset(options->pg_bindir, 0, sizeof(options->pg_bindir));
+	memset(options->repmgr_bindir, 0, sizeof(options->repmgr_bindir));
 	options->replication_type = REPLICATION_TYPE_PHYSICAL;
 
 	/*-------------
@@ -335,6 +333,7 @@ _parse_config(t_configuration_options *options, ItemList *error_list, ItemList *
 	 * standby switchover settings
 	 *------------------------
 	 */
+	options->shutdown_check_timeout = DEFAULT_SHUTDOWN_CHECK_TIMEOUT;
 	options->standby_reconnect_timeout = DEFAULT_STANDBY_RECONNECT_TIMEOUT;
 
 	/*-----------------
@@ -489,6 +488,8 @@ _parse_config(t_configuration_options *options, ItemList *error_list, ItemList *
 		}
 		else if (strcmp(name, "pg_bindir") == 0)
 			strncpy(options->pg_bindir, value, MAXPGPATH);
+		else if (strcmp(name, "repmgr_bindir") == 0)
+			strncpy(options->repmgr_bindir, value, MAXPGPATH);
 
 		else if (strcmp(name, "replication_type") == 0)
 		{
@@ -545,6 +546,8 @@ _parse_config(t_configuration_options *options, ItemList *error_list, ItemList *
 			options->standby_follow_timeout = repmgr_atoi(value, name, error_list, 0);
 
 		/* standby switchover settings */
+		else if (strcmp(name, "shutdown_check_timeout") == 0)
+			options->shutdown_check_timeout = repmgr_atoi(value, name, error_list, 0);
 		else if (strcmp(name, "standby_reconnect_timeout") == 0)
 			options->standby_reconnect_timeout = repmgr_atoi(value, name, error_list, 0);
 
@@ -1105,8 +1108,8 @@ reload_config(t_configuration_options *orig_options, t_server_type server_type)
 
 		initPQExpBuffer(&errors);
 
-		appendPQExpBuffer(&errors,
+		appendPQExpBufferStr(&errors,
-						  "following errors were detected:\n");
+							 "following errors were detected:\n");
 
 		for (cell = config_errors.head; cell; cell = cell->next)
 		{
@@ -1528,6 +1531,9 @@ parse_bool(const char *s, const char *config_item, ItemList *error_list)
 {
 	PQExpBufferData errors;
 
+	if (s == NULL)
+		return true;
+
 	if (strcasecmp(s, "0") == 0)
 		return false;
 

configfile.h

@@ -75,6 +75,7 @@ typedef struct
 	char		data_directory[MAXPGPATH];
 	char		config_directory[MAXPGPATH];
 	char		pg_bindir[MAXPGPATH];
+	char		repmgr_bindir[MAXPGPATH];
 	int			replication_type;
 
 	/* log settings */
@@ -103,6 +104,7 @@ typedef struct
 	int			standby_follow_timeout;
 
 	/* standby switchover settings */
+	int			shutdown_check_timeout;
 	int			standby_reconnect_timeout;
 
 	/* node rejoin settings */
@@ -170,7 +172,7 @@ typedef struct
 
 #define T_CONFIGURATION_OPTIONS_INITIALIZER { \
 		/* node information */ \
-		UNKNOWN_NODE_ID, "", "", "", "", "", "", REPLICATION_TYPE_PHYSICAL,	\
+		UNKNOWN_NODE_ID, "", "", "", "", "", "", "", REPLICATION_TYPE_PHYSICAL,	\
 		/* log settings */ \
 		"", "", "", DEFAULT_LOG_STATUS_INTERVAL,	\
 		/* standby clone settings */ \
@@ -181,6 +183,7 @@ typedef struct
 		DEFAULT_PRIMARY_FOLLOW_TIMEOUT,	\
 		DEFAULT_STANDBY_FOLLOW_TIMEOUT,	\
 		/* standby switchover settings */ \
+		DEFAULT_SHUTDOWN_CHECK_TIMEOUT, \
 		DEFAULT_STANDBY_RECONNECT_TIMEOUT, \
 		/* node rejoin settings */ \
 		DEFAULT_NODE_REJOIN_TIMEOUT, \

configure

@@ -1,6 +1,6 @@
 #! /bin/sh
 # Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.69 for repmgr 4.1.2.
+# Generated by GNU Autoconf 2.69 for repmgr 4.2.
 #
 # Report bugs to <pgsql-bugs@postgresql.org>.
 #
@@ -582,8 +582,8 @@ MAKEFLAGS=
 # Identity of this package.
 PACKAGE_NAME='repmgr'
 PACKAGE_TARNAME='repmgr'
-PACKAGE_VERSION='4.1.2'
+PACKAGE_VERSION='4.2'
-PACKAGE_STRING='repmgr 4.1.2'
+PACKAGE_STRING='repmgr 4.2'
 PACKAGE_BUGREPORT='pgsql-bugs@postgresql.org'
 PACKAGE_URL='https://2ndquadrant.com/en/resources/repmgr/'
 
@@ -1178,7 +1178,7 @@ if test "$ac_init_help" = "long"; then
   # Omit some internal or obsolete options to make the list less imposing.
   # This message is too long to be a string in the A/UX 3.1 sh.
   cat <<_ACEOF
-\`configure' configures repmgr 4.1.2 to adapt to many kinds of systems.
+\`configure' configures repmgr 4.2 to adapt to many kinds of systems.
 
 Usage: $0 [OPTION]... [VAR=VALUE]...
 
@@ -1239,7 +1239,7 @@ fi
 
 if test -n "$ac_init_help"; then
   case $ac_init_help in
-     short | recursive ) echo "Configuration of repmgr 4.1.2:";;
+     short | recursive ) echo "Configuration of repmgr 4.2:";;
    esac
   cat <<\_ACEOF
 
@@ -1313,7 +1313,7 @@ fi
 test -n "$ac_init_help" && exit $ac_status
 if $ac_init_version; then
   cat <<\_ACEOF
-repmgr configure 4.1.2
+repmgr configure 4.2
 generated by GNU Autoconf 2.69
 
 Copyright (C) 2012 Free Software Foundation, Inc.
@@ -1332,7 +1332,7 @@ cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.
 
-It was created by repmgr $as_me 4.1.2, which was
+It was created by repmgr $as_me 4.2, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   $ $0 $@
@@ -2359,7 +2359,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 # report actual input values of CONFIG_FILES etc. instead of their
 # values after options handling.
 ac_log="
-This file was extended by repmgr $as_me 4.1.2, which was
+This file was extended by repmgr $as_me 4.2, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   CONFIG_FILES    = $CONFIG_FILES
@@ -2422,7 +2422,7 @@ _ACEOF
 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
 ac_cs_version="\\
-repmgr config.status 4.1.2
+repmgr config.status 4.2
 configured by $0, generated by GNU Autoconf 2.69,
   with options \\"\$ac_cs_config\\"
 

configure.in

@@ -1,4 +1,4 @@
-AC_INIT([repmgr], [4.1.2], [pgsql-bugs@postgresql.org], [repmgr], [https://2ndquadrant.com/en/resources/repmgr/])
+AC_INIT([repmgr], [4.2], [pgsql-bugs@postgresql.org], [repmgr], [https://2ndquadrant.com/en/resources/repmgr/])
 
 AC_COPYRIGHT([Copyright (c) 2010-2018, 2ndQuadrant Ltd.])
 

dbutils.c

@@ -115,20 +115,20 @@ wrap_ddl_query(PQExpBufferData *query_buf, int replication_type, const char *fmt
 	if (replication_type == REPLICATION_TYPE_BDR)
 	{
 		if (bdr_version_num < 3)
-			appendPQExpBuffer(query_buf, "SELECT bdr.bdr_replicate_ddl_command($repmgr$");
+			appendPQExpBufferStr(query_buf, "SELECT bdr.bdr_replicate_ddl_command($repmgr$");
 		else
-			appendPQExpBuffer(query_buf, "SELECT bdr.replicate_ddl_command($repmgr$");
+			appendPQExpBufferStr(query_buf, "SELECT bdr.replicate_ddl_command($repmgr$");
 	}
 
 	va_start(arglist, fmt);
 	vsnprintf(buf, MAXLEN, fmt, arglist);
 	va_end(arglist);
 
-	appendPQExpBuffer(query_buf, "%s", buf);
+	appendPQExpBufferStr(query_buf, buf);
 
 	if (replication_type == REPLICATION_TYPE_BDR)
 	{
-		appendPQExpBuffer(query_buf, "$repmgr$)");
+		appendPQExpBufferStr(query_buf, "$repmgr$)");
 	}
 }
 
@@ -1070,9 +1070,9 @@ get_cluster_size(PGconn *conn, char *size)
 	bool		success = true;
 
 	initPQExpBuffer(&query);
-	appendPQExpBuffer(&query,
+	appendPQExpBufferStr(&query,
-					  "SELECT pg_catalog.pg_size_pretty(pg_catalog.sum(pg_catalog.pg_database_size(oid))::bigint) "
+						 "SELECT pg_catalog.pg_size_pretty(pg_catalog.sum(pg_catalog.pg_database_size(oid))::bigint) "
-					  "	 FROM pg_catalog.pg_database ");
+						 "	 FROM pg_catalog.pg_database ");
 
 	log_verbose(LOG_DEBUG, "get_cluster_size():\n%s", query.data);
 
@@ -1228,13 +1228,13 @@ _get_primary_connection(PGconn *conn,
 	log_verbose(LOG_INFO, _("searching for primary node"));
 
 	initPQExpBuffer(&query);
-	appendPQExpBuffer(&query,
+	appendPQExpBufferStr(&query,
-					  "  SELECT node_id, conninfo, "
+						 "  SELECT node_id, conninfo, "
-					  "         CASE WHEN type = 'primary' THEN 1 ELSE 2 END AS type_priority"
+						 "         CASE WHEN type = 'primary' THEN 1 ELSE 2 END AS type_priority"
-					  "	   FROM repmgr.nodes "
+						 "	   FROM repmgr.nodes "
-					  "   WHERE active IS TRUE "
+						 "   WHERE active IS TRUE "
-					  "     AND type != 'witness' "
+						 "     AND type != 'witness' "
-					  "ORDER BY active DESC, type_priority, priority, node_id");
+						 "ORDER BY active DESC, type_priority, priority, node_id");
 
 	log_verbose(LOG_DEBUG, "get_primary_connection():\n%s", query.data);
 
@@ -1341,11 +1341,11 @@ get_primary_node_id(PGconn *conn)
 	int			retval = NODE_NOT_FOUND;
 
 	initPQExpBuffer(&query);
-	appendPQExpBuffer(&query,
+	appendPQExpBufferStr(&query,
-					  "SELECT node_id		  "
+						 "SELECT node_id		  "
-					  "	 FROM repmgr.nodes    "
+						 "	 FROM repmgr.nodes    "
-					  " WHERE type = 'primary' "
+						 " WHERE type = 'primary' "
-					  "   AND active IS TRUE  ");
+						 "   AND active IS TRUE  ");
 
 	log_verbose(LOG_DEBUG, "get_primary_node_id():\n%s", query.data);
 
@@ -1384,38 +1384,38 @@ get_replication_info(PGconn *conn, ReplInfo *replication_info)
 		server_version_num = get_server_version(conn, NULL);
 
 	initPQExpBuffer(&query);
-	appendPQExpBuffer(&query,
+	appendPQExpBufferStr(&query,
-					  " SELECT ts, "
+						 " SELECT ts, "
-					  "        last_wal_receive_lsn, "
+						 "        last_wal_receive_lsn, "
-					  "        last_wal_replay_lsn, "
+						 "        last_wal_replay_lsn, "
-					  "        last_xact_replay_timestamp, "
+						 "        last_xact_replay_timestamp, "
-					  "        CASE WHEN (last_wal_receive_lsn = last_wal_replay_lsn) "
+						 "        CASE WHEN (last_wal_receive_lsn = last_wal_replay_lsn) "
-					  "          THEN 0::INT "
+						 "          THEN 0::INT "
-					  "        ELSE "
+						 "        ELSE "
-					  "          EXTRACT(epoch FROM (pg_catalog.clock_timestamp() - last_xact_replay_timestamp))::INT "
+						 "          EXTRACT(epoch FROM (pg_catalog.clock_timestamp() - last_xact_replay_timestamp))::INT "
-					  "        END AS replication_lag_time, "
+						 "        END AS replication_lag_time, "
-					  "        COALESCE(last_wal_receive_lsn, '0/0') >= last_wal_replay_lsn AS receiving_streamed_wal "
+						 "        COALESCE(last_wal_receive_lsn, '0/0') >= last_wal_replay_lsn AS receiving_streamed_wal "
-					  "   FROM ( ");
+						 "   FROM ( ");
 
 	if (server_version_num >= 100000)
 	{
-		appendPQExpBuffer(&query,
+		appendPQExpBufferStr(&query,
-						  " SELECT CURRENT_TIMESTAMP AS ts, "
+							 " SELECT CURRENT_TIMESTAMP AS ts, "
-						  "        pg_catalog.pg_last_wal_receive_lsn()       AS last_wal_receive_lsn, "
+							 "        pg_catalog.pg_last_wal_receive_lsn()       AS last_wal_receive_lsn, "
-						  "        pg_catalog.pg_last_wal_replay_lsn()        AS last_wal_replay_lsn, "
+							 "        pg_catalog.pg_last_wal_replay_lsn()        AS last_wal_replay_lsn, "
-						  "        pg_catalog.pg_last_xact_replay_timestamp() AS last_xact_replay_timestamp ");
+							 "        pg_catalog.pg_last_xact_replay_timestamp() AS last_xact_replay_timestamp ");
 	}
 	else
 	{
-		appendPQExpBuffer(&query,
+		appendPQExpBufferStr(&query,
-						  " SELECT CURRENT_TIMESTAMP AS ts, "
+							 " SELECT CURRENT_TIMESTAMP AS ts, "
-						  "        pg_catalog.pg_last_xlog_receive_location() AS last_wal_receive_lsn, "
+							 "        pg_catalog.pg_last_xlog_receive_location() AS last_wal_receive_lsn, "
-						  "        pg_catalog.pg_last_xlog_replay_location()  AS last_wal_replay_lsn, "
+							 "        pg_catalog.pg_last_xlog_replay_location()  AS last_wal_replay_lsn, "
-						  "        pg_catalog.pg_last_xact_replay_timestamp() AS last_xact_replay_timestamp ");
+							 "        pg_catalog.pg_last_xact_replay_timestamp() AS last_xact_replay_timestamp ");
 	}
 
-	appendPQExpBuffer(&query,
+	appendPQExpBufferStr(&query,
-					  "          ) q ");
+						 "          ) q ");
 
 	log_verbose(LOG_DEBUG, "get_replication_info():\n%s", query.data);
 
@@ -1538,21 +1538,21 @@ get_replication_lag_seconds(PGconn *conn)
 
 	if (server_version_num >= 100000)
 	{
-		appendPQExpBuffer(&query,
+		appendPQExpBufferStr(&query,
-						  " SELECT CASE WHEN (pg_catalog.pg_last_wal_receive_lsn() = pg_catalog.pg_last_wal_replay_lsn()) ");
+							 " SELECT CASE WHEN (pg_catalog.pg_last_wal_receive_lsn() = pg_catalog.pg_last_wal_replay_lsn()) ");
 
 	}
 	else
 	{
-		appendPQExpBuffer(&query,
+		appendPQExpBufferStr(&query,
-						  " SELECT CASE WHEN (pg_catalog.pg_last_xlog_receive_location() = pg_catalog.pg_last_xlog_replay_location()) ");
+							 " SELECT CASE WHEN (pg_catalog.pg_last_xlog_receive_location() = pg_catalog.pg_last_xlog_replay_location()) ");
 	}
 
-	appendPQExpBuffer(&query,
+	appendPQExpBufferStr(&query,
-					  "          THEN 0 "
+						 "          THEN 0 "
-					  "        ELSE EXTRACT(epoch FROM (pg_catalog.clock_timestamp() - pg_catalog.pg_last_xact_replay_timestamp()))::INT "
+						 "        ELSE EXTRACT(epoch FROM (pg_catalog.clock_timestamp() - pg_catalog.pg_last_xact_replay_timestamp()))::INT "
-					  "          END "
+						 "          END "
-					  "        AS lag_seconds");
+						 "        AS lag_seconds");
 
 	res = PQexec(conn, query.data);
 	log_verbose(LOG_DEBUG, "get_replication_lag_seconds():\n%s", query.data);
@@ -1627,7 +1627,6 @@ repmgrd_set_local_node_id(PGconn *conn, int local_node_id)
 }
 
 
-
 int
 repmgrd_get_local_node_id(PGconn *conn)
 {
@@ -1686,6 +1685,135 @@ server_in_exclusive_backup_mode(PGconn *conn)
 }
 
 
+void
+repmgrd_set_pid(PGconn *conn, pid_t repmgrd_pid, const char *pidfile)
+{
+	PQExpBufferData query;
+	PGresult   *res = NULL;
+
+	log_verbose(LOG_DEBUG, "repmgrd_set_pid(): pid is %i", (int) repmgrd_pid);
+
+	initPQExpBuffer(&query);
+
+	appendPQExpBuffer(&query,
+					  "SELECT repmgr.set_repmgrd_pid(%i, '%s')",
+					  (int) repmgrd_pid, pidfile);
+
+	res = PQexec(conn, query.data);
+	termPQExpBuffer(&query);
+
+	if (PQresultStatus(res) != PGRES_TUPLES_OK)
+	{
+		log_error(_("unable to execute \"SELECT repmgr.set_repmgrd_pid()\""));
+		log_detail("%s", PQerrorMessage(conn));
+	}
+
+	PQclear(res);
+
+	return;
+}
+
+
+pid_t
+repmgrd_get_pid(PGconn *conn)
+{
+	PGresult   *res = NULL;
+	pid_t		repmgrd_pid = UNKNOWN_PID;
+
+	res = PQexec(conn, "SELECT repmgr.get_repmgrd_pid()");
+
+	if (PQresultStatus(res) != PGRES_TUPLES_OK)
+	{
+		log_error(_("unable to execute \"SELECT repmgr.get_repmgrd_pid()\""));
+		log_detail("%s", PQerrorMessage(conn));
+	}
+	else if (!PQgetisnull(res, 0, 0))
+	{
+		repmgrd_pid = atoi(PQgetvalue(res, 0, 0));
+	}
+
+	PQclear(res);
+
+	return repmgrd_pid;
+}
+
+
+bool
+repmgrd_is_running(PGconn *conn)
+{
+	PGresult   *res = NULL;
+	bool		is_running = false;
+
+	res = PQexec(conn, "SELECT repmgr.repmgrd_is_running()");
+
+	if (PQresultStatus(res) != PGRES_TUPLES_OK)
+	{
+		log_error(_("unable to execute \"SELECT repmgr.repmgrd_is_running()\""));
+		log_detail("%s", PQerrorMessage(conn));
+	}
+	else if (!PQgetisnull(res, 0, 0))
+	{
+		is_running = atobool(PQgetvalue(res, 0, 0));
+	}
+
+	PQclear(res);
+
+	return is_running;
+}
+
+
+bool
+repmgrd_is_paused(PGconn *conn)
+{
+	PGresult   *res = NULL;
+	bool		is_paused = false;
+
+	res = PQexec(conn, "SELECT repmgr.repmgrd_is_paused()");
+
+	if (PQresultStatus(res) != PGRES_TUPLES_OK)
+	{
+		log_error(_("unable to execute \"SELECT repmgr.repmgrd_is_paused()\""));
+		log_detail("%s", PQerrorMessage(conn));
+	}
+	else if (!PQgetisnull(res, 0, 0))
+	{
+		is_paused = atobool(PQgetvalue(res, 0, 0));
+	}
+
+	PQclear(res);
+
+	return is_paused;
+}
+
+
+bool
+repmgrd_pause(PGconn *conn, bool pause)
+{
+	PQExpBufferData query;
+	PGresult   *res = NULL;
+	bool		success = true;
+
+	initPQExpBuffer(&query);
+
+	appendPQExpBuffer(&query,
+					  "SELECT repmgr.repmgrd_pause(%s)",
+					  pause == true ? "TRUE" : "FALSE");
+	res = PQexec(conn, query.data);
+	termPQExpBuffer(&query);
+
+	if (PQresultStatus(res) != PGRES_TUPLES_OK)
+	{
+		log_error(_("unable to execute \"SELECT repmgr.repmgrd_pause()\""));
+		log_detail("%s", PQerrorMessage(conn));
+
+		success = false;
+	}
+
+	PQclear(res);
+
+	return success;
+}
+
 /* ================ */
 /* result functions */
 /* ================ */
@@ -1714,12 +1842,12 @@ get_repmgr_extension_status(PGconn *conn)
 
 	initPQExpBuffer(&query);
 
-	appendPQExpBuffer(&query,
+	appendPQExpBufferStr(&query,
-					  "	  SELECT ae.name, e.extname "
+						 "	  SELECT ae.name, e.extname "
-					  "     FROM pg_catalog.pg_available_extensions ae "
+						 "     FROM pg_catalog.pg_available_extensions ae "
-					  "LEFT JOIN pg_catalog.pg_extension e "
+						 "LEFT JOIN pg_catalog.pg_extension e "
-					  "       ON e.extname=ae.name "
+						 "       ON e.extname=ae.name "
-					  "	   WHERE ae.name='repmgr' ");
+						 "	   WHERE ae.name='repmgr' ");
 
 	res = PQexec(conn, query.data);
 
@@ -2098,38 +2226,39 @@ _populate_node_records(PGresult *res, NodeInfoList *node_list)
 }
 
 
-void
+bool
 get_all_node_records(PGconn *conn, NodeInfoList *node_list)
 {
 	PQExpBufferData query;
 	PGresult   *res = NULL;
-
+	bool success = true;
 	initPQExpBuffer(&query);
 
-	appendPQExpBuffer(&query,
+	appendPQExpBufferStr(&query,
-					  "  SELECT " REPMGR_NODES_COLUMNS
+						 "  SELECT " REPMGR_NODES_COLUMNS
-					  "    FROM repmgr.nodes n "
+						 "    FROM repmgr.nodes n "
-					  "ORDER BY n.node_id ");
+						 "ORDER BY n.node_id ");
 
 	log_verbose(LOG_DEBUG, "get_all_node_records():\n%s", query.data);
 
 	res = PQexec(conn, query.data);
 
-	if (PQresultStatus(res) != PGRES_TUPLES_OK)
-	{
-		log_db_error(conn, query.data, _("get_all_node_records(): unable to execute query"));
-	}
-
-	termPQExpBuffer(&query);
-
 	/* this will return an empty list if there was an error executing the query */
 	_populate_node_records(res, node_list);
 
-	PQclear(res);
+	if (PQresultStatus(res) != PGRES_TUPLES_OK)
+	{
+		log_db_error(conn, query.data, _("get_all_node_records(): unable to execute query"));
+		success = false;
+	}
 
-	return;
+	PQclear(res);
+	termPQExpBuffer(&query);
+
+	return success;
 }
 
+
 void
 get_downstream_node_records(PGconn *conn, int node_id, NodeInfoList *node_list)
 {
@@ -2211,10 +2340,10 @@ get_node_records_by_priority(PGconn *conn, NodeInfoList *node_list)
 
 	initPQExpBuffer(&query);
 
-	appendPQExpBuffer(&query,
+	appendPQExpBufferStr(&query,
-					  "  SELECT " REPMGR_NODES_COLUMNS
+						 "  SELECT " REPMGR_NODES_COLUMNS
-					  "    FROM repmgr.nodes n "
+						 "    FROM repmgr.nodes n "
-					  "ORDER BY n.priority DESC, n.node_name ");
+						 "ORDER BY n.priority DESC, n.node_name ");
 
 	log_verbose(LOG_DEBUG, "get_node_records_by_priority():\n%s", query.data);
 
@@ -2248,13 +2377,13 @@ get_all_node_records_with_upstream(PGconn *conn, NodeInfoList *node_list)
 
 	initPQExpBuffer(&query);
 
-	appendPQExpBuffer(&query,
+	appendPQExpBufferStr(&query,
-					  "    SELECT n.node_id, n.type, n.upstream_node_id, n.node_name, n.conninfo, n.repluser, "
+						 "    SELECT n.node_id, n.type, n.upstream_node_id, n.node_name, n.conninfo, n.repluser, "
-					  "           n.slot_name, n.location, n.priority, n.active, n.config_file, un.node_name AS upstream_node_name "
+						 "           n.slot_name, n.location, n.priority, n.active, n.config_file, un.node_name AS upstream_node_name "
-					  "      FROM repmgr.nodes n "
+						 "      FROM repmgr.nodes n "
-					  " LEFT JOIN repmgr.nodes un "
+						 " LEFT JOIN repmgr.nodes un "
-					  "        ON un.node_id = n.upstream_node_id"
+						 "        ON un.node_id = n.upstream_node_id"
-					  "  ORDER BY n.node_id ");
+						 "  ORDER BY n.node_id ");
 
 	log_verbose(LOG_DEBUG, "get_all_node_records_with_upstream():\n%s", query.data);
 
@@ -2397,31 +2526,30 @@ _create_update_node_record(PGconn *conn, char *action, t_node_info *node_info)
 
 	if (strcmp(action, "create") == 0)
 	{
-		appendPQExpBuffer(&query,
+		appendPQExpBufferStr(&query,
-						  "INSERT INTO repmgr.nodes "
+							 "INSERT INTO repmgr.nodes "
-						  "       (node_id, type, upstream_node_id, "
+							 "       (node_id, type, upstream_node_id, "
-						  "        node_name, conninfo, repluser, slot_name, "
+							 "        node_name, conninfo, repluser, slot_name, "
-						  "        location, priority, active, config_file) "
+							 "        location, priority, active, config_file) "
-						  "VALUES ($11, $1, $2, $3, $4, $5, $6, $7, $8, $9, $10) ");
+							 "VALUES ($11, $1, $2, $3, $4, $5, $6, $7, $8, $9, $10) ");
 	}
 	else
 	{
-		appendPQExpBuffer(&query,
+		appendPQExpBufferStr(&query,
-						  "UPDATE repmgr.nodes SET "
+							 "UPDATE repmgr.nodes SET "
-						  "       type = $1, "
+							 "       type = $1, "
-						  "       upstream_node_id = $2, "
+							 "       upstream_node_id = $2, "
-						  "       node_name = $3, "
+							 "       node_name = $3, "
-						  "       conninfo = $4, "
+							 "       conninfo = $4, "
-						  "       repluser = $5, "
+							 "       repluser = $5, "
-						  "       slot_name = $6, "
+							 "       slot_name = $6, "
-						  "       location = $7, "
+							 "       location = $7, "
-						  "       priority = $8, "
+							 "       priority = $8, "
-						  "       active = $9, "
+							 "       active = $9, "
-						  "       config_file = $10 "
+							 "       config_file = $10 "
-						  " WHERE node_id = $11 ");
+							 " WHERE node_id = $11 ");
 	}
 
-
 	res = PQexecParams(conn,
 					   query.data,
 					   param_count,
@@ -2858,31 +2986,31 @@ get_node_replication_stats(PGconn *conn, int server_version_num, t_node_info *no
 
 	initPQExpBuffer(&query);
 
-	appendPQExpBuffer(&query,
+	appendPQExpBufferStr(&query,
-					  " SELECT pg_catalog.current_setting('max_wal_senders')::INT AS max_wal_senders, "
+						 " SELECT pg_catalog.current_setting('max_wal_senders')::INT AS max_wal_senders, "
-					  "        (SELECT pg_catalog.count(*) FROM pg_catalog.pg_stat_replication) AS attached_wal_receivers, ");
+						 "        (SELECT pg_catalog.count(*) FROM pg_catalog.pg_stat_replication) AS attached_wal_receivers, ");
 
 	/* no replication slots in PostgreSQL 9.3 */
 	if (server_version_num < 90400)
 	{
-		appendPQExpBuffer(&query,
+		appendPQExpBufferStr(&query,
-						  "        0 AS max_replication_slots, "
+							 "        0 AS max_replication_slots, "
-						  "        0 AS total_replication_slots, "
+							 "        0 AS total_replication_slots, "
-						  "        0 AS active_replication_slots, "
+							 "        0 AS active_replication_slots, "
-						  "        0 AS inactive_replication_slots, ");
+							 "        0 AS inactive_replication_slots, ");
 	}
 	else
 	{
-		appendPQExpBuffer(&query,
+		appendPQExpBufferStr(&query,
-						  "        current_setting('max_replication_slots')::INT AS max_replication_slots, "
+							 "        current_setting('max_replication_slots')::INT AS max_replication_slots, "
-						  "        (SELECT pg_catalog.count(*) FROM pg_catalog.pg_replication_slots) AS total_replication_slots, "
+							 "        (SELECT pg_catalog.count(*) FROM pg_catalog.pg_replication_slots) AS total_replication_slots, "
-						  "        (SELECT pg_catalog.count(*) FROM pg_catalog.pg_replication_slots WHERE active IS TRUE)  AS active_replication_slots, "
+							 "        (SELECT pg_catalog.count(*) FROM pg_catalog.pg_replication_slots WHERE active IS TRUE)  AS active_replication_slots, "
-						  "        (SELECT pg_catalog.count(*) FROM pg_catalog.pg_replication_slots WHERE active IS FALSE) AS inactive_replication_slots, ");
+							 "        (SELECT pg_catalog.count(*) FROM pg_catalog.pg_replication_slots WHERE active IS FALSE) AS inactive_replication_slots, ");
 	}
 
 
-	appendPQExpBuffer(&query,
+	appendPQExpBufferStr(&query,
-					  "        pg_catalog.pg_is_in_recovery() AS in_recovery");
+						 "        pg_catalog.pg_is_in_recovery() AS in_recovery");
 
 	log_verbose(LOG_DEBUG, "get_node_replication_stats():\n%s", query.data);
 
@@ -3023,26 +3151,26 @@ get_datadir_configuration_files(PGconn *conn, KeyValueList *list)
 
 	initPQExpBuffer(&query);
 
-	appendPQExpBuffer(&query,
+	appendPQExpBufferStr(&query,
-					  "WITH files AS ( "
+						 "WITH files AS ( "
-					  "  WITH dd AS ( "
+						 "  WITH dd AS ( "
-					  "   SELECT setting "
+						 "   SELECT setting "
-					  "     FROM pg_catalog.pg_settings "
+						 "     FROM pg_catalog.pg_settings "
-					  "    WHERE name = 'data_directory') "
+						 "    WHERE name = 'data_directory') "
-					  "   SELECT distinct(sourcefile) AS config_file"
+						 "   SELECT distinct(sourcefile) AS config_file"
-					  "     FROM dd, pg_catalog.pg_settings ps "
+						 "     FROM dd, pg_catalog.pg_settings ps "
-					  "    WHERE ps.sourcefile IS NOT NULL "
+						 "    WHERE ps.sourcefile IS NOT NULL "
-					  "      AND ps.sourcefile ~ ('^' || dd.setting) "
+						 "      AND ps.sourcefile ~ ('^' || dd.setting) "
-					  "       UNION "
+						 "       UNION "
-					  "   SELECT ps.setting  AS config_file"
+						 "   SELECT ps.setting  AS config_file"
-					  "     FROM dd, pg_catalog.pg_settings ps "
+						 "     FROM dd, pg_catalog.pg_settings ps "
-					  "    WHERE ps.name IN ('config_file', 'hba_file', 'ident_file') "
+						 "    WHERE ps.name IN ('config_file', 'hba_file', 'ident_file') "
-					  "      AND ps.setting ~ ('^' || dd.setting) "
+						 "      AND ps.setting ~ ('^' || dd.setting) "
-					  ") "
+						 ") "
-					  "  SELECT config_file, "
+						 "  SELECT config_file, "
-					  "         pg_catalog.regexp_replace(config_file, '^.*\\/','') AS filename "
+						 "         pg_catalog.regexp_replace(config_file, '^.*\\/','') AS filename "
-					  "    FROM files "
+						 "    FROM files "
-					  "ORDER BY config_file");
+						 "ORDER BY config_file");
 
 	res = PQexec(conn, query.data);
 
@@ -3079,18 +3207,18 @@ get_configuration_file_locations(PGconn *conn, t_configfile_list *list)
 
 	initPQExpBuffer(&query);
 
-	appendPQExpBuffer(&query,
+	appendPQExpBufferStr(&query,
-					  "  WITH dd AS ( "
+						 "  WITH dd AS ( "
-					  "    SELECT setting AS data_directory"
+						 "    SELECT setting AS data_directory"
-					  "      FROM pg_catalog.pg_settings "
+						 "      FROM pg_catalog.pg_settings "
-					  "     WHERE name = 'data_directory' "
+						 "     WHERE name = 'data_directory' "
-					  "  ) "
+						 "  ) "
-					  "    SELECT DISTINCT(sourcefile), "
+						 "    SELECT DISTINCT(sourcefile), "
-					  "           pg_catalog.regexp_replace(sourcefile, '^.*\\/', '') AS filename, "
+						 "           pg_catalog.regexp_replace(sourcefile, '^.*\\/', '') AS filename, "
-					  "           sourcefile ~ ('^' || dd.data_directory) AS in_data_dir "
+						 "           sourcefile ~ ('^' || dd.data_directory) AS in_data_dir "
-					  "      FROM dd, pg_catalog.pg_settings ps "
+						 "      FROM dd, pg_catalog.pg_settings ps "
-					  "     WHERE sourcefile IS NOT NULL "
+						 "     WHERE sourcefile IS NOT NULL "
-					  "  ORDER BY 1 ");
+						 "  ORDER BY 1 ");
 
 	log_verbose(LOG_DEBUG, "get_configuration_file_locations():\n  %s",
 				query.data);
@@ -3129,19 +3257,18 @@ get_configuration_file_locations(PGconn *conn, t_configfile_list *list)
 	/* Fetch locations of pg_hba.conf and pg_ident.conf */
 	initPQExpBuffer(&query);
 
-	appendPQExpBuffer(&query,
+	appendPQExpBufferStr(&query,
-					  "  WITH dd AS ( "
+						 "  WITH dd AS ( "
-					  "    SELECT setting AS data_directory"
+						 "    SELECT setting AS data_directory"
-					  "      FROM pg_catalog.pg_settings "
+						 "      FROM pg_catalog.pg_settings "
-					  "     WHERE name = 'data_directory' "
+						 "     WHERE name = 'data_directory' "
-					  "  ) "
+						 "  ) "
-					  "    SELECT ps.setting, "
+						 "    SELECT ps.setting, "
-					  "           pg_catalog.regexp_replace(setting, '^.*\\/', '') AS filename, "
+						 "           pg_catalog.regexp_replace(setting, '^.*\\/', '') AS filename, "
-					  "           ps.setting ~ ('^' || dd.data_directory) AS in_data_dir "
+						 "           ps.setting ~ ('^' || dd.data_directory) AS in_data_dir "
-					  "      FROM dd, pg_catalog.pg_settings ps "
+						 "      FROM dd, pg_catalog.pg_settings ps "
-					  "     WHERE ps.name IN ('hba_file', 'ident_file') "
+						 "     WHERE ps.name IN ('hba_file', 'ident_file') "
-					  "  ORDER BY 1 ");
+						 "  ORDER BY 1 ");
-
 
 	log_verbose(LOG_DEBUG, "get_configuration_file_locations():\n  %s",
 				query.data);
@@ -3315,15 +3442,15 @@ _create_event(PGconn *conn, t_configuration_options *options, int node_id, char
 		int			binary[4] = {1, 0, 0, 0};
 
 		initPQExpBuffer(&query);
-		appendPQExpBuffer(&query,
+		appendPQExpBufferStr(&query,
-						  " INSERT INTO repmgr.events ( "
+							 " INSERT INTO repmgr.events ( "
-						  "             node_id, "
+							 "             node_id, "
-						  "             event, "
+							 "             event, "
-						  "             successful, "
+							 "             successful, "
-						  "             details "
+							 "             details "
-						  "            ) "
+							 "            ) "
-						  "      VALUES ($1, $2, $3, $4) "
+							 "      VALUES ($1, $2, $3, $4) "
-						  "   RETURNING event_timestamp ");
+							 "   RETURNING event_timestamp ");
 
 		log_verbose(LOG_DEBUG, "_create_event():\n  %s", query.data);
 
@@ -3554,12 +3681,12 @@ get_event_records(PGconn *conn, int node_id, const char *node_name, const char *
 	initPQExpBuffer(&where_clause);
 
 	/* LEFT JOIN used here as a node record may have been removed */
-	appendPQExpBuffer(&query,
+	appendPQExpBufferStr(&query,
-					  "   SELECT e.node_id, n.node_name, e.event, e.successful, "
+						 "   SELECT e.node_id, n.node_name, e.event, e.successful, "
-					  "          pg_catalog.to_char(e.event_timestamp, 'YYYY-MM-DD HH24:MI:SS') AS timestamp, "
+						 "          pg_catalog.to_char(e.event_timestamp, 'YYYY-MM-DD HH24:MI:SS') AS timestamp, "
-					  "          e.details "
+						 "          e.details "
-					  "     FROM repmgr.events e "
+						 "     FROM repmgr.events e "
-					  "LEFT JOIN repmgr.nodes n ON e.node_id = n.node_id ");
+						 "LEFT JOIN repmgr.nodes n ON e.node_id = n.node_id ");
 
 	if (node_id != UNKNOWN_NODE_ID)
 	{
@@ -3605,8 +3732,8 @@ get_event_records(PGconn *conn, int node_id, const char *node_name, const char *
 	appendPQExpBuffer(&query, "\n%s\n",
 					  where_clause.data);
 
-	appendPQExpBuffer(&query,
+	appendPQExpBufferStr(&query,
-					  " ORDER BY e.event_timestamp DESC");
+						 " ORDER BY e.event_timestamp DESC");
 
 	if (all == false && limit > 0)
 	{
@@ -3808,10 +3935,10 @@ get_free_replication_slot_count(PGconn *conn)
 
 	initPQExpBuffer(&query);
 
-	appendPQExpBuffer(&query,
+	appendPQExpBufferStr(&query,
-					  " SELECT pg_catalog.current_setting('max_replication_slots')::INT - "
+						 " SELECT pg_catalog.current_setting('max_replication_slots')::INT - "
-					  "        pg_catalog.count(*) AS free_slots"
+						 "        pg_catalog.count(*) AS free_slots"
-					  "   FROM pg_catalog.pg_replication_slots");
+						 "   FROM pg_catalog.pg_replication_slots");
 
 	res = PQexec(conn, query.data);
 
@@ -3847,11 +3974,11 @@ get_inactive_replication_slots(PGconn *conn, KeyValueList *list)
 
 	initPQExpBuffer(&query);
 
-	appendPQExpBuffer(&query,
+	appendPQExpBufferStr(&query,
-					  "   SELECT slot_name, slot_type "
+						 "   SELECT slot_name, slot_type "
-					  "     FROM pg_catalog.pg_replication_slots "
+						 "     FROM pg_catalog.pg_replication_slots "
-					  "    WHERE active IS FALSE "
+						 "    WHERE active IS FALSE "
-					  " ORDER BY slot_name ");
+						 " ORDER BY slot_name ");
 
 	res = PQexec(conn, query.data);
 
@@ -4166,7 +4293,7 @@ add_monitoring_record(PGconn *primary_conn,
 
 
 int
-get_number_of_monitoring_records_to_delete(PGconn *primary_conn, int keep_history)
+get_number_of_monitoring_records_to_delete(PGconn *primary_conn, int keep_history, int node_id)
 {
 	PQExpBufferData query;
 	int				record_count = -1;
@@ -4180,8 +4307,15 @@ get_number_of_monitoring_records_to_delete(PGconn *primary_conn, int keep_histor
 					  " WHERE pg_catalog.age(pg_catalog.now(), last_monitor_time) >= '%d days'::interval",
 					  keep_history);
 
-	res = PQexec(primary_conn, query.data);
+	if (node_id != UNKNOWN_NODE_ID)
+	{
+		appendPQExpBuffer(&query,
+						  "  AND standby_node_id = %i", node_id);
+	}
 
+	log_verbose(LOG_DEBUG, "get_number_of_monitoring_records_to_delete():\n  %s", query.data);
+
+	res = PQexec(primary_conn, query.data);
 
 	if (PQresultStatus(res) != PGRES_TUPLES_OK)
 	{
@@ -4201,7 +4335,7 @@ get_number_of_monitoring_records_to_delete(PGconn *primary_conn, int keep_histor
 
 
 bool
-delete_monitoring_records(PGconn *primary_conn, int keep_history)
+delete_monitoring_records(PGconn *primary_conn, int keep_history, int node_id)
 {
 	PQExpBufferData query;
 	bool			success = true;
@@ -4209,17 +4343,23 @@ delete_monitoring_records(PGconn *primary_conn, int keep_history)
 
 	initPQExpBuffer(&query);
 
-	if (keep_history > 0)
+	if (keep_history > 0 || node_id != UNKNOWN_NODE_ID)
 	{
 		appendPQExpBuffer(&query,
 						  "DELETE FROM repmgr.monitoring_history "
 						  " WHERE pg_catalog.age(pg_catalog.now(), last_monitor_time) >= '%d days'::INTERVAL ",
 						  keep_history);
+
+		if (node_id != UNKNOWN_NODE_ID)
+		{
+			appendPQExpBuffer(&query,
+							  "  AND standby_node_id = %i", node_id);
+		}
 	}
 	else
 	{
-		appendPQExpBuffer(&query,
+		appendPQExpBufferStr(&query,
-						  "TRUNCATE TABLE repmgr.monitoring_history");
+							 "TRUNCATE TABLE repmgr.monitoring_history");
 	}
 
 	res = PQexec(primary_conn, query.data);
@@ -4494,10 +4634,10 @@ _is_bdr_db(PGconn *conn, PQExpBufferData *output, bool quiet)
 
 	initPQExpBuffer(&query);
 
-	appendPQExpBuffer(&query,
+	appendPQExpBufferStr(&query,
-					  " SELECT (pg_catalog.regexp_matches(extversion, '^\\d+'))[1] AS major_version "
+						 " SELECT (pg_catalog.regexp_matches(extversion, '^\\d+'))[1] AS major_version "
-					  "  FROM pg_catalog.pg_extension "
+						 "  FROM pg_catalog.pg_extension "
-					  " WHERE extname = 'bdr' ");
+						 " WHERE extname = 'bdr' ");
 
 	res = PQexec(conn, query.data);
 	termPQExpBuffer(&query);
@@ -4522,7 +4662,7 @@ _is_bdr_db(PGconn *conn, PQExpBufferData *output, bool quiet)
 		const char *warning = _("BDR extension is not available for this database");
 
 		if (output != NULL)
-			appendPQExpBuffer(output, "%s", warning);
+			appendPQExpBufferStr(output, warning);
 		else if (quiet == false)
 			log_warning("%s", warning);
 
@@ -4533,8 +4673,8 @@ _is_bdr_db(PGconn *conn, PQExpBufferData *output, bool quiet)
 	{
 		initPQExpBuffer(&query);
 
-		appendPQExpBuffer(&query,
+		appendPQExpBufferStr(&query,
-						  "SELECT bdr.bdr_is_active_in_db()");
+							 "SELECT bdr.bdr_is_active_in_db()");
 		res = PQexec(conn, query.data);
 		termPQExpBuffer(&query);
 
@@ -4545,7 +4685,7 @@ _is_bdr_db(PGconn *conn, PQExpBufferData *output, bool quiet)
 			const char *warning = _("BDR extension available for this database, but the database is not configured for BDR");
 
 			if (output != NULL)
-				appendPQExpBuffer(output, "%s", warning);
+				appendPQExpBufferStr(output, warning);
 			else if (quiet == false)
 				log_warning("%s", warning);
 		}
@@ -4640,10 +4780,10 @@ is_bdr_repmgr(PGconn *conn)
 
 	initPQExpBuffer(&query);
 
-	appendPQExpBuffer(&query,
+	appendPQExpBufferStr(&query,
-					  "SELECT pg_catalog.count(*)"
+						 "SELECT pg_catalog.count(*)"
-					  "  FROM repmgr.nodes n"
+						 "  FROM repmgr.nodes n"
-					  " WHERE n.type != 'bdr' ");
+						 " WHERE n.type != 'bdr' ");
 
 	res = PQexec(conn, query.data);
 	termPQExpBuffer(&query);
@@ -4688,11 +4828,11 @@ get_default_bdr_replication_set(PGconn *conn)
 
 	initPQExpBuffer(&query);
 
-	appendPQExpBuffer(&query,
+	appendPQExpBufferStr(&query,
-					  "     SELECT rs.set_name "
+						 "     SELECT rs.set_name "
-					  "       FROM pglogical.replication_set rs "
+						 "       FROM pglogical.replication_set rs "
-					  " INNER JOIN bdr.node_group ng "
+						 " INNER JOIN bdr.node_group ng "
-					  "         ON ng.node_group_default_repset = rs.set_id ");
+						 "         ON ng.node_group_default_repset = rs.set_id ");
 
 	res = PQexec(conn, query.data);
 	termPQExpBuffer(&query);
@@ -4827,13 +4967,13 @@ bdr_node_name_matches(PGconn *conn, const char *node_name, PQExpBufferData *bdr_
 
 	if (bdr_version_num < 3)
 	{
-		appendPQExpBuffer(&query,
+		appendPQExpBufferStr(&query,
-						  "SELECT bdr.bdr_get_local_node_name() AS node_name");
+							 "SELECT bdr.bdr_get_local_node_name() AS node_name");
 	}
 	else
 	{
-		appendPQExpBuffer(&query,
+		appendPQExpBufferStr(&query,
-						  "SELECT node_name FROM bdr.local_node_info()");
+							 "SELECT node_name FROM bdr.local_node_info()");
 	}
 
 	res = PQexec(conn, query.data);
@@ -4956,13 +5096,13 @@ add_extension_tables_to_bdr_replication_set(PGconn *conn)
 
 	initPQExpBuffer(&query);
 
-	appendPQExpBuffer(&query,
+	appendPQExpBufferStr(&query,
-					  "    SELECT c.relname "
+						 "    SELECT c.relname "
-					  "      FROM pg_class c "
+						 "      FROM pg_class c "
-					  "INNER JOIN pg_namespace n "
+						 "INNER JOIN pg_namespace n "
-					  "        ON c.relnamespace = n.oid "
+						 "        ON c.relnamespace = n.oid "
-					  "     WHERE n.nspname = 'repmgr' "
+						 "     WHERE n.nspname = 'repmgr' "
-					  "       AND c.relkind = 'r' ");
+						 "       AND c.relkind = 'r' ");
 
 	res = PQexec(conn, query.data);
 	termPQExpBuffer(&query);
@@ -4998,17 +5138,17 @@ get_all_bdr_node_records(PGconn *conn, BdrNodeInfoList *node_list)
 
 	if (bdr_version_num < 3)
 	{
-		appendPQExpBuffer(&query,
+		appendPQExpBufferStr(&query,
-						  "  SELECT " BDR2_NODES_COLUMNS
+							 "  SELECT " BDR2_NODES_COLUMNS
-						  "    FROM bdr.bdr_nodes "
+							 "    FROM bdr.bdr_nodes "
-						  "ORDER BY node_seq_id ");
+							 "ORDER BY node_seq_id ");
 	}
 	else
 	{
-		appendPQExpBuffer(&query,
+		appendPQExpBufferStr(&query,
-						  "     SELECT " BDR3_NODES_COLUMNS
+							 "     SELECT " BDR3_NODES_COLUMNS
-						  "       FROM bdr.node_summary ns "
+							 "       FROM bdr.node_summary ns "
-						  "   ORDER BY node_name");
+							 "   ORDER BY node_name");
 	}
 
 	log_verbose(LOG_DEBUG, "get_all_bdr_node_records():\n%s", query.data);

dbutils.h

@@ -327,6 +327,21 @@ typedef struct
     UNKNOWN_TIMELINE_ID, \
 	InvalidXLogRecPtr \
 }
+
+
+typedef struct RepmgrdInfo {
+	int node_id;
+	int pid;
+	char pid_text[MAXLEN];
+	char pid_file[MAXLEN];
+	bool pg_running;
+	char pg_running_text[MAXLEN];
+	bool running;
+	char repmgrd_running[MAXLEN];
+	bool paused;
+} RepmgrdInfo;
+
+
 /* global variables */
 
 extern int	server_version_num;
@@ -399,6 +414,11 @@ bool		identify_system(PGconn *repl_conn, t_system_identification *identification
 bool		repmgrd_set_local_node_id(PGconn *conn, int local_node_id);
 int			repmgrd_get_local_node_id(PGconn *conn);
 BackupState	server_in_exclusive_backup_mode(PGconn *conn);
+void		repmgrd_set_pid(PGconn *conn, pid_t repmgrd_pid, const char *pidfile);
+pid_t		repmgrd_get_pid(PGconn *conn);
+bool		repmgrd_is_running(PGconn *conn);
+bool		repmgrd_is_paused(PGconn *conn);
+bool		repmgrd_pause(PGconn *conn, bool pause);
 
 /* extension functions */
 ExtensionStatus get_repmgr_extension_status(PGconn *conn);
@@ -421,7 +441,7 @@ t_node_info *get_node_record_pointer(PGconn *conn, int node_id);
 bool		get_local_node_record(PGconn *conn, int node_id, t_node_info *node_info);
 bool		get_primary_node_record(PGconn *conn, t_node_info *node_info);
 
-void		get_all_node_records(PGconn *conn, NodeInfoList *node_list);
+bool		get_all_node_records(PGconn *conn, NodeInfoList *node_list);
 void		get_downstream_node_records(PGconn *conn, int node_id, NodeInfoList *nodes);
 void		get_active_sibling_node_records(PGconn *conn, int node_id, int upstream_node_id, NodeInfoList *node_list);
 void		get_node_records_by_priority(PGconn *conn, NodeInfoList *node_list);
@@ -491,8 +511,8 @@ add_monitoring_record(PGconn *primary_conn,
 					  long long unsigned int apply_lag_bytes
 );
 
-int			get_number_of_monitoring_records_to_delete(PGconn *primary_conn, int keep_history);
+int			get_number_of_monitoring_records_to_delete(PGconn *primary_conn, int keep_history, int node_id);
-bool		delete_monitoring_records(PGconn *primary_conn, int keep_history);
+bool		delete_monitoring_records(PGconn *primary_conn, int keep_history, int node_id);
 
 
 

doc/appendix-packages.sgml

@@ -12,10 +12,17 @@
 
   <sect1 id="packages-centos" xreflabel="CentOS packages">
     <title>CentOS Packages</title>
+
     <indexterm>
       <primary>packages</primary>
       <secondary>CentOS packages</secondary>
     </indexterm>
+
+    <indexterm>
+      <primary>CentOS</primary>
+      <secondary>package information</secondary>
+    </indexterm>
+
     <para>
       Currently, &repmgr; RPM packages are provided for versions 6.x and 7.x of CentOS. These should also
       work on matching versions of Red Hat Enterprise Linux, Scientific Linux and Oracle Enterprise Linux;
@@ -237,6 +244,12 @@
       <primary>packages</primary>
       <secondary>Debian/Ubuntu packages</secondary>
     </indexterm>
+
+    <indexterm>
+      <primary>Debian/Ubuntu</primary>
+      <secondary>package information</secondary>
+    </indexterm>
+
     <para>
       &repmgr; <literal>.deb</literal> packages are provided via the
       PostgreSQL Community APT repository, and are available for each community-supported

doc/appendix-release-notes.sgml

@@ -15,6 +15,136 @@
     See also: <xref linkend="upgrading-repmgr">
   </para>
 
+  <sect1 id="release-4.2">
+    <title>Release 4.2</title>
+    <para><emphasis>???, 2018</emphasis></para>
+    <para>
+    </para>
+
+    <sect2>
+      <title>Configuration file changes</title>
+      <para>
+        <itemizedlist>
+          <listitem>
+            <para>
+              New parameter <varname>shutdown_check_timeout</varname> (default: 60 seconds) added;
+              this provides an explicit timeout for
+              <command><link linkend="repmgr-standby-switchover">repmgr standby switchover</link></command>
+              to check that the demotion candidate (current primary) has shut down. Previously, the parameters
+              <literal>reconnect_attempts</literal> and <literal>reconnect_interval</literal>
+              were used to calculate a timeout, but these are actually
+              intended for primary failure detection. (GitHub #504).
+            </para>
+          </listitem>
+        </itemizedlist>
+
+        <itemizedlist>
+          <listitem>
+            <para>
+              New parameter <varname>repmgr_bindir</varname> added, to facilitate remote invocation of repmgr
+              when the repmgr binary is located somewhere other than the PostgreSQL binary directory, as it
+              cannot be assumed all package maintainers will install &repmgr; there.
+            </para>
+            <para>
+              This parameter is optional; if not set (the default), &repmgr; will fall back
+              to <option>pg_bindir</option> (if set).
+            </para>
+            <para>
+              (GitHub #246).
+            </para>
+          </listitem>
+        </itemizedlist>
+      </para>
+
+    </sect2>
+
+    <sect2>
+      <title>repmgr enhancements</title>
+      <para>
+       <itemizedlist>
+
+          <listitem>
+            <para>
+              <command><link linkend="repmgr-cluster-cleanup">repmgr cluster cleanup</link></command>
+              now accepts the <option>--node-id</option> option to delete records for only one
+              node. (GitHub #493).
+            </para>
+          </listitem>
+
+          <listitem>
+            <para>
+              When running
+              <command><link linkend="repmgr-cluster-matrix">repmgr cluster matrix</link></command> and
+              <command><link linkend="repmgr-cluster-crosscheck">repmgr cluster crosscheck</link></command>,
+              &repmgr; will report nodes unreachable via SSH, and emit return code <literal>ERR_BAD_SSH</literal>.
+              (GitHub #246).
+            </para>
+            <note>
+              <para>
+                Users relying on
+                <command><link linkend="repmgr-cluster-crosscheck">repmgr cluster crosscheck</link></command>
+                to return a non-zero return code as a way of detecting connectivity errors should be aware
+                that <literal>ERR_BAD_SSH</literal> will be returned if there is an SSH connection error
+                from the node where the command is executed, even if the command is able to establish
+                that PostgreSQL connectivity is fine. Therefore the exact return code should be checked
+                to determine what kind of connectivity error has been detected.
+              </para>
+            </note>
+          </listitem>
+
+        </itemizedlist>
+      </para>
+    </sect2>
+
+
+    <sect2>
+      <title>repmgrd enhancements</title>
+      <para>
+       <itemizedlist>
+
+          <listitem>
+            <para>
+              <application>repmgrd</application> can now be &quot;paused&quot;, i.e. instructed
+              not to take any action such as a failover, even if the prerequisites for such an
+              action are detected.
+            </para>
+            <para>
+              This removes the need to stop <application>repmgrd</application> on all nodes when
+              performing a planned operation such as a switchover.
+            </para>
+            <para>
+              For further details, see <link linkend="repmgrd-pausing">Pausing repmgrd</link>.
+            </para>
+          </listitem>
+
+       </itemizedlist>
+      </para>
+    </sect2>
+
+    <sect2>
+      <title>Bug fixes</title>
+      <para>
+        <itemizedlist>
+
+          <listitem>
+            <para>
+              &repmgr;: fix &quot;Missing replication slots&quot; label in
+              <command><link linkend="repmgr-node-check">repmgr node check</link></command>.  (GitHub #507)
+            </para>
+          </listitem>
+
+
+          <listitem>
+            <para>
+              <application>repmgrd</application>: fix parsing of <option>-d/--daemonize</option> option.
+            </para>
+          </listitem>
+
+        </itemizedlist>
+      </para>
+    </sect2>
+  </sect1>
+
   <sect1 id="release-4.1.1">
     <title>Release 4.1.1</title>
     <para><emphasis>Wed September 5, 2018</emphasis></para>

doc/cloning-standbys.sgml

@@ -352,10 +352,12 @@
       provide additional parameters for <command>pg_basebackup</command> to customise the
       cloning process.
     </para>
+
     <para>
      By default, <command>pg_basebackup</command> performs a checkpoint before beginning the backup
      process. However, a normal checkpoint may take some time to complete;
-     a fast checkpoint can be forced with the <literal>-c/--fast-checkpoint</literal> option.
+     a fast checkpoint can be forced with <command><link linkend="repmgr-standby-clone">repmgr standby clone</link></command>'s
+     <literal>-c/--fast-checkpoint</literal> option.
      Note that this may impact performance of the server being cloned from (typically the primary)
      so should be used with care.
     </para>
@@ -370,6 +372,18 @@
       Other options can be passed to <command>pg_basebackup</command> by including them
       in the <filename>repmgr.conf</filename> setting <varname>pg_basebackup_options</varname>.
     </para>
+
+    <para>
+      Not that by default, &repmgr; executes <command>pg_basebackup</command> with <option>-X/--wal-method</option>
+      (PostgreSQL 9.6 and earlier: <option>-X/--xlog-method</option>) set to <literal>stream</literal>.
+      From PostgreSQL 9.6, if replication slots are in use, it will also create a replication slot before
+      running the base backup, and execute <command>pg_basebackup</command> with the
+      <option>-S/--slot</option> option set to the name of the previously created replication slot.
+    </para>
+    <para>
+      These parameters can set by the user in <varname>pg_basebackup_options</varname>, in which case they
+      will override the &repmgr; default values. However normally there's no reason to do this.
+    </para>
     <para>
       If using a separate directory to store WAL files, provide the option <literal>--waldir</literal>
       (<literal>--xlogdir</literal> in PostgreSQL 9.6 and earlier) with the absolute path to the

doc/filelist.sgml

@@ -58,6 +58,7 @@
 <!ENTITY repmgrd-cascading-replication SYSTEM "repmgrd-cascading-replication.sgml">
 <!ENTITY repmgrd-network-split SYSTEM "repmgrd-network-split.sgml">
 <!ENTITY repmgrd-witness-server SYSTEM "repmgrd-witness-server.sgml">
+<!ENTITY repmgrd-pausing SYSTEM "repmgrd-pausing.sgml">
 <!ENTITY repmgrd-bdr SYSTEM "repmgrd-bdr.sgml">
 
 <!ENTITY repmgr-primary-register SYSTEM "repmgr-primary-register.sgml">
@@ -78,6 +79,9 @@
 <!ENTITY repmgr-cluster-crosscheck SYSTEM "repmgr-cluster-crosscheck.sgml">
 <!ENTITY repmgr-cluster-event SYSTEM "repmgr-cluster-event.sgml">
 <!ENTITY repmgr-cluster-cleanup SYSTEM "repmgr-cluster-cleanup.sgml">
+<!ENTITY repmgr-daemon-status SYSTEM "repmgr-daemon-status.sgml">
+<!ENTITY repmgr-daemon-pause SYSTEM "repmgr-daemon-pause.sgml">
+<!ENTITY repmgr-daemon-unpause SYSTEM "repmgr-daemon-unpause.sgml">
 
 <!ENTITY appendix-release-notes  SYSTEM "appendix-release-notes.sgml">
 <!ENTITY appendix-faq      SYSTEM "appendix-faq.sgml">

doc/install-requirements.sgml

@@ -36,7 +36,7 @@
   </para>
 
   <para>
-   A dedicated system user for &repmgr; is *not* required; as many &repmgr; and
+   A dedicated system user for &repmgr; is <emphasis>not</emphasis> required; as many &repmgr; and
    <application>repmgrd</application> actions require direct access to the PostgreSQL data directory,
    these commands should be executed by the <literal>postgres</literal> user.
   </para>
@@ -61,6 +61,13 @@
    </itemizedlist>
   </para>
 
+  <tip>
+    <simpara>
+      Consider setting <varname>ConnectTimeout</varname> to a low value in your SSH configuration.
+      This will make it faster to detect any SSH connection errors.
+    </simpara>
+  </tip>
+
   <tip>
    <simpara>
     We recommend using a session multiplexer utility such as <command>screen</command> or

doc/quickstart.sgml

@@ -237,13 +237,6 @@
    server. See sections <xref linkend="configuration"> and <xref linkend="configuration-file">
    for further details about <filename>repmgr.conf</filename>.
   </para>
-  <tip>
-   <simpara>
-    For Debian-based distributions we recommend explictly setting
-    <option>pg_bindir</option> to the directory where <command>pg_ctl</command> and other binaries
-    not in the standard path are located. For PostgreSQL 9.6 this would be <filename>/usr/lib/postgresql/9.6/bin/</filename>.
-   </simpara>
-  </tip>
 
   <note>
     <para>
@@ -262,6 +255,24 @@
     </para>
   </note>
 
+  <tip>
+   <simpara>
+    For Debian-based distributions we recommend explictly setting
+    <option>pg_bindir</option> to the directory where <command>pg_ctl</command> and other binaries
+    not in the standard path are located. For PostgreSQL 9.6 this would be <filename>/usr/lib/postgresql/9.6/bin/</filename>.
+   </simpara>
+  </tip>
+
+  <tip>
+    <simpara>
+      If your distribution places the &repmgr; binaries in a location other than the
+      PostgreSQL installation directory, specify this with <option>repmgr_bindir</option>
+      to enable &repmgr; to perform operations (e.g.
+      <command><link linkend="repmgr-cluster-crosscheck">repmgr cluster crosscheck</link></command>)
+      on other nodes.
+    </simpara>
+  </tip>
+
   <para>
    See the file
    <ulink url="https://raw.githubusercontent.com/2ndQuadrant/repmgr/master/repmgr.conf.sample">repmgr.conf.sample</>

doc/repmgr-cluster-cleanup.sgml

@@ -51,6 +51,20 @@
     </para>
   </refsect1>
 
+  <refsect1>
+    <title>Options</title>
+    <variablelist>
+
+      <varlistentry>
+        <term><option>--node-id</option></term>
+        <listitem>
+          <para>
+            Only delete monitoring records for the specified node.
+          </para>
+        </listitem>
+      </varlistentry>
+    </variablelist>
+  </refsect1>
 
   <refsect1>
     <title>See also</title>

doc/repmgr-cluster-crosscheck.sgml

@@ -55,12 +55,37 @@
         </listitem>
       </varlistentry>
 
+      <varlistentry>
+        <term><option>ERR_BAD_SSH (12)</option></term>
+        <listitem>
+          <para>
+            One or more nodes could not be accessed via SSH.
+          </para>
+          <note>
+            <simpara>
+              This only applies to nodes unreachable from the node where
+              this command is executed.
+            </simpara>
+            <simpara>
+              It's also possible that the crosscheck establishes that
+              connections between PostgreSQL on all nodes are functioning,
+              even if SSH access between some nodes is not possible.
+            </simpara>
+          </note>
+        </listitem>
+      </varlistentry>
+
       <varlistentry>
         <term><option>ERR_NODE_STATUS (25)</option></term>
         <listitem>
           <para>
-            One or more nodes could not be reached.
+            PostgreSQL on one or more nodes could not be reached.
           </para>
+          <note>
+            <simpara>
+              This error code overrides <option>ERR_BAD_SSH</option>.
+            </simpara>
+          </note>
         </listitem>
       </varlistentry>
 

doc/repmgr-cluster-matrix.sgml

@@ -115,12 +115,26 @@
         </listitem>
       </varlistentry>
 
+      <varlistentry>
+        <term><option>ERR_BAD_SSH (12)</option></term>
+        <listitem>
+          <para>
+            One or more nodes could not be accessed via SSH.
+          </para>
+        </listitem>
+      </varlistentry>
+
       <varlistentry>
         <term><option>ERR_NODE_STATUS (25)</option></term>
         <listitem>
           <para>
-            One or more nodes could not be reached.
+            PostgreSQL on one or more nodes could not be reached.
           </para>
+          <note>
+            <simpara>
+              This error code overrides <option>ERR_BAD_SSH</option>.
+            </simpara>
+          </note>
         </listitem>
       </varlistentry>
 

doc/repmgr-cluster-show.sgml

@@ -90,7 +90,7 @@
 		  <para>
 			<command>repmgr cluster show</command> accepts an optional parameter <literal>--csv</literal>, which
 			outputs the replication cluster's status in a simple CSV format, suitable for
-			parsing by scripts:
+			parsing by scripts, e.g.:
 			<programlisting>
     $ repmgr -f /etc/repmgr.conf cluster show --csv
     1,-1,-1
@@ -165,7 +165,7 @@
   <refsect1>
     <title>See also</title>
     <para>
-     <xref linkend="repmgr-node-status">, <xref linkend="repmgr-node-check">
+     <xref linkend="repmgr-node-status">, <xref linkend="repmgr-node-check">, <xref linkend="repmgr-daemon-status">
     </para>
   </refsect1>
 

doc/repmgr-daemon-pause.sgml

@@ -0,0 +1,109 @@
+<refentry id="repmgr-daemon-pause">
+  <indexterm>
+    <primary>repmgr daemon pause</primary>
+  </indexterm>
+
+  <refmeta>
+    <refentrytitle>repmgr daemon pause</refentrytitle>
+  </refmeta>
+
+  <refnamediv>
+    <refname>repmgr daemon pause</refname>
+    <refpurpose>Instruct all <application>repmgrd</application> instances in the replication cluster to pause failover operations</refpurpose>
+  </refnamediv>
+
+  <refsect1>
+    <title>Description</title>
+    <para>
+      This command can be run on any active node in the replication cluster to instruct all
+      running <application>repmgrd</application> instances to &quot;pause&quot; themselves, i.e. take no
+      action (such as promoting themselves or following a new primary) if a failover event is detected.
+    </para>
+    <para>
+      This functionality is useful for performing maintenance operations, such as switchovers
+      or upgrades, which might otherwise trigger a failover if <application>repmgrd</application>
+      is running normally.
+    </para>
+    <note>
+      <para>
+        It's important to wait a few seconds after restarting PostgreSQL on any node before running
+        <command>repmgr daemon pause</command>, as the <application>repmgrd</application> instance
+        on the restarted node will take a second or two before it has updated its status.
+      </para>
+    </note>
+    <para>
+      <xref linkend="repmgr-daemon-unpause"> will instruct all previously paused <application>repmgrd</application>
+      instances to resume normal failover operation.
+    </para>
+  </refsect1>
+
+  <refsect1>
+    <title>Execution</title>
+    <para>
+      <command>repmgr daemon pause</command> can be executed on any active node in the
+      replication cluster. A valid <filename>repmgr.conf</filename> file is required.
+      It will have no effect on previously paused nodes.
+    </para>
+  </refsect1>
+
+  <refsect1>
+    <title>Example</title>
+    <para>
+    <programlisting>
+$ repmgr -f /etc/repmgr.conf daemon pause
+NOTICE: node 1 (node1) paused
+NOTICE: node 2 (node2) paused
+NOTICE: node 3 (node3) paused</programlisting>
+    </para>
+  </refsect1>
+
+  <refsect1>
+    <title>Options</title>
+    <variablelist>
+      <varlistentry>
+        <term><option>--dry-run</option></term>
+        <listitem>
+          <para>
+            Check if nodes are reachable but don't pause <application>repmgrd</application>.
+          </para>
+        </listitem>
+      </varlistentry>
+    </variablelist>
+  </refsect1>
+
+  <refsect1>
+    <title>Exit codes</title>
+    <para>
+      Following exit codes can be emitted by <command>repmgr daemon unpause</command>:
+    </para>
+    <variablelist>
+
+      <varlistentry>
+        <term><option>SUCCESS (0)</option></term>
+        <listitem>
+          <para>
+            <application>repmgrd</application> could be paused on all nodes.
+          </para>
+        </listitem>
+      </varlistentry>
+
+      <varlistentry>
+        <term><option>ERR_REPMGRD_PAUSE (26)</option></term>
+        <listitem>
+          <para>
+           <application>repmgrd</application> could not be paused on one or mode nodes.
+          </para>
+        </listitem>
+      </varlistentry>
+
+    </variablelist>
+  </refsect1>
+
+  <refsect1>
+    <title>See also</title>
+    <para>
+      <xref linkend="repmgr-daemon-unpause">, <xref linkend="repmgr-daemon-status">
+    </para>
+  </refsect1>
+</refentry>
+

doc/repmgr-daemon-status.sgml

@@ -0,0 +1,165 @@
+<refentry id="repmgr-daemon-status">
+  <indexterm>
+    <primary>repmgr daemon status</primary>
+  </indexterm>
+
+  <refmeta>
+    <refentrytitle>repmgr daemon status</refentrytitle>
+  </refmeta>
+
+  <refnamediv>
+    <refname>repmgr daemon status</refname>
+    <refpurpose>display information about the status of <application>repmgrd</application> on each node in the cluster</refpurpose>
+  </refnamediv>
+
+  <refsect1>
+    <title>Description</title>
+    <para>
+      This command provides an overview over all active nodes in the cluster and the state
+      of each node's <application>repmgrd</application> instance. It can be used to check
+      the result of <xref linkend="repmgr-daemon-pause"> and <xref linkend="repmgr-daemon-unpause">
+      operations.
+    </para>
+  </refsect1>
+
+  <refsect1>
+    <title>Execution</title>
+    <para>
+      <command>repmgr daemon status</command> can be executed on any active node in the
+      replication cluster. A valid <filename>repmgr.conf</filename> file is required.
+    </para>
+
+    <note>
+      <para>
+        After restarting PostgreSQL on any node, the <application>repmgrd</application> instance
+        will take a second or two before it is able to update its status. Until then,
+        <application>repmgrd</application> will be shown as not running.
+      </para>
+    </note>
+
+  </refsect1>
+
+  <refsect1>
+    <title>Examples</title>
+    <para>
+      <application>repmgrd</application> running normally on all nodes:
+    <programlisting>$ repmgr -f /etc/repmgr.conf daemon status
+ ID | Name  | Role    | Status  | repmgrd | PID  | Paused?
+----+-------+---------+---------+---------+------+---------
+ 1  | node1 | primary | running | running | 7851 | no
+ 2  | node2 | standby | running | running | 7889 | no
+ 3  | node3 | standby | running | running | 7918 | no</programlisting>
+    </para>
+
+    <para>
+      <application>repmgrd</application> paused on all nodes (using <xref linkend="repmgr-daemon-pause">):
+    <programlisting>$ repmgr -f /etc/repmgr.conf daemon status
+ ID | Name  | Role    | Status  | repmgrd | PID  | Paused?
+----+-------+---------+---------+---------+------+---------
+ 1  | node1 | primary | running | running | 7851 | yes
+ 2  | node2 | standby | running | running | 7889 | yes
+ 3  | node3 | standby | running | running | 7918 | yes</programlisting>
+    </para>
+
+    <para>
+      <application>repmgrd</application> not running on one node:
+    <programlisting>$ repmgr -f /etc/repmgr.conf daemon status
+ ID | Name  | Role    | Status  | repmgrd     | PID  | Paused?
+----+-------+---------+---------+-------------+------+---------
+ 1  | node1 | primary | running | running     | 7851 | yes
+ 2  | node2 | standby | running | not running | n/a  | n/a
+ 3  | node3 | standby | running | running     | 7918 | yes</programlisting>
+    </para>
+
+  </refsect1>
+
+  <refsect1>
+    <title>Options</title>
+
+    <variablelist>
+
+      <varlistentry>
+        <term><option>--csv</option></term>
+        <listitem>
+		  <para>
+			<command>repmgr daemon status</command> accepts an optional parameter <literal>--csv</literal>, which
+			outputs the replication cluster's status in a simple CSV format, suitable for
+			parsing by scripts, e.g.:
+			<programlisting>
+    $ repmgr -f /etc/repmgr.conf daemon status --csv
+    1,node1,primary,1,1,10204,1
+    2,node2,standby,1,0,-1,1
+    3,node3,standby,1,1,10225,1</programlisting>
+		  </para>
+		  <para>
+			The columns have following meanings:
+			<itemizedlist spacing="compact" mark="bullet">
+			  <listitem>
+				<simpara>
+				  node ID
+				</simpara>
+			  </listitem>
+
+			  <listitem>
+				<simpara>
+                  node name
+				</simpara>
+			  </listitem>
+
+			  <listitem>
+				<simpara>
+                  node type (primary or standby)
+				</simpara>
+			  </listitem>
+
+			  <listitem>
+				<simpara>
+                  PostgreSQL server running
+				</simpara>
+			  </listitem>
+
+			  <listitem>
+				<simpara>
+                  <application>repmgrd</application> running (1 = running, 0 = not running)
+				</simpara>
+			  </listitem>
+
+			  <listitem>
+				<simpara>
+                  <application>repmgrd</application> PID (-1 if not running)
+				</simpara>
+			  </listitem>
+
+			  <listitem>
+				<simpara>
+                  <application>repmgrd</application> paused (1 = paused, 0 = not paused)
+				</simpara>
+			  </listitem>
+
+			</itemizedlist>
+		  </para>
+		</listitem>
+	  </varlistentry>
+
+      <varlistentry>
+        <term><option>--verbose</option></term>
+        <listitem>
+          <para>
+			Display the full text of any database connection error messages
+          </para>
+        </listitem>
+      </varlistentry>
+
+	</variablelist>
+
+  </refsect1>
+
+
+
+  <refsect1>
+    <title>See also</title>
+    <para>
+      <xref linkend="repmgr-daemon-pause">, <xref linkend="repmgr-daemon-unpause">, <xref linkend="repmgr-cluster-show">
+    </para>
+  </refsect1>
+</refentry>

doc/repmgr-daemon-unpause.sgml

@@ -0,0 +1,103 @@
+<refentry id="repmgr-daemon-unpause">
+  <indexterm>
+    <primary>repmgr daemon unpause</primary>
+  </indexterm>
+
+  <refmeta>
+    <refentrytitle>repmgr daemon unpause</refentrytitle>
+  </refmeta>
+
+  <refnamediv>
+    <refname>repmgr daemon unpause</refname>
+    <refpurpose>Instruct all <application>repmgrd</application> instances in the replication cluster to resume failover operations</refpurpose>
+  </refnamediv>
+
+  <refsect1>
+    <title>Description</title>
+    <para>
+      This command can be run on any active node in the replication cluster to instruct all
+      running <application>repmgrd</application> instances to &quot;unpause&quot;
+      (following a previous execution of <xref linkend="repmgr-daemon-pause">)
+      and resume normal failover/monitoring operation.
+    </para>
+
+    <note>
+      <para>
+        It's important to wait a few seconds after restarting PostgreSQL on any node before running
+        <command>repmgr daemon pause</command>, as the <application>repmgrd</application> instance
+        on the restarted node will take a second or two before it has updated its status.
+      </para>
+    </note>
+
+  </refsect1>
+
+  <refsect1>
+    <title>Execution</title>
+    <para>
+     <command>repmgr daemon unpause</command> can be executed on any active node in the
+      replication cluster. A valid <filename>repmgr.conf</filename> file is required.
+      It will have no effect on nodes which are not already paused.
+    </para>
+  </refsect1>
+
+  <refsect1>
+    <title>Example</title>
+    <para>
+    <programlisting>
+$ repmgr -f /etc/repmgr.conf daemon unpause
+NOTICE: node 1 (node1) unpaused
+NOTICE: node 2 (node2) unpaused
+NOTICE: node 3 (node3) unpaused</programlisting>
+    </para>
+  </refsect1>
+
+  <refsect1>
+    <title>Options</title>
+    <variablelist>
+      <varlistentry>
+        <term><option>--dry-run</option></term>
+        <listitem>
+          <para>
+            Check if nodes are reachable but don't unpause <application>repmgrd</application>.
+          </para>
+        </listitem>
+      </varlistentry>
+    </variablelist>
+  </refsect1>
+
+  <refsect1>
+    <title>Exit codes</title>
+    <para>
+      Following exit codes can be emitted by <command>repmgr daemon unpause</command>:
+    </para>
+    <variablelist>
+
+      <varlistentry>
+        <term><option>SUCCESS (0)</option></term>
+        <listitem>
+          <para>
+            <application>repmgrd</application> could be unpaused on all nodes.
+          </para>
+        </listitem>
+      </varlistentry>
+
+      <varlistentry>
+        <term><option>ERR_REPMGRD_PAUSE (26)</option></term>
+        <listitem>
+          <para>
+           <application>repmgrd</application> could not be unpaused on one or mode nodes.
+          </para>
+        </listitem>
+      </varlistentry>
+
+    </variablelist>
+  </refsect1>
+
+  <refsect1>
+    <title>See also</title>
+    <para>
+      <xref linkend="repmgr-daemon-pause">, <xref linkend="repmgr-daemon-status">
+    </para>
+  </refsect1>
+</refentry>
+

doc/repmgr-node-check.sgml

@@ -30,7 +30,8 @@
             Replication lag: OK (N/A - node is primary)
             WAL archiving: OK (0 pending files)
             Downstream servers: OK (2 of 2 downstream nodes attached)
-            Replication slots: OK (node has no replication slots)</programlisting>
+            Replication slots: OK (node has no replication slots)
+            Missing replication slots: OK (node has no missing replication slots)</programlisting>
     </para>
   </refsect1>
   <refsect1>

doc/repmgr-standby-switchover.sgml

@@ -35,6 +35,10 @@
         &repmgr; will attempt to check for potential issues but cannot guarantee
         a successful switchover.
       </para>
+      <para>
+        &repmgr; will refuse to perform the switchover if an exclusive backup is running on
+        the current primary.
+      </para>
     </note>
     <para>
       For more details on performing a switchover, including preparation and configuration,
@@ -43,11 +47,14 @@
 
     <note>
       <para>
-        <application>repmgrd</application> should not be active on any nodes while a switchover is being
+        From <link linkend="release-4.2">repmgr 4.2</link>, &repmgr; will instruct any running
-        executed. This restriction may be lifted in a later version.
+        <application>repmgrd</application> instances to pause operations while the switchover
+        is being carried out, to prevent <application>repmgrd</application> from
+        unintentionally promoting a node. For more details, see <xref linkend="repmgrd-pausing">.
       </para>
       <para>
-        &repmgr; will not perform the switchover if an exclusive backup is running on the current primary.
+        Users of &repmgr; versions prior to 4.2 should ensure that <application>repmgrd</application>
+        is not running on any nodes while a switchover is being executed.
       </para>
     </note>
 
@@ -61,8 +68,9 @@
         <term><option>--always-promote</option></term>
         <listitem>
           <para>
-            Promote standby to primary, even if it is behind original primary
+            Promote standby to primary, even if it is behind or has diverged
-            (original primary will be shut down in any case).
+            from the original primary. The original primary will be shut down in any case,
+            and will need to be manually reintegrated into the replication cluster.
           </para>
         </listitem>
       </varlistentry>
@@ -122,6 +130,21 @@
         </listitem>
       </varlistentry>
 
+      <varlistentry>
+        <term><option>--repmgrd-no-pause</option></term>
+        <listitem>
+          <para>
+            Don't pause <application>repmgrd</application> while executing a switchover.
+          </para>
+          <para>
+            This option should not be used unless you take steps by other means
+            to ensure <application>repmgrd</application> is paused or not
+            running on all nodes.
+          </para>
+        </listitem>
+      </varlistentry>
+
+
      <varlistentry>
         <term><option>--siblings-follow</option></term>
         <listitem>
@@ -141,19 +164,7 @@
      Note that following parameters in <filename>repmgr.conf</filename> are relevant to the
      switchover operation:
      <itemizedlist spacing="compact" mark="bullet">
-       <listitem>
+
-         <simpara>
-           <literal>reconnect_attempts</literal>: number of times to check the original primary
-           for a clean shutdown after executing the shutdown command, before aborting
-         </simpara>
-       </listitem>
-       <listitem>
-         <simpara>
-           <literal>reconnect_interval</literal>: interval (in seconds) to check the original
-           primary for a clean shutdown after executing the shutdown command (up to a maximum
-           of <literal>reconnect_attempts</literal> tries)
-         </simpara>
-       </listitem>
        <listitem>
          <simpara>
            <literal>replication_lag_critical</literal>:
@@ -163,10 +174,29 @@
          </simpara>
        </listitem>
 
+       <listitem>
+         <simpara>
+           <literal>shutdown_check_timeout</literal>: maximum number of seconds to wait for the
+           demotion candidate (current primary) to shut down, before aborting the switchover.
+         </simpara>
+         <simpara>
+           Note that this parameter is set on the node where <command>repmgr standby switchover</command>
+           is executed (promotion candidate); setting it on the demotion candidate (former primary) will
+           have no effect.
+         </simpara>
+         <note>
+           <para>
+             In versions prior to <link linkend="release-4.2">&repmgr; 4.2</link>, <command>repmgr standby switchover</command> would
+             use the values defined in <literal>reconnect_attempts</literal> and <literal>reconnect_interval</literal>
+             to determine the timeout for demotion candidate shutdown.
+           </para>
+         </note>
+       </listitem>
+
        <listitem>
          <simpara>
            <literal>standby_reconnect_timeout</literal>:
-           number of seconds to attempt to wait for the demoted primary
+           maximum number of seconds to attempt to wait for the demotion candidate (former primary)
            to reconnect to the promoted primary (default: 60 seconds)
          </simpara>
        </listitem>

doc/repmgr.sgml

@@ -25,13 +25,7 @@
    <para>
    This is the official documentation of &repmgr; &repmgrversion; for
    use with PostgreSQL 9.3 - PostgreSQL 10.
-   </para>
+   It describes the functionality supported by the current version of &repmgr;.
-   <para>
-     &repmgr; is being continually developed and we strongly recommend using the
-     latest version. Please check the
-     <ulink url="https://repmgr.org/">repmgr website</ulink> for details
-     about the current &repmgr; version as well as the
-     <ulink url="https://repmgr.org/docs/current/index.html">current documentation</ulink>.
    </para>
 
    <para>
@@ -92,6 +86,7 @@
   &repmgrd-cascading-replication;
   &repmgrd-network-split;
   &repmgrd-witness-server;
+  &repmgrd-pausing;
   &repmgrd-degraded-monitoring;
   &repmgrd-monitoring;
   &repmgrd-bdr;
@@ -118,6 +113,9 @@
   &repmgr-cluster-crosscheck;
   &repmgr-cluster-event;
   &repmgr-cluster-cleanup;
+  &repmgr-daemon-status;
+  &repmgr-daemon-pause;
+  &repmgr-daemon-unpause;
  </part>
 
  &appendix-release-notes;

doc/repmgrd-configuration.sgml

@@ -454,7 +454,7 @@ REPMGRD_ENABLED=no
 #REPMGRD_CONF="/path/to/repmgr.conf"
 
 # additional options
-#REPMGRD_OPTS=""
+REPMGRD_OPTS="--daemonize=false"
 
 # user to run repmgrd as
 #REPMGRD_USER=postgres
@@ -469,6 +469,19 @@ REPMGRD_ENABLED=no
         Set <varname>REPMGRD_ENABLED</varname> to <literal>yes</literal>, and <varname>REPMGRD_CONF</varname>
         to the <filename>repmgr.conf</filename> file you are using.
       </para>
+      <tip>
+        <para>
+          See <xref linkend="packages-debian-ubuntu"> for details of the Debian/Ubuntu packages and
+          typical file locations (including <filename>repmgr.conf</filename>).
+        </para>
+      </tip>
+      <para>
+        From <application>repmgrd</application> 4.1, ensure <varname>REPMGRD_OPTS</varname> includes
+        <option>--daemonize=false</option>, as daemonization is handled by the service command.
+        We recommend setting <varname>repmgrd_pid_file</varname> in <filename>repmgr.conf</filename> to the
+        same value set in <varname>REPMGRD_PIDFILE</varname> to prevent another <application>repmgrd</application>
+        instance from being started manually.
+      </para>
       <para>
         If using <application>systemd</application>, you may need to execute <command>systemctl daemon-reload</command>.
         Also, if you attempted to start <application>repmgrd</application> using <command>systemctl start repmgrd</command>,

doc/repmgrd-pausing.sgml

@@ -0,0 +1,169 @@
+<chapter id="repmgrd-pausing" xreflabel="Pausing repmgrd">
+
+  <indexterm>
+    <primary>repmgrd</primary>
+    <secondary>pausing</secondary>
+  </indexterm>
+
+  <indexterm>
+    <primary>pausing repmgrd</primary>
+  </indexterm>
+
+  <title>Pausing repmgrd</title>
+
+  <para>
+    In normal operation, <application>repmgrd</application> monitors the state of the
+    PostgreSQL node it is running on, and will take appropriate action if problems
+    are detected, e.g. (if so configured) promote the node to primary, if the existing
+    primary has been determined as failed.
+  </para>
+
+  <para>
+    However, <application>repmgrd</application> is unable to distinguish between
+    planned outages (such as performing a <link linkend="performing-switchover">switchover</link>
+    or upgrading a server), and an actual server outage. In versions prior to &repmgr; 4.2
+    it was necessary to stop <application>repmgrd</application> on all nodes (or at least
+    on all nodes where <application>repmgrd</application> is
+    <link linkend="repmgrd-automatic-failover">configured for automatic failover</link>)
+    to prevent <application>repmgrd</application> from making changes to the
+    replication cluster.
+  </para>
+  <para>
+    From <link linkend="release-4.2">&repmgr; 4.2</link>, <application>repmgrd</application>
+    can now be &quot;paused&quot;, i.e. instructed not to take any action such as performing a failover.
+    This can be done from any node in the cluster, removing the need to stop/restart
+    each <application>repmgrd</application> individually.
+  </para>
+
+  <sect1 id="repmgrd-pausing-prerequisites">
+    <title>Prerequisites for pausing <application>repmgrd</application></title>
+    <para>
+      In order to be able to pause/unpause <application>repmgrd</application>, following
+      prerequisites must be met:
+      <itemizedlist spacing="compact" mark="bullet">
+
+        <listitem>
+          <simpara><link linkend="release-4.2">&repmgr; 4.2</link> or later must be installed on all nodes.</simpara>
+        </listitem>
+
+        <listitem>
+          <simpara>The same major &repmgr; version (e.g. 4.2) must be installed on all nodes (and preferably the same minor version).</simpara>
+        </listitem>
+
+        <listitem>
+          <simpara>
+            PostgreSQL on all nodes must be accessible from the node where the
+            <literal>pause</literal>/<literal>unpause</literal> operation is executed, using the
+            <varname>conninfo</varname> string shown by <link linkend="repmgr-cluster-show"><command>repmgr cluster show</command></link>.
+          </simpara>
+        </listitem>
+      </itemizedlist>
+    </para>
+    <note>
+      <para>
+        These conditions are required for normal &repmgr; operation in any case.
+      </para>
+    </note>
+
+  </sect1>
+
+  <sect1 id="repmgrd-pausing-execution">
+    <title>Pausing/unpausing <application>repmgrd</application></title>
+    <para>
+      To pause <application>repmgrd</application>, execute <link linkend="repmgr-daemon-pause"><command>repmgr daemon pause</command></link>, e.g.:
+   <programlisting>
+$ repmgr -f /etc/repmgr.conf daemon pause
+NOTICE: node 1 (node1) paused
+NOTICE: node 2 (node2) paused
+NOTICE: node 3 (node3) paused</programlisting>
+    </para>
+    <para>
+      The state of <application>repmgrd</application> on each node can be checked with
+      <link linkend="repmgr-daemon-status"><command>repmgr daemon status</command></link>, e.g.:
+    <programlisting>$ repmgr -f /etc/repmgr.conf daemon status
+ ID | Name  | Role    | Status  | repmgrd | PID  | Paused?
+----+-------+---------+---------+---------+------+---------
+ 1  | node1 | primary | running | running | 7851 | yes
+ 2  | node2 | standby | running | running | 7889 | yes
+ 3  | node3 | standby | running | running | 7918 | yes</programlisting>
+    </para>
+
+    <note>
+      <para>
+        If executing a switchover with  <link linkend="repmgr-standby-switchover"><command>repmgr standby switchover</command></link>,
+		&repmgr; will automatically pause/unpause <application>repmgrd</application> as part of the switchover process.
+      </para>
+    </note>
+
+    <para>
+      If the primary (in this example, <literal>node1</literal>) is stopped, <application>repmgrd</application>
+      running on one of the standbys (here: <literal>node2</literal>) will react like this:
+      <programlisting>
+[2018-09-20 12:22:21] [WARNING] unable to connect to upstream node "node1" (node ID: 1)
+[2018-09-20 12:22:21] [INFO] checking state of node 1, 1 of 5 attempts
+[2018-09-20 12:22:21] [INFO] sleeping 1 seconds until next reconnection attempt
+...
+[2018-09-20 12:22:24] [INFO] sleeping 1 seconds until next reconnection attempt
+[2018-09-20 12:22:25] [INFO] checking state of node 1, 5 of 5 attempts
+[2018-09-20 12:22:25] [WARNING] unable to reconnect to node 1 after 5 attempts
+[2018-09-20 12:22:25] [NOTICE] node is paused
+[2018-09-20 12:22:33] [INFO] node "node2" (node ID: 2) monitoring upstream node "node1" (node ID: 1) in degraded state
+[2018-09-20 12:22:33] [DETAIL] repmgrd paused by administrator
+[2018-09-20 12:22:33] [HINT] execute "repmgr daemon unpause" to resume normal failover mode</programlisting>
+    </para>
+    <para>
+      If the primary becomes available again (e.g. following a software upgrade), <application>repmgrd</application>
+      will automatically reconnect, e.g.:
+      <programlisting>
+[2018-09-20 13:12:41] [NOTICE] reconnected to upstream node 1 after 8 seconds, resuming monitoring</programlisting>
+    </para>
+
+    <para>
+      To unpause <application>repmgrd</application>, execute <link linkend="repmgr-daemon-unpause"><command>repmgr daemon unpause</command></link>, e.g.:
+   <programlisting>
+$ repmgr -f /etc/repmgr.conf daemon pause
+NOTICE: node 1 (node1) unpaused
+NOTICE: node 2 (node2) unpaused
+NOTICE: node 3 (node3) unpaused</programlisting>
+    </para>
+
+    <note>
+      <para>
+        If the previous primary is no longer accessible when <application>repmgrd</application>
+        is unpaused, no failover action will be taken. Instead, a new primary must be manually promoted using
+        <link linkend="repmgr-standby-promote"><command>repmgr standby promote</command></link>,
+		and any standbys attached to the new primary with
+		<link linkend="repmgr-standby-follow"><command>repmgr standby follow</command></link>.
+      </para>
+      <para>
+        This is to prevent <link linkend="repmgr-daemon-unpause"><command>repmgr daemon unpause</command></link>
+        resulting in the automatic promotion of a new primary, which may be a problem particularly
+        in larger clusters, where <application>repmgrd</application> could select a different promotion
+        candidate to the one intended by the administrator.
+      </para>
+    </note>
+
+  <sect2 id="repmgrd-pausing-details">
+    <title>Details on the <application>repmgrd</application> pausing mechanism</title>
+
+    <para>
+      The pause state of each node will be stored over a PostgreSQL restart.
+    </para>
+
+	<para>
+	  <link linkend="repmgr-daemon-pause"><command>repmgr daemon pause</command></link> and
+	  <link linkend="repmgr-daemon-unpause"><command>repmgr daemon unpause</command></link> can be
+	  executed even if <application>repmgrd</application> is not running; in this case,
+	  <application>repmgrd</application> will start up in whichever pause state has been set.
+	</para>
+    <note>
+      <para>
+		<link linkend="repmgr-daemon-pause"><command>repmgr daemon pause</command></link> and
+		<link linkend="repmgr-daemon-unpause"><command>repmgr daemon unpause</command></link>
+		<emphasis>do not</emphasis> stop/start <application>repmgrd</application>.
+      </para>
+    </note>
+  </sect2>
+  </sect1>
+</chapter>
+

doc/switchover.sgml

@@ -19,9 +19,10 @@
  </para>
  <para>
   <command>repmgr standby switchover</command> differs from other &repmgr;
-  actions in that it also performs actions on another server (the demotion
+  actions in that it also performs actions on other servers (the demotion
-  candidate), which means passwordless SSH access is required to that server
+  candidate, and optionally any other servers which are to follow the new primary),
-  from the one where <command>repmgr standby switchover</command> is executed.
+  which means passwordless SSH access is required to those servers from the one where
+  <command>repmgr standby switchover</command> is executed.
  </para>
  <note>
   <simpara>
@@ -153,12 +154,18 @@
     manually with <command>repmgr node check --archive-ready</command>.
    </para>
 
-   <note>
+    <note>
-     <para>
+      <para>
-       Ensure that <application>repmgrd</application> is *not* running anywhere to prevent it unintentionally
+        From <link linkend="release-4.2">repmgr 4.2</link>, &repmgr; will instruct any running
-       promoting a node. This restriction will be removed in a future &repmgr; version.
+        <application>repmgrd</application> instances to pause operations while the switchover
-     </para>
+        is being carried out, to prevent <application>repmgrd</application> from
-   </note>
+        unintentionally promoting a node. For more details, see <xref linkend="repmgrd-pausing">.
+      </para>
+      <para>
+        Users of &repmgr; versions prior to 4.2 should ensure that <application>repmgrd</application>
+        is not running on any nodes while a switchover is being executed.
+      </para>
+    </note>
 
 
    <para>
@@ -303,7 +310,21 @@
      2  | node2 | primary | * running |          | default  | host=node2 dbname=repmgr user=repmgr
    </programlisting>
   </para>
+  <para>
+    If <application>repmgrd</application> is in use, it's worth double-checking that
+    all nodes are unpaused by executing <command><link linkend="repmgr-daemon-status">repmgr-daemon-status</link></command>.
+  </para>
+
+   <note>
+     <para>
+       Users of &repmgr; versions prior to 4.2 will need to manually restart <application>repmgrd</application>
+       on all nodes after the switchover is completed.
+     </para>
+    </note>
+
  </sect1>
+
+
  <sect1 id="switchover-caveats" xreflabel="Caveats">
   <indexterm>
    <primary>switchover</primary>
@@ -329,17 +350,76 @@
       for details.
      </simpara>
     </listitem>
-    <listitem>
-     <simpara>
-      <application>repmgrd</application> should not be running with setting <varname>failover=automatic</varname>
-      in <filename>repmgr.conf</filename> when a switchover is carried out, otherwise the
-      <application>repmgrd</application> daemon may try and promote a standby by itself.
-     </simpara>
-    </listitem>
    </itemizedlist>
   </para>
-  <para>
-   We hope to remove some of these restrictions in future versions of &repmgr;.
-  </para>
  </sect1>
+
+ <sect1 id="switchover-troubleshooting" xreflabel="Troubleshooting">
+   <indexterm>
+     <primary>switchover</primary>
+     <secondary>troubleshooting</secondary>
+   </indexterm>
+   <title>Troubleshooting switchover issues</title>
+
+   <para>
+     As <link linkend="performing-switchover">emphasised previously</link>, performing a switchover
+     is a non-trivial operation and there are a number of potential issues which can occur.
+     While &repmgr; attempts to perform sanity checks, there's no guaranteed way of determining the success of
+     a switchover without actually carrying it out.
+   </para>
+
+   <sect2 id="switchover-troubleshooting-primary-shutdown">
+     <title>Demotion candidate (old primary) does not shut down</title>
+     <para>
+       &repmgr; may abort a switchover with a message like:
+       <programlisting>
+ERROR: shutdown of the primary server could not be confirmed
+HINT: check the primary server status before performing any further actions</programlisting>
+     </para>
+     <para>
+       This means the shutdown of the old primary has taken longer than &repmgr; expected,
+       and it has given up waiting.
+     </para>
+     <para>
+       In this case, check the PostgreSQL log on the primary server to see what is going
+       on. It's entirely possible the shutdown process is just taking longer than the
+       timeout set by the configuration parameter <varname>shutdown_check_timeout</varname>
+       (default: 60 seconds), in which case you may need to adjust this parameter.
+     </para>
+     <note>
+       <para>
+         Note that <varname>shutdown_check_timeout</varname>is set on the node where
+         <command>repmgr standby switchover</command> is executed (promotion candidate); setting it on the
+         demotion candidate (former primary) will have no effect.
+       </para>
+     </note>
+     <para>
+       If the primary server has shut down cleanly, and no other node has been promoted,
+       it is safe to restart it, in which case the replication cluster will be restored
+       to its original configuration.
+     </para>
+   </sect2>
+
+   <sect2 id="switchover-troubleshooting-exclusive-backup">
+     <title>Switchover aborts with an &quot;exclusive backup&quot; error</title>
+     <para>
+       &repmgr; may abort a switchover with a message like:
+       <programlisting>
+ERROR: unable to perform a switchover while primary server is in exclusive backup mode
+HINT: stop backup before attempting the switchover</programlisting>
+     </para>
+     <para>
+       This means an exclusive backup is running on the current primary; interrupting this
+       will not only abort the backup, but potentially leave the primary with an ambiguous
+       backup state.
+     </para>
+     <para>
+       To proceed, either wait until the backup has finished, or cancel it with the command
+       <command>SELECT pg_stop_backup()</command>. For more details see the PostgreSQL
+       documentation section
+       <ulink url="https://www.postgresql.org/docs/current/static/continuous-archiving.html#BACKUP-LOWLEVEL-BASE-BACKUP-EXCLUSIVE">Making an exclusive low level backup</ulink>.
+     </para>
+   </sect2>
+ </sect1>
+
 </chapter>

doc/upgrading-repmgr.sgml

@@ -7,9 +7,9 @@
  <title>Upgrading repmgr</title>
 
  <para>
-  &repmgr; is updated regularly with point releases (e.g. 4.0.1 to 4.0.2)
+  &repmgr; is updated regularly with minor releases (e.g. 4.0.1 to 4.0.2)
   containing bugfixes and other minor improvements. Any substantial new
-  functionality will be included in a feature release (e.g. 4.0.x to 4.1.x).
+  functionality will be included in a major release (e.g. 4.0 to 4.1).
  </para>
 
  <sect1 id="upgrading-repmgr-extension" xreflabel="Upgrading repmgr 4.x and later">
@@ -19,43 +19,201 @@
   </indexterm>
   <title>Upgrading repmgr 4.x and later</title>
   <para>
-    &repmgr; 4.x is implemented as a PostgreSQL extension; normally the upgrade consists
+    From version 4, &repmgr; consists of three elements:
-    of the two following steps:
+     <itemizedlist spacing="compact" mark="bullet">
-    <orderedlist>
+
-      <listitem>
+       <listitem>
-        <simpara>
+         <simpara>
-          Install the updated package (or compile the updated source)
+           the <application>repmgr</application> and <application>repmgrd</application> executables
-        </simpara>
+         </simpara>
-      </listitem>
+       </listitem>
-      <listitem>
+
-        <simpara>
+       <listitem>
-          <application>repmgrd</application> (if running) must be restarted.
+         <simpara>
-        </simpara>
+           the objects for the &repmgr; PostgreSQL extension (SQL files for creating/updating
-      </listitem>
+           repmgr metadata, and the extension control file)
-      <listitem>
+         </simpara>
-        <simpara>
+       </listitem>
-          For major releases, e.g. from <literal>4.0.x</literal> to <literal>4.1</literal>,
+
-          execute <command>ALTER EXTENSION repmgr UPDATE</command>
+       <listitem>
-          on the primary node in the database where the &repmgr; extension is installed.
+         <simpara>
-        </simpara>
+           the shared library module used by <application>repmgrd</application> which
-        <simpara>
+           is resident in the PostgreSQL backend
-          This will update the extension metadata and, if necessary, apply
+         </simpara>
-          changes to the &repmgr; extension objects.
+       </listitem>
-        </simpara>
+     </itemizedlist>
-      </listitem>
+  </para>
-    </orderedlist>
+  <para>
+    With <emphasis>minor releases</emphasis>, usually changes are only made to the <application>repmgr</application>
+    and <application>repmgrd</application> executables. In this case, the upgrade is quite straightforward,
+    and is simply a case of installing the new version, and restarting <application>repmgrd</application>
+    (if running).
   </para>
 
   <para>
-    Always check the <link linkend="appendix-release-notes">release notes</link> for every
+    For <emphasis>major releases</emphasis>, the &repmgr; PostgreSQL extension will need to be updated
-    release as they may contain upgrade instructions particular to individual versions.
+    to the latest version. Additionally, if the shared library module has been updated (this is sometimes,
+    but not always the case), PostgreSQL itself will need to be restarted on each node.
   </para>
+  <important>
+    <para>
+      Always check the <link linkend="appendix-release-notes">release notes</link> for every
+      release as they may contain upgrade instructions particular to individual versions.
+    </para>
+  </important>
 
-  <para>
+  <sect2 id="upgrading-minor-version" xreflabel="Upgrading a minor version release">
-    Note that it may be necessary to restart the PostgreSQL server if the upgrade contains
+	<indexterm>
-    changes to the shared object file used by <application>repmgrd</application>; check the
+	  <primary>upgrading</primary>
-    release notes for details.
+	  <secondary>minor release</secondary>
-  </para>
+	</indexterm>
+	<title>Upgrading a minor version release</title>
+
+    <para>
+      The process for installing minor version upgrades is quite straightforward:
+
+      <itemizedlist spacing="compact" mark="bullet">
+
+        <listitem>
+          <simpara>
+            install the new &repmgr; version
+          </simpara>
+        </listitem>
+
+        <listitem>
+          <simpara>
+            restart <application>repmgrd</application> on all nodes where it is running
+          </simpara>
+        </listitem>
+
+      </itemizedlist>
+
+    </para>
+
+    <note>
+	  <para>
+        Some packaging systems (e.g. <link linkend="packages-debian-ubuntu">Debian/Ubuntu</link>
+        may restart <application>repmgrd</application> as part of the package upgrade process.
+      </para>
+    </note>
+
+	<para>
+	  Minor version upgrades can be performed in any order on the nodes in the replication
+	  cluster.
+	</para>
+
+	<para>
+	  A PostgreSQL restart is <emphasis>not</emphasis> required for minor version upgrades.
+	</para>
+
+    <note>
+	  <para>
+	    The same &repmgr; &quot;major version&quot; (e.g. <literal>4.2</literal>) must be
+	    installed on all nodes in the replication cluster. While it's possible to have differing
+	    &repmgr; &quot;minor versions&quot; (e.g. <literal>4.2.1</literal>)  on different nodes,
+	    we strongly recommend updating all nodes to the latest minor version.
+	  </para>
+    </note>
+
+  </sect2>
+
+  <sect2 id="upgrading-major-version" xreflabel="Upgrading a major version release">
+	<indexterm>
+	  <primary>upgrading</primary>
+	  <secondary>major release</secondary>
+	</indexterm>
+	<title>Upgrading a major version release</title>
+	<para>
+	  &quot;major version&quot; upgrades need to be planned more carefully, as they may include
+	  changes to the &repmgr; metadata (which need to be propagated from the primary to all
+	  standbys) and/or changes to the shared object file used by <application>repmgrd</application>
+	  (which require a PostgreSQL restart).
+	</para>
+	<para>
+	  With this in mind,
+	</para>
+
+	<para>
+      <orderedlist>
+
+		<listitem>
+          <simpara>
+			Stop <application>repmgrd</application> (if in use) on all nodes where it is running.
+          </simpara>
+		</listitem>
+
+		<listitem>
+          <simpara>
+			Disable the <application>repmgrd</application> service on all nodes where it is in use;
+            this is to prevent packages from prematurely restarting <application>repmgrd</application>.
+          </simpara>
+		</listitem>
+
+		<listitem>
+          <simpara>
+			Install the updated package (or compile the updated source) on all nodes.
+          </simpara>
+		</listitem>
+
+        <listitem>
+          <para>
+            If running a <literal>systemd</literal>-based Linux distribution, execute (as <literal>root</literal>,
+            or with appropriate <literal>sudo</literal> permissions):
+            <programlisting>
+systemctl daemon-reload</programlisting>
+          </para>
+        </listitem>
+
+		<listitem>
+          <simpara>
+			If the &repmgr; shared library module has been updated (check the <link linkend="appendix-release-notes">release notes</link>!),
+            restart PostgreSQL, then <application>repmgrd</application> (if in use)	on each node,
+            The order in which this is applied to individual nodes is not critical,
+			and it's also fine to restart PostgreSQL on all nodes first before starting <application>repmgrd</application>.
+		  </simpara>
+		  <simpara>
+			Note that if the upgrade requires a PostgreSQL restart, <application>repmgrd</application>
+			will only function correctly once all nodes have been restarted.
+          </simpara>
+		</listitem>
+
+		<listitem>
+          <para>
+			On the primary node, execute
+			<programlisting>
+ALTER EXTENSION repmgr UPDATE</programlisting>
+			in the database where &repmgr; is installed.
+          </para>
+		</listitem>
+
+		<listitem>
+          <simpara>
+			Reenable the <application>repmgrd</application> service on all nodes where it is in use.
+          </simpara>
+		</listitem>
+
+	  </orderedlist>
+	</para>
+	<tip>
+	  <para>
+		If the &repmgr; upgrade requires a PostgreSQL restart, combine the &repmgr; upgrade
+		with a PostgreSQL minor version upgrade, which will require a restart in any case.
+		New PostgreSQL minor version are usually released every couple of months.
+	  </para>
+	</tip>
+  </sect2>
+
+  <sect2 id="upgrading-check-repmgrd" xreflabel="Checking repmgrd status after an upgrade">
+	<indexterm>
+	  <primary>upgrading</primary>
+	  <secondary>checking repmgrd status</secondary>
+	</indexterm>
+	<title>Checking repmgrd status after an upgrade</title>
+	<para>
+      From &repmgr; 4.2, once the upgrade is complete, execute the <command><link linkend="repmgr-daemon-status">repmgr daemon status</link></command>
+      command (on any node) to show an overview of the status of <application>repmgrd</application> on all nodes.
+    </para>
+  </sect2>
  </sect1>
 
  <sect1 id="upgrading-and-pg-upgrade" xreflabel="pg_upgrade and repmgr">
@@ -95,6 +253,13 @@
     be recreated by <application>pg_upgrade</application>. These will need to
     be recreated manually.
   </para>
+  <tip>
+	<para>
+	  Use <command><link linkend="repmgr-node-check">repmgr node check</link></command>
+	  to determine which replacation slots need to be recreated.
+	</para>
+  </tip>
+
  </sect1>
 
 

doc/version.sgml

@@ -1 +1 @@
-<!ENTITY repmgrversion "4.1.1">
+<!ENTITY repmgrversion "4.2">

errcode.h

@@ -47,5 +47,6 @@
 #define ERR_FOLLOW_FAIL 23
 #define ERR_REJOIN_FAIL 24
 #define ERR_NODE_STATUS 25
+#define ERR_REPMGRD_PAUSE 26
 
 #endif							/* _ERRCODE_H_ */

repmgr--4.1--4.2.sql

@@ -0,0 +1,32 @@
+-- complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "CREATE EXTENSION repmgr" to load this file. \quit
+
+CREATE FUNCTION get_repmgrd_pid()
+  RETURNS INT
+  AS 'MODULE_PATHNAME', 'get_repmgrd_pid'
+  LANGUAGE C STRICT;
+
+CREATE FUNCTION get_repmgrd_pidfile()
+  RETURNS TEXT
+  AS 'MODULE_PATHNAME', 'get_repmgrd_pidfile'
+  LANGUAGE C STRICT;
+
+CREATE FUNCTION set_repmgrd_pid(INT, TEXT)
+  RETURNS VOID
+  AS 'MODULE_PATHNAME', 'set_repmgrd_pid'
+  LANGUAGE C STRICT;
+
+CREATE FUNCTION repmgrd_is_running()
+  RETURNS BOOL
+  AS 'MODULE_PATHNAME', 'repmgrd_is_running'
+  LANGUAGE C STRICT;
+
+CREATE FUNCTION repmgrd_pause(BOOL)
+  RETURNS VOID
+  AS 'MODULE_PATHNAME', 'repmgrd_pause'
+  LANGUAGE C STRICT;
+
+CREATE FUNCTION repmgrd_is_paused()
+  RETURNS BOOL
+  AS 'MODULE_PATHNAME', 'repmgrd_is_paused'
+  LANGUAGE C STRICT;

repmgr--4.1.sql

@@ -145,7 +145,6 @@ CREATE FUNCTION unset_bdr_failover_handler()
   AS 'MODULE_PATHNAME', 'unset_bdr_failover_handler'
   LANGUAGE C STRICT;
 
-
 CREATE VIEW repmgr.replication_status AS
   SELECT m.primary_node_id, m.standby_node_id, n.node_name AS standby_name,
  	     n.type AS node_type, n.active, last_monitor_time,

repmgr--4.2.sql

@@ -0,0 +1,197 @@
+-- complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "CREATE EXTENSION repmgr" to load this file. \quit
+
+CREATE TABLE repmgr.nodes (
+  node_id          INTEGER     PRIMARY KEY,
+  upstream_node_id INTEGER     NULL REFERENCES nodes (node_id) DEFERRABLE,
+  active           BOOLEAN     NOT NULL DEFAULT TRUE,
+  node_name        TEXT        NOT NULL,
+  type             TEXT        NOT NULL CHECK (type IN('primary','standby','witness','bdr')),
+  location         TEXT        NOT NULL DEFAULT 'default',
+  priority         INT         NOT NULL DEFAULT 100,
+  conninfo         TEXT        NOT NULL,
+  repluser         VARCHAR(63) NOT NULL,
+  slot_name        TEXT        NULL,
+  config_file      TEXT        NOT NULL
+);
+
+CREATE TABLE repmgr.events (
+  node_id          INTEGER NOT NULL,
+  event            TEXT NOT NULL,
+  successful       BOOLEAN NOT NULL DEFAULT TRUE,
+  event_timestamp  TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  details          TEXT NULL
+);
+
+DO $repmgr$
+DECLARE
+  DECLARE server_version_num INT;
+BEGIN
+  SELECT setting
+    FROM pg_catalog.pg_settings
+   WHERE name = 'server_version_num'
+    INTO server_version_num;
+  IF server_version_num >= 90400 THEN
+    EXECUTE $repmgr_func$
+CREATE TABLE repmgr.monitoring_history (
+  primary_node_id                INTEGER NOT NULL,
+  standby_node_id                INTEGER NOT NULL,
+  last_monitor_time              TIMESTAMP WITH TIME ZONE NOT NULL,
+  last_apply_time                TIMESTAMP WITH TIME ZONE,
+  last_wal_primary_location      PG_LSN NOT NULL,
+  last_wal_standby_location      PG_LSN,
+  replication_lag                BIGINT NOT NULL,
+  apply_lag                      BIGINT NOT NULL
+)
+    $repmgr_func$;
+  ELSE
+    EXECUTE $repmgr_func$
+CREATE TABLE repmgr.monitoring_history (
+  primary_node_id                INTEGER NOT NULL,
+  standby_node_id                INTEGER NOT NULL,
+  last_monitor_time              TIMESTAMP WITH TIME ZONE NOT NULL,
+  last_apply_time                TIMESTAMP WITH TIME ZONE,
+  last_wal_primary_location      TEXT NOT NULL,
+  last_wal_standby_location      TEXT,
+  replication_lag                BIGINT NOT NULL,
+  apply_lag                      BIGINT NOT NULL
+)
+    $repmgr_func$;
+  END IF;
+END$repmgr$;
+
+
+
+CREATE INDEX idx_monitoring_history_time
+          ON repmgr.monitoring_history (last_monitor_time, standby_node_id);
+
+CREATE VIEW repmgr.show_nodes AS
+   SELECT n.node_id,
+          n.node_name,
+          n.active,
+          n.upstream_node_id,
+          un.node_name AS upstream_node_name,
+          n.type,
+          n.priority,
+          n.conninfo
+     FROM repmgr.nodes n
+LEFT JOIN repmgr.nodes un
+       ON un.node_id = n.upstream_node_id;
+
+
+/* XXX update upgrade scripts! */
+CREATE TABLE repmgr.voting_term (
+  term INT NOT NULL
+);
+
+CREATE UNIQUE INDEX voting_term_restrict
+ON repmgr.voting_term ((TRUE));
+
+CREATE RULE voting_term_delete AS
+   ON DELETE TO repmgr.voting_term
+   DO INSTEAD NOTHING;
+
+
+/* ================= */
+/* repmgrd functions */
+/* ================= */
+
+/* monitoring functions */
+
+CREATE FUNCTION set_local_node_id(INT)
+  RETURNS VOID
+  AS 'MODULE_PATHNAME', 'set_local_node_id'
+  LANGUAGE C STRICT;
+
+CREATE FUNCTION get_local_node_id()
+  RETURNS INT
+  AS 'MODULE_PATHNAME', 'get_local_node_id'
+  LANGUAGE C STRICT;
+
+CREATE FUNCTION standby_set_last_updated()
+  RETURNS TIMESTAMP WITH TIME ZONE
+  AS 'MODULE_PATHNAME', 'standby_set_last_updated'
+  LANGUAGE C STRICT;
+
+CREATE FUNCTION standby_get_last_updated()
+  RETURNS TIMESTAMP WITH TIME ZONE
+  AS 'MODULE_PATHNAME', 'standby_get_last_updated'
+  LANGUAGE C STRICT;
+
+/* failover functions */
+
+CREATE FUNCTION notify_follow_primary(INT)
+  RETURNS VOID
+  AS 'MODULE_PATHNAME', 'notify_follow_primary'
+  LANGUAGE C STRICT;
+
+CREATE FUNCTION get_new_primary()
+  RETURNS INT
+  AS 'MODULE_PATHNAME', 'get_new_primary'
+  LANGUAGE C STRICT;
+
+CREATE FUNCTION reset_voting_status()
+  RETURNS VOID
+  AS 'MODULE_PATHNAME', 'reset_voting_status'
+  LANGUAGE C STRICT;
+
+CREATE FUNCTION am_bdr_failover_handler(INT)
+  RETURNS BOOL
+  AS 'MODULE_PATHNAME', 'am_bdr_failover_handler'
+  LANGUAGE C STRICT;
+
+CREATE FUNCTION unset_bdr_failover_handler()
+  RETURNS VOID
+  AS 'MODULE_PATHNAME', 'unset_bdr_failover_handler'
+  LANGUAGE C STRICT;
+
+CREATE FUNCTION get_repmgrd_pid()
+  RETURNS INT
+  AS 'MODULE_PATHNAME', 'get_repmgrd_pid'
+  LANGUAGE C STRICT;
+
+CREATE FUNCTION get_repmgrd_pidfile()
+  RETURNS TEXT
+  AS 'MODULE_PATHNAME', 'get_repmgrd_pidfile'
+  LANGUAGE C STRICT;
+
+CREATE FUNCTION set_repmgrd_pid(INT, TEXT)
+  RETURNS VOID
+  AS 'MODULE_PATHNAME', 'set_repmgrd_pid'
+  LANGUAGE C STRICT;
+
+CREATE FUNCTION repmgrd_is_running()
+  RETURNS BOOL
+  AS 'MODULE_PATHNAME', 'repmgrd_is_running'
+  LANGUAGE C STRICT;
+
+CREATE FUNCTION repmgrd_pause(BOOL)
+  RETURNS VOID
+  AS 'MODULE_PATHNAME', 'repmgrd_pause'
+  LANGUAGE C STRICT;
+
+CREATE FUNCTION repmgrd_is_paused()
+  RETURNS BOOL
+  AS 'MODULE_PATHNAME', 'repmgrd_is_paused'
+  LANGUAGE C STRICT;
+
+
+CREATE VIEW repmgr.replication_status AS
+  SELECT m.primary_node_id, m.standby_node_id, n.node_name AS standby_name,
+ 	     n.type AS node_type, n.active, last_monitor_time,
+         CASE WHEN n.type='standby' THEN m.last_wal_primary_location ELSE NULL END AS last_wal_primary_location,
+         m.last_wal_standby_location,
+         CASE WHEN n.type='standby' THEN pg_catalog.pg_size_pretty(m.replication_lag) ELSE NULL END AS replication_lag,
+         CASE WHEN n.type='standby' THEN
+           CASE WHEN replication_lag > 0 THEN age(now(), m.last_apply_time) ELSE '0'::INTERVAL END
+           ELSE NULL
+         END AS replication_time_lag,
+         CASE WHEN n.type='standby' THEN pg_catalog.pg_size_pretty(m.apply_lag) ELSE NULL END AS apply_lag,
+         AGE(NOW(), CASE WHEN pg_catalog.pg_is_in_recovery() THEN repmgr.standby_get_last_updated() ELSE m.last_monitor_time END) AS communication_time_lag
+    FROM repmgr.monitoring_history m
+    JOIN repmgr.nodes n ON m.standby_node_id = n.node_id
+   WHERE (m.standby_node_id, m.last_monitor_time) IN (
+	          SELECT m1.standby_node_id, MAX(m1.last_monitor_time)
+			    FROM repmgr.monitoring_history m1 GROUP BY 1
+         );
+

repmgr-action-bdr.c

@@ -191,7 +191,7 @@ do_bdr_register(void)
 	{
 		NodeInfoList local_node_records = T_NODE_INFO_LIST_INITIALIZER;
 
-		get_all_node_records(conn, &local_node_records);
+		(void) get_all_node_records(conn, &local_node_records);
 
 		if (local_node_records.node_count == 0)
 		{
@@ -239,7 +239,7 @@ do_bdr_register(void)
 					continue;
 				}
 
-				get_all_node_records(bdr_node_conn, &existing_nodes);
+				(void) get_all_node_records(bdr_node_conn, &existing_nodes);
 
 				for (cell = existing_nodes.head; cell; cell = cell->next)
 				{

repmgr-action-cluster.c

@@ -26,7 +26,6 @@
 
 #define SHOW_HEADER_COUNT 7
 
-
 typedef enum
 {
 	SHOW_ID = 0,
@@ -51,21 +50,13 @@ typedef enum
 }			EventHeader;
 
 
-
-struct ColHeader
-{
-	char		title[MAXLEN];
-	int			max_length;
-	int			cur_length;
-};
-
 struct ColHeader headers_show[SHOW_HEADER_COUNT];
 struct ColHeader headers_event[EVENT_HEADER_COUNT];
 
 
 
-static int	build_cluster_matrix(t_node_matrix_rec ***matrix_rec_dest, int *name_length);
+static int	build_cluster_matrix(t_node_matrix_rec ***matrix_rec_dest, int *name_length, ItemList *warnings, int *error_code);
-static int	build_cluster_crosscheck(t_node_status_cube ***cube_dest, int *name_length);
+static int	build_cluster_crosscheck(t_node_status_cube ***cube_dest, int *name_length, ItemList *warnings, int *error_code);
 static void cube_set_node_status(t_node_status_cube **cube, int n, int node_id, int matrix_node_id, int connection_node_id, int connection_status);
 
 /*
@@ -159,7 +150,7 @@ do_cluster_show(void)
 			else
 			{
 				item_list_append_format(&warnings,
-										"unable to  connect to node \"%s\" (ID: %i)",
+										"unable to connect to node \"%s\" (ID: %i)",
 										cell->node_info->node_name, cell->node_info->node_id);
 			}
 		}
@@ -183,16 +174,16 @@ do_cluster_show(void)
 							switch (cell->node_info->recovery_type)
 							{
 								case RECTYPE_PRIMARY:
-									appendPQExpBuffer(&details, "* running");
+									appendPQExpBufferStr(&details, "* running");
 									break;
 								case RECTYPE_STANDBY:
-									appendPQExpBuffer(&details, "! running as standby");
+									appendPQExpBufferStr(&details, "! running as standby");
 									item_list_append_format(&warnings,
 															"node \"%s\" (ID: %i) is registered as primary but running as standby",
 															cell->node_info->node_name, cell->node_info->node_id);
 									break;
 								case RECTYPE_UNKNOWN:
-									appendPQExpBuffer(&details, "! unknown");
+									appendPQExpBufferStr(&details, "! unknown");
 									item_list_append_format(&warnings,
 															"node \"%s\" (ID: %i) has unknown replication status",
 															cell->node_info->node_name, cell->node_info->node_id);
@@ -203,14 +194,14 @@ do_cluster_show(void)
 						{
 							if (cell->node_info->recovery_type == RECTYPE_PRIMARY)
 							{
-								appendPQExpBuffer(&details, "! running");
+								appendPQExpBufferStr(&details, "! running");
 								item_list_append_format(&warnings,
 														"node \"%s\" (ID: %i) is running but the repmgr node record is inactive",
 														cell->node_info->node_name, cell->node_info->node_id);
 							}
 							else
 							{
-								appendPQExpBuffer(&details, "! running as standby");
+								appendPQExpBufferStr(&details, "! running as standby");
 								item_list_append_format(&warnings,
 														"node \"%s\" (ID: %i) is registered as an inactive primary but running as standby",
 														cell->node_info->node_name, cell->node_info->node_id);
@@ -223,7 +214,7 @@ do_cluster_show(void)
 						/* node is unreachable but marked active */
 						if (cell->node_info->active == true)
 						{
-							appendPQExpBuffer(&details, "? unreachable");
+							appendPQExpBufferStr(&details, "? unreachable");
 							item_list_append_format(&warnings,
 													"node \"%s\" (ID: %i) is registered as an active primary but is unreachable",
 													cell->node_info->node_name, cell->node_info->node_id);
@@ -231,7 +222,7 @@ do_cluster_show(void)
 						/* node is unreachable and marked as inactive */
 						else
 						{
-							appendPQExpBuffer(&details, "- failed");
+							appendPQExpBufferStr(&details, "- failed");
 							error_found = true;
 						}
 					}
@@ -247,16 +238,16 @@ do_cluster_show(void)
 							switch (cell->node_info->recovery_type)
 							{
 								case RECTYPE_STANDBY:
-									appendPQExpBuffer(&details, "  running");
+									appendPQExpBufferStr(&details, "  running");
 									break;
 								case RECTYPE_PRIMARY:
-									appendPQExpBuffer(&details, "! running as primary");
+									appendPQExpBufferStr(&details, "! running as primary");
 									item_list_append_format(&warnings,
 															"node \"%s\" (ID: %i) is registered as standby but running as primary",
 															cell->node_info->node_name, cell->node_info->node_id);
 									break;
 								case RECTYPE_UNKNOWN:
-									appendPQExpBuffer(&details, "! unknown");
+									appendPQExpBufferStr(&details, "! unknown");
 									item_list_append_format(
 															&warnings,
 															"node \"%s\" (ID: %i) has unknown replication status",
@@ -268,14 +259,14 @@ do_cluster_show(void)
 						{
 							if (cell->node_info->recovery_type == RECTYPE_STANDBY)
 							{
-								appendPQExpBuffer(&details, "! running");
+								appendPQExpBufferStr(&details, "! running");
 								item_list_append_format(&warnings,
 														"node \"%s\" (ID: %i) is running but the repmgr node record is inactive",
 														cell->node_info->node_name, cell->node_info->node_id);
 							}
 							else
 							{
-								appendPQExpBuffer(&details, "! running as primary");
+								appendPQExpBufferStr(&details, "! running as primary");
 								item_list_append_format(&warnings,
 														"node \"%s\" (ID: %i) is running as primary but the repmgr node record is inactive",
 														cell->node_info->node_name, cell->node_info->node_id);
@@ -288,14 +279,14 @@ do_cluster_show(void)
 						/* node is unreachable but marked active */
 						if (cell->node_info->active == true)
 						{
-							appendPQExpBuffer(&details, "? unreachable");
+							appendPQExpBufferStr(&details, "? unreachable");
 							item_list_append_format(&warnings,
 													"node \"%s\" (ID: %i) is registered as an active standby but is unreachable",
 													cell->node_info->node_name, cell->node_info->node_id);
 						}
 						else
 						{
-							appendPQExpBuffer(&details, "- failed");
+							appendPQExpBufferStr(&details, "- failed");
 							error_found = true;
 						}
 					}
@@ -309,11 +300,11 @@ do_cluster_show(void)
 					{
 						if (cell->node_info->active == true)
 						{
-							appendPQExpBuffer(&details, "* running");
+							appendPQExpBufferStr(&details, "* running");
 						}
 						else
 						{
-							appendPQExpBuffer(&details, "! running");
+							appendPQExpBufferStr(&details, "! running");
 							error_found = true;
 						}
 					}
@@ -322,11 +313,11 @@ do_cluster_show(void)
 					{
 						if (cell->node_info->active == true)
 						{
-							appendPQExpBuffer(&details, "? unreachable");
+							appendPQExpBufferStr(&details, "? unreachable");
 						}
 						else
 						{
-							appendPQExpBuffer(&details, "- failed");
+							appendPQExpBufferStr(&details, "- failed");
 							error_found = true;
 						}
 					}
@@ -335,7 +326,7 @@ do_cluster_show(void)
 			case UNKNOWN:
 				{
 					/* this should never happen */
-					appendPQExpBuffer(&details, "? unknown node type");
+					appendPQExpBufferStr(&details, "? unknown node type");
 						error_found = true;
 				}
 				break;
@@ -364,36 +355,10 @@ do_cluster_show(void)
 
 	}
 
+	/* Print column header row (text mode only) */
 	if (runtime_options.output_mode == OM_TEXT)
 	{
-		for (i = 0; i < SHOW_HEADER_COUNT; i++)
+		print_status_header(SHOW_HEADER_COUNT, headers_show);
-		{
-			if (i == 0)
-				printf(" ");
-			else
-				printf(" | ");
-
-			printf("%-*s",
-				   headers_show[i].max_length,
-				   headers_show[i].title);
-		}
-		printf("\n");
-		printf("-");
-
-		for (i = 0; i < SHOW_HEADER_COUNT; i++)
-		{
-			int			j;
-
-			for (j = 0; j < headers_show[i].max_length; j++)
-				printf("-");
-
-			if (i < (SHOW_HEADER_COUNT - 1))
-				printf("-+-");
-			else
-				printf("-");
-		}
-
-		printf("\n");
 	}
 
 	for (cell = nodes.head; cell; cell = cell->next)
@@ -637,9 +602,12 @@ do_cluster_crosscheck(void)
 
 	t_node_status_cube **cube;
 
-	bool		error_found = false;
+	bool		connection_error_found = false;
+	int			error_code = SUCCESS;
+	ItemList	warnings = {NULL, NULL};
+
+	n = build_cluster_crosscheck(&cube, &name_length, &warnings, &error_code);
 
-	n = build_cluster_crosscheck(&cube, &name_length);
 	if (runtime_options.output_mode == OM_CSV)
 	{
 		for (i = 0; i < n; i++)
@@ -661,6 +629,11 @@ do_cluster_crosscheck(void)
 					   cube[i]->node_id,
 					   cube[j]->node_id,
 					   max_node_status);
+
+				if (max_node_status == -1)
+				{
+					connection_error_found = true;
+				}
 			}
 
 		}
@@ -718,16 +691,16 @@ do_cluster_crosscheck(void)
 				{
 					case -2:
 						c = '?';
-						error_found = true;
 						break;
 					case -1:
 						c = 'x';
-						error_found = true;
+						connection_error_found = true;
 						break;
 					case 0:
 						c = '*';
 						break;
 					default:
+						log_error("unexpected node status value %i", max_node_status);
 						exit(ERR_INTERNAL);
 				}
 
@@ -736,6 +709,13 @@ do_cluster_crosscheck(void)
 
 			printf("\n");
 		}
+
+		if (warnings.head != NULL && runtime_options.terse == false)
+		{
+			log_warning(_("following problems detected:"));
+			print_item_list(&warnings);
+		}
+
 	}
 
 	/* clean up allocated cube array */
@@ -762,13 +742,23 @@ do_cluster_crosscheck(void)
 		free(cube);
 	}
 
-	if (error_found == true)
+	/* errors detected by build_cluster_crosscheck() have priority */
+	if (connection_error_found == true)
 	{
-		exit(ERR_NODE_STATUS);
+		error_code = ERR_NODE_STATUS;
 	}
+
+	exit(error_code);
+
 }
 
 
+/*
+ * CLUSTER MATRIX
+ *
+ * Parameters:
+ *   --csv
+ */
 void
 do_cluster_matrix()
 {
@@ -781,18 +771,30 @@ do_cluster_matrix()
 
 	t_node_matrix_rec **matrix_rec_list;
 
-	bool		error_found = false;
+	bool		connection_error_found = false;
+	int			error_code = SUCCESS;
+	ItemList	warnings = {NULL, NULL};
 
-	n = build_cluster_matrix(&matrix_rec_list, &name_length);
+	n = build_cluster_matrix(&matrix_rec_list, &name_length, &warnings, &error_code);
 
 	if (runtime_options.output_mode == OM_CSV)
 	{
 		for (i = 0; i < n; i++)
+		{
 			for (j = 0; j < n; j++)
+			{
 				printf("%d,%d,%d\n",
 					   matrix_rec_list[i]->node_id,
 					   matrix_rec_list[i]->node_status_list[j]->node_id,
 					   matrix_rec_list[i]->node_status_list[j]->node_status);
+
+				if (matrix_rec_list[i]->node_status_list[j]->node_status == -2
+					|| matrix_rec_list[i]->node_status_list[j]->node_status == -1)
+				{
+					connection_error_found = true;
+				}
+			}
+		}
 	}
 	else
 	{
@@ -821,16 +823,16 @@ do_cluster_matrix()
 				{
 					case -2:
 						c = '?';
-						error_found = true;
 						break;
 					case -1:
 						c = 'x';
-						error_found = true;
+						connection_error_found = true;
 						break;
 					case 0:
 						c = '*';
 						break;
 					default:
+						log_error("unexpected node status value %i", matrix_rec_list[i]->node_status_list[j]->node_status);
 						exit(ERR_INTERNAL);
 				}
 
@@ -838,6 +840,13 @@ do_cluster_matrix()
 			}
 			printf("\n");
 		}
+
+		if (warnings.head != NULL && runtime_options.terse == false)
+		{
+			log_warning(_("following problems detected:"));
+			print_item_list(&warnings);
+		}
+
 	}
 
 	for (i = 0; i < n; i++)
@@ -852,10 +861,13 @@ do_cluster_matrix()
 
 	free(matrix_rec_list);
 
-	if (error_found == true)
+	/* actual database connection errors have priority */
+	if (connection_error_found == true)
 	{
-		exit(ERR_NODE_STATUS);
+		error_code = ERR_NODE_STATUS;
 	}
+
+	exit(error_code);
 }
 
 
@@ -884,7 +896,7 @@ matrix_set_node_status(t_node_matrix_rec **matrix_rec_list, int n, int node_id,
 
 
 static int
-build_cluster_matrix(t_node_matrix_rec ***matrix_rec_dest, int *name_length)
+build_cluster_matrix(t_node_matrix_rec ***matrix_rec_dest, int *name_length, ItemList *warnings, int *error_code)
 {
 	PGconn	   *conn = NULL;
 	int			i = 0,
@@ -913,7 +925,12 @@ build_cluster_matrix(t_node_matrix_rec ***matrix_rec_dest, int *name_length)
 		local_node_id = runtime_options.node_id;
 	}
 
-	get_all_node_records(conn, &nodes);
+	if (get_all_node_records(conn, &nodes) == false)
+	{
+		/* get_all_node_records() will display the error */
+		PQfinish(conn);
+		exit(ERR_BAD_CONFIG);
+	}
 
 	PQfinish(conn);
 	conn = NULL;
@@ -927,7 +944,7 @@ build_cluster_matrix(t_node_matrix_rec ***matrix_rec_dest, int *name_length)
 	/*
 	 * Allocate an empty matrix record list
 	 *
-	 * -2 == NULL  ? -1 == Error x 0 == OK    *
+	 * -2 == NULL  ? -1 == Error x 0 == OK
 	 */
 
 	matrix_rec_list = (t_node_matrix_rec **) pg_malloc0(sizeof(t_node_matrix_rec) * nodes.node_count);
@@ -990,7 +1007,7 @@ build_cluster_matrix(t_node_matrix_rec ***matrix_rec_dest, int *name_length)
 
 		host = param_get(&remote_conninfo, "host");
 
-		node_conn = establish_db_connection(cell->node_info->conninfo, false);
+		node_conn = establish_db_connection_quiet(cell->node_info->conninfo);
 
 		connection_status =
 			(PQstatus(node_conn) == CONNECTION_OK) ? 0 : -1;
@@ -1027,24 +1044,12 @@ build_cluster_matrix(t_node_matrix_rec ***matrix_rec_dest, int *name_length)
 		 * remote repmgr - those are the only values it needs to work, and
 		 * saves us making assumptions about the location of repmgr.conf
 		 */
-		appendPQExpBuffer(&command,
+		appendPQExpBufferChar(&command, '"');
-						  "\"%s -d '%s' ",
-						  make_pg_path(progname()),
-						  cell->node_info->conninfo);
 
+		make_remote_repmgr_path(&command, cell->node_info);
 
-		if (strlen(pg_bindir))
+		appendPQExpBufferStr(&command,
-		{
+							 " cluster show --csv -L NOTICE --terse\"");
-			appendPQExpBuffer(&command,
-							  "--pg_bindir=");
-			appendShellString(&command,
-							  pg_bindir);
-			appendPQExpBuffer(&command,
-							  " ");
-		}
-
-		appendPQExpBuffer(&command,
-						  " cluster show --csv\"");
 
 		log_verbose(LOG_DEBUG, "build_cluster_matrix(): executing:\n  %s", command.data);
 
@@ -1059,32 +1064,50 @@ build_cluster_matrix(t_node_matrix_rec ***matrix_rec_dest, int *name_length)
 
 		termPQExpBuffer(&command);
 
-		for (j = 0; j < nodes.node_count; j++)
+		/* no output returned - probably SSH error */
+		if (p[0] == '\0' || p[0] == '\n')
 		{
-			if (sscanf(p, "%d,%d", &x, &y) != 2)
+			item_list_append_format(warnings,
+									"node %i inaccessible via SSH",
+									connection_node_id);
+			*error_code = ERR_BAD_SSH;
+		}
+		else
+		{
+			for (j = 0; j < nodes.node_count; j++)
 			{
-				fprintf(stderr, _("cannot parse --csv output: %s\n"), p);
+				if (sscanf(p, "%d,%d", &x, &y) != 2)
-				PQfinish(node_conn);
+				{
-				exit(ERR_INTERNAL);
+					matrix_set_node_status(matrix_rec_list,
+										   nodes.node_count,
+										   connection_node_id,
+										   x,
+										   -2);
+
+					item_list_append_format(warnings,
+											"unable to parse --csv output for node %i; output returned was:\n\"%s\"",
+											connection_node_id, p);
+					*error_code = ERR_INTERNAL;
+				}
+				else
+				{
+					matrix_set_node_status(matrix_rec_list,
+										   nodes.node_count,
+										   connection_node_id,
+										   x,
+										   (y == -1) ? -1 : 0);
+				}
+
+				while (*p && (*p != '\n'))
+					p++;
+				if (*p == '\n')
+					p++;
 			}
-
-			matrix_set_node_status(matrix_rec_list,
-								   nodes.node_count,
-								   connection_node_id,
-								   x,
-								   (y == -1) ? -1 : 0);
-
-			while (*p && (*p != '\n'))
-				p++;
-			if (*p == '\n')
-				p++;
 		}
 
 		termPQExpBuffer(&command_output);
 		PQfinish(node_conn);
 		free_conninfo_params(&remote_conninfo);
-
-		node_conn = NULL;
 	}
 
 	*matrix_rec_dest = matrix_rec_list;
@@ -1097,7 +1120,7 @@ build_cluster_matrix(t_node_matrix_rec ***matrix_rec_dest, int *name_length)
 
 
 static int
-build_cluster_crosscheck(t_node_status_cube ***dest_cube, int *name_length)
+build_cluster_crosscheck(t_node_status_cube ***dest_cube, int *name_length, ItemList *warnings, int *error_code)
 {
 	PGconn	   *conn = NULL;
 	int			h,
@@ -1118,7 +1141,12 @@ build_cluster_crosscheck(t_node_status_cube ***dest_cube, int *name_length)
 	else
 		conn = establish_db_connection_by_params(&source_conninfo, true);
 
-	get_all_node_records(conn, &nodes);
+	if (get_all_node_records(conn, &nodes) == false)
+	{
+		/* get_all_node_records() will display the error */
+		PQfinish(conn);
+		exit(ERR_BAD_CONFIG);
+	}
 
 	PQfinish(conn);
 	conn = NULL;
@@ -1205,28 +1233,13 @@ build_cluster_crosscheck(t_node_status_cube ***dest_cube, int *name_length)
 
 		initPQExpBuffer(&command);
 
-		appendPQExpBuffer(&command,
+		make_remote_repmgr_path(&command, cell->node_info);
-						  "%s -d '%s' --node-id=%i ",
-						  make_pg_path(progname()),
-						  cell->node_info->conninfo,
-						  remote_node_id);
 
-		if (strlen(pg_bindir))
+		appendPQExpBufferStr(&command,
-		{
+							 " cluster matrix --csv -L NOTICE --terse");
-			appendPQExpBuffer(&command,
-							  "--pg_bindir=");
-			appendShellString(&command,
-							  pg_bindir);
-			appendPQExpBuffer(&command,
-							  " ");
-		}
-
-		appendPQExpBuffer(&command,
-						  "cluster matrix --csv 2>/dev/null");
 
 		initPQExpBuffer(&command_output);
 
-		/* fix to work with --node-id */
 		if (cube[i]->node_id == config_file_options.node_id)
 		{
 			(void) local_command_simple(command.data,
@@ -1267,9 +1280,13 @@ build_cluster_crosscheck(t_node_status_cube ***dest_cube, int *name_length)
 
 		p = command_output.data;
 
-		if (!strlen(command_output.data))
+		if (p[0] == '\0' || p[0] == '\n')
 		{
+			item_list_append_format(warnings,
+									"node %i inaccessible via SSH",
+									remote_node_id);
 			termPQExpBuffer(&command_output);
+			*error_code = ERR_BAD_SSH;
 			continue;
 		}
 
@@ -1281,16 +1298,23 @@ build_cluster_crosscheck(t_node_status_cube ***dest_cube, int *name_length)
 
 			if (sscanf(p, "%d,%d,%d", &matrix_rec_node_id, &node_status_node_id, &node_status) != 3)
 			{
-				fprintf(stderr, _("cannot parse --csv output: %s\n"), p);
+				cube_set_node_status(cube,
-				exit(ERR_INTERNAL);
+									 nodes.node_count,
+									 remote_node_id,
+									 matrix_rec_node_id,
+									 node_status_node_id,
+									 -2);
+				*error_code = ERR_INTERNAL;
+			}
+			else
+			{
+				cube_set_node_status(cube,
+									 nodes.node_count,
+									 remote_node_id,
+									 matrix_rec_node_id,
+									 node_status_node_id,
+									 node_status);
 			}
-
-			cube_set_node_status(cube,
-								 nodes.node_count,
-								 remote_node_id,
-								 matrix_rec_node_id,
-								 node_status_node_id,
-								 node_status);
 
 			while (*p && (*p != '\n'))
 				p++;
@@ -1362,7 +1386,9 @@ do_cluster_cleanup(void)
 
 	log_debug(_("number of days of monitoring history to retain: %i"), runtime_options.keep_history);
 
-	entries_to_delete = get_number_of_monitoring_records_to_delete(primary_conn, runtime_options.keep_history);
+	entries_to_delete = get_number_of_monitoring_records_to_delete(primary_conn,
+																   runtime_options.keep_history,
+																   runtime_options.node_id);
 
 	if (entries_to_delete < 0)
 	{
@@ -1382,9 +1408,9 @@ do_cluster_cleanup(void)
 
 	initPQExpBuffer(&event_details);
 
-	if (delete_monitoring_records(primary_conn, runtime_options.keep_history) == false)
+	if (delete_monitoring_records(primary_conn, runtime_options.keep_history, runtime_options.node_id) == false)
 	{
-		appendPQExpBuffer(&event_details,
+		appendPQExpBufferStr(&event_details,
 						  _("unable to delete monitoring records"));
 
 		log_error("%s", event_details.data);
@@ -1408,8 +1434,21 @@ do_cluster_cleanup(void)
 		log_detail("%s", PQerrorMessage(primary_conn));
 	}
 
-	appendPQExpBuffer(&event_details,
+	if (runtime_options.keep_history == 0)
-					  _("monitoring records deleted"));
+	{
+		appendPQExpBufferStr(&event_details,
+						  _("all monitoring records deleted"));
+	}
+	else
+	{
+		appendPQExpBufferStr(&event_details,
+						  _("monitoring records deleted"));
+	}
+
+	if (runtime_options.node_id != UNKNOWN_NODE_ID)
+		appendPQExpBuffer(&event_details,
+						  _(" for node %i"),
+						  runtime_options.node_id);
 
 	if (runtime_options.keep_history > 0)
 		appendPQExpBuffer(&event_details,
@@ -1423,18 +1462,11 @@ do_cluster_cleanup(void)
 							  true,
 							  event_details.data);
 
+	log_notice("%s", event_details.data);
+
 	termPQExpBuffer(&event_details);
 	PQfinish(primary_conn);
 
-	if (runtime_options.keep_history > 0)
-	{
-		log_notice(_("monitoring records older than %i day(s) deleted"),
-				   runtime_options.keep_history);
-	}
-	else
-	{
-		log_info(_("all monitoring records deleted"));
-	}
 
 	return;
 }

repmgr-action-daemon.c

@@ -0,0 +1,420 @@
+/*
+ * repmgr-action-daemon.c
+ *
+ * Implements repmgrd actions for the repmgr command line utility
+ * Copyright (c) 2ndQuadrant, 2010-2018
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "repmgr.h"
+
+#include "repmgr-client-global.h"
+#include "repmgr-action-daemon.h"
+
+
+
+/*
+ * Possibly also show:
+ *  - repmgrd start time?
+ *  - repmgrd mode
+ *  - priority
+ *  - whether promotion candidate (due to zero priority/different location)
+ */
+
+typedef enum
+{
+	STATUS_ID = 0,
+	STATUS_NAME,
+	STATUS_ROLE,
+	STATUS_PG,
+	STATUS_RUNNING,
+	STATUS_PID,
+	STATUS_PAUSED
+} StatusHeader;
+
+#define STATUS_HEADER_COUNT 7
+
+struct ColHeader headers_status[STATUS_HEADER_COUNT];
+
+static void fetch_node_records(PGconn *conn, NodeInfoList *node_list);
+static void _do_repmgr_pause(bool pause);
+
+
+void
+do_daemon_status(void)
+{
+	PGconn	   *conn = NULL;
+	NodeInfoList nodes = T_NODE_INFO_LIST_INITIALIZER;
+	NodeInfoListCell *cell = NULL;
+	int i;
+	RepmgrdInfo **repmgrd_info;
+	ItemList	warnings = {NULL, NULL};
+
+	/* Connect to local database to obtain cluster connection data */
+	log_verbose(LOG_INFO, _("connecting to database"));
+
+	if (strlen(config_file_options.conninfo))
+		conn = establish_db_connection(config_file_options.conninfo, true);
+	else
+		conn = establish_db_connection_by_params(&source_conninfo, true);
+
+	fetch_node_records(conn, &nodes);
+
+	repmgrd_info = (RepmgrdInfo **) pg_malloc0(sizeof(RepmgrdInfo *) * nodes.node_count);
+
+	if (repmgrd_info == NULL)
+	{
+		log_error(_("unable to allocate memory"));
+		exit(ERR_OUT_OF_MEMORY);
+	}
+
+	strncpy(headers_status[STATUS_ID].title, _("ID"), MAXLEN);
+	strncpy(headers_status[STATUS_NAME].title, _("Name"), MAXLEN);
+	strncpy(headers_status[STATUS_ROLE].title, _("Role"), MAXLEN);
+	strncpy(headers_status[STATUS_PG].title, _("Status"), MAXLEN);
+	strncpy(headers_status[STATUS_RUNNING].title, _("repmgrd"), MAXLEN);
+	strncpy(headers_status[STATUS_PID].title, _("PID"), MAXLEN);
+	strncpy(headers_status[STATUS_PAUSED].title, _("Paused?"), MAXLEN);
+
+	for (i = 0; i < STATUS_HEADER_COUNT; i++)
+	{
+		headers_status[i].max_length = strlen(headers_status[i].title);
+	}
+
+	i = 0;
+
+	for (cell = nodes.head; cell; cell = cell->next)
+	{
+		int j;
+
+		repmgrd_info[i] = pg_malloc0(sizeof(RepmgrdInfo));
+		repmgrd_info[i]->node_id = cell->node_info->node_id;
+		repmgrd_info[i]->pid = UNKNOWN_PID;
+		repmgrd_info[i]->paused = false;
+		repmgrd_info[i]->running = false;
+		repmgrd_info[i]->pg_running = true;
+
+		cell->node_info->conn = establish_db_connection_quiet(cell->node_info->conninfo);
+
+		if (PQstatus(cell->node_info->conn) != CONNECTION_OK)
+		{
+			if (runtime_options.verbose)
+			{
+				char		error[MAXLEN];
+
+				strncpy(error, PQerrorMessage(cell->node_info->conn), MAXLEN);
+
+				item_list_append_format(&warnings,
+										"when attempting to connect to node \"%s\" (ID: %i), following error encountered :\n\"%s\"",
+										cell->node_info->node_name, cell->node_info->node_id, trim(error));
+			}
+			else
+			{
+				item_list_append_format(&warnings,
+										"unable to  connect to node \"%s\" (ID: %i)",
+										cell->node_info->node_name, cell->node_info->node_id);
+			}
+
+			repmgrd_info[i]->pg_running = false;
+			maxlen_snprintf(repmgrd_info[i]->pg_running_text, "%s", _("not running"));
+			maxlen_snprintf(repmgrd_info[i]->repmgrd_running, "%s", _("n/a"));
+			maxlen_snprintf(repmgrd_info[i]->pid_text, "%s", _("n/a"));
+		}
+		else
+		{
+			maxlen_snprintf(repmgrd_info[i]->pg_running_text, "%s", _("running"));
+
+			repmgrd_info[i]->pid = repmgrd_get_pid(cell->node_info->conn);
+
+			repmgrd_info[i]->running = repmgrd_is_running(cell->node_info->conn);
+
+			if (repmgrd_info[i]->running == true)
+			{
+				maxlen_snprintf(repmgrd_info[i]->repmgrd_running, "%s", _("running"));
+			}
+			else
+			{
+				maxlen_snprintf(repmgrd_info[i]->repmgrd_running, "%s", _("not running"));
+			}
+
+			if (repmgrd_info[i]->pid == UNKNOWN_PID)
+			{
+				maxlen_snprintf(repmgrd_info[i]->pid_text, "%s", _("n/a"));
+			}
+			else
+			{
+				maxlen_snprintf(repmgrd_info[i]->pid_text, "%i", repmgrd_info[i]->pid);
+			}
+
+			repmgrd_info[i]->paused = repmgrd_is_paused(cell->node_info->conn);
+
+			PQfinish(cell->node_info->conn);
+		}
+
+
+		headers_status[STATUS_NAME].cur_length = strlen(cell->node_info->node_name);
+		headers_status[STATUS_ROLE].cur_length = strlen(get_node_type_string(cell->node_info->type));
+		headers_status[STATUS_PID].cur_length = strlen(repmgrd_info[i]->pid_text);
+		headers_status[STATUS_RUNNING].cur_length = strlen(repmgrd_info[i]->repmgrd_running);
+		headers_status[STATUS_PG].cur_length = strlen(repmgrd_info[i]->pg_running_text);
+
+		for (j = 0; j < STATUS_HEADER_COUNT; j++)
+		{
+			if (headers_status[j].cur_length > headers_status[j].max_length)
+			{
+				headers_status[j].max_length = headers_status[j].cur_length;
+			}
+		}
+
+		i++;
+	}
+
+	/* Print column header row (text mode only) */
+	if (runtime_options.output_mode == OM_TEXT)
+	{
+		print_status_header(STATUS_HEADER_COUNT, headers_status);
+	}
+
+	i = 0;
+
+	for (cell = nodes.head; cell; cell = cell->next)
+	{
+		if (runtime_options.output_mode == OM_CSV)
+		{
+			printf("%i,%s,%s,%i,%i,%i,%i\n",
+				   cell->node_info->node_id,
+				   cell->node_info->node_name,
+				   get_node_type_string(cell->node_info->type),
+				   repmgrd_info[i]->pg_running ? 1 : 0,
+				   repmgrd_info[i]->running ? 1 : 0,
+				   repmgrd_info[i]->pid,
+				   repmgrd_info[i]->paused ? 1 : 0);
+		}
+		else
+		{
+			printf(" %-*i ",  headers_status[STATUS_ID].max_length, cell->node_info->node_id);
+			printf("| %-*s ", headers_status[STATUS_NAME].max_length, cell->node_info->node_name);
+			printf("| %-*s ", headers_status[STATUS_ROLE].max_length, get_node_type_string(cell->node_info->type));
+
+			printf("| %-*s ", headers_status[STATUS_PG].max_length, repmgrd_info[i]->pg_running_text);
+			printf("| %-*s ", headers_status[STATUS_RUNNING].max_length, repmgrd_info[i]->repmgrd_running);
+			printf("| %-*s ", headers_status[STATUS_PID].max_length, repmgrd_info[i]->pid_text);
+
+			if (repmgrd_info[i]->pid == UNKNOWN_PID)
+				printf("| %-*s ", headers_status[STATUS_PAUSED].max_length, "n/a");
+			else
+				printf("| %-*s ", headers_status[STATUS_PAUSED].max_length, repmgrd_info[i]->paused ? "yes" : "no");
+
+			printf("\n");
+		}
+
+		free(repmgrd_info[i]);
+		i++;
+	}
+
+	free(repmgrd_info);
+
+	/* emit any warnings */
+
+	if (warnings.head != NULL && runtime_options.terse == false && runtime_options.output_mode != OM_CSV)
+	{
+		ItemListCell *cell = NULL;
+
+		printf(_("\nWARNING: following issues were detected\n"));
+		for (cell = warnings.head; cell; cell = cell->next)
+		{
+			printf(_("  - %s\n"), cell->string);
+		}
+
+		if (runtime_options.verbose == false)
+		{
+			log_hint(_("execute with --verbose option to see connection error messages"));
+		}
+	}
+}
+
+void
+do_daemon_pause(void)
+{
+	_do_repmgr_pause(true);
+}
+
+void
+do_daemon_unpause(void)
+{
+	_do_repmgr_pause(false);
+}
+
+
+static void
+_do_repmgr_pause(bool pause)
+{
+	PGconn	   *conn = NULL;
+	NodeInfoList nodes = T_NODE_INFO_LIST_INITIALIZER;
+	NodeInfoListCell *cell = NULL;
+	RepmgrdInfo **repmgrd_info;
+	int i;
+	int error_nodes = 0;
+
+	repmgrd_info = (RepmgrdInfo **) pg_malloc0(sizeof(RepmgrdInfo *) * nodes.node_count);
+
+	if (repmgrd_info == NULL)
+	{
+		log_error(_("unable to allocate memory"));
+		exit(ERR_OUT_OF_MEMORY);
+	}
+
+	/* Connect to local database to obtain cluster connection data */
+	log_verbose(LOG_INFO, _("connecting to database"));
+
+	if (strlen(config_file_options.conninfo))
+		conn = establish_db_connection(config_file_options.conninfo, true);
+	else
+		conn = establish_db_connection_by_params(&source_conninfo, true);
+
+	fetch_node_records(conn, &nodes);
+
+	i = 0;
+
+	for (cell = nodes.head; cell; cell = cell->next)
+	{
+		repmgrd_info[i] = pg_malloc0(sizeof(RepmgrdInfo));
+		repmgrd_info[i]->node_id = cell->node_info->node_id;
+
+		log_verbose(LOG_DEBUG, "pausing node %i (%s)",
+					cell->node_info->node_id,
+					cell->node_info->node_name);
+		cell->node_info->conn = establish_db_connection_quiet(cell->node_info->conninfo);
+
+		if (PQstatus(cell->node_info->conn) != CONNECTION_OK)
+		{
+			log_warning(_("unable to connect to node %i"),
+						cell->node_info->node_id);
+			error_nodes++;
+		}
+		else
+		{
+			if (runtime_options.dry_run == true)
+			{
+				if (pause == true)
+				{
+					log_info(_("would pause node %i (%s) "),
+							 cell->node_info->node_id,
+							 cell->node_info->node_name);
+				}
+				else
+				{
+					log_info(_("would unpause node %i (%s) "),
+							 cell->node_info->node_id,
+							 cell->node_info->node_name);
+				}
+			}
+			else
+			{
+				bool success = repmgrd_pause(cell->node_info->conn, pause);
+
+				if (success == false)
+					error_nodes++;
+
+				log_notice(_("node %i (%s) %s"),
+						   cell->node_info->node_id,
+						   cell->node_info->node_name,
+						   success == true
+								? pause == true ? "paused" : "unpaused"
+		   						: pause == true ? "not paused" : "not unpaused");
+			}
+			PQfinish(cell->node_info->conn);
+		}
+		i++;
+	}
+
+	if (error_nodes > 0)
+	{
+		if (pause == true)
+		{
+			log_error(_("unable to pause %i node(s)"), error_nodes);
+		}
+		else
+		{
+			log_error(_("unable to unpause %i node(s)"), error_nodes);
+		}
+
+		log_hint(_("execute \"repmgr daemon status\" to view current status"));
+
+		exit(ERR_REPMGRD_PAUSE);
+	}
+
+	exit(SUCCESS);
+}
+
+
+
+void
+fetch_node_records(PGconn *conn, NodeInfoList *node_list)
+{
+	bool success = get_all_node_records(conn, node_list);
+
+	if (success == false)
+	{
+		/* get_all_node_records() will display any error message */
+		PQfinish(conn);
+		exit(ERR_BAD_CONFIG);
+	}
+
+	if (node_list->node_count == 0)
+	{
+		log_error(_("no node records were found"));
+		log_hint(_("ensure at least one node is registered"));
+		PQfinish(conn);
+		exit(ERR_BAD_CONFIG);
+	}
+}
+
+
+void do_daemon_help(void)
+{
+	print_help_header();
+
+	printf(_("Usage:\n"));
+	printf(_("    %s [OPTIONS] daemon status\n"),  progname());
+	printf(_("    %s [OPTIONS] daemon pause\n"),   progname());
+	printf(_("    %s [OPTIONS] daemon unpause\n"), progname());
+	puts("");
+
+	printf(_("DAEMON STATUS\n"));
+	puts("");
+	printf(_("  \"daemon status\" shows the status of repmgrd on each node in the cluster\n"));
+	puts("");
+	printf(_("    --csv                     emit output as CSV\n"));
+	printf(_("    --verbose                 show text of database connection error messages\n"));
+	puts("");
+
+	printf(_("DAEMON PAUSE\n"));
+	puts("");
+	printf(_("  \"daemon pause\" instructs repmgrd on each node to pause failover detection\n"));
+	puts("");
+	printf(_("    --dry-run               check if nodes are reachable but don't pause repmgrd\n"));
+	puts("");
+
+	printf(_("DAEMON PAUSE\n"));
+	puts("");
+	printf(_("  \"daemon unpause\"  instructs repmgrd on each node to resume failover detection\n"));
+	puts("");
+	printf(_("    --dry-run               check if nodes are reachable but don't unpause repmgrd\n"));
+	puts("");
+
+
+	puts("");
+}

repmgr-action-daemon.h

@@ -0,0 +1,28 @@
+/*
+ * repmgr-action-daemon.h
+ * Copyright (c) 2ndQuadrant, 2010-2018
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _REPMGR_ACTION_DAEMON_H_
+#define _REPMGR_ACTION_DAEMON_H_
+
+
+extern void do_daemon_status(void);
+extern void do_daemon_pause(void);
+extern void do_daemon_unpause(void);
+
+extern void do_daemon_help(void);
+#endif

repmgr-action-node.c

@@ -189,16 +189,16 @@ do_node_status(void)
 
 		if (enabled == true)
 		{
-			appendPQExpBuffer(&archiving_status, "enabled");
+			appendPQExpBufferStr(&archiving_status, "enabled");
 		}
 		else
 		{
-			appendPQExpBuffer(&archiving_status, "disabled");
+			appendPQExpBufferStr(&archiving_status, "disabled");
 		}
 
 		if (enabled == false && recovery_type == RECTYPE_STANDBY)
 		{
-			appendPQExpBuffer(&archiving_status, " (on standbys \"archive_mode\" must be set to \"always\" to be effective)");
+			appendPQExpBufferStr(&archiving_status, " (on standbys \"archive_mode\" must be set to \"always\" to be effective)");
 		}
 
 		key_value_list_set(&node_status,
@@ -506,7 +506,7 @@ do_node_status(void)
 
 		/* output missing slot information */
 
-		appendPQExpBuffer(&output, "\n");
+		appendPQExpBufferChar(&output, '\n');
 		appendPQExpBuffer(&output,
 						  "\"missing_replication_slots\",%i",
 						  missing_slots.node_count);
@@ -590,13 +590,13 @@ _do_node_status_is_shutdown_cleanly(void)
 
 	initPQExpBuffer(&output);
 
-	appendPQExpBuffer(&output,
+	appendPQExpBufferStr(&output,
 					  "--state=");
 
 	/* sanity-check we're dealing with a PostgreSQL directory */
 	if (is_pg_dir(config_file_options.data_directory) == false)
 	{
-		appendPQExpBuffer(&output, "UNKNOWN");
+		appendPQExpBufferStr(&output, "UNKNOWN");
 		printf("%s\n", output.data);
 		termPQExpBuffer(&output);
 		return;
@@ -659,10 +659,10 @@ _do_node_status_is_shutdown_cleanly(void)
 	switch (node_status)
 	{
 		case NODE_STATUS_UP:
-			appendPQExpBuffer(&output, "RUNNING");
+			appendPQExpBufferStr(&output, "RUNNING");
 			break;
 		case NODE_STATUS_SHUTTING_DOWN:
-			appendPQExpBuffer(&output, "SHUTTING_DOWN");
+			appendPQExpBufferStr(&output, "SHUTTING_DOWN");
 			break;
 		case NODE_STATUS_DOWN:
 			appendPQExpBuffer(&output,
@@ -670,10 +670,10 @@ _do_node_status_is_shutdown_cleanly(void)
 							  format_lsn(checkPoint));
 			break;
 		case NODE_STATUS_UNCLEAN_SHUTDOWN:
-			appendPQExpBuffer(&output, "UNCLEAN_SHUTDOWN");
+			appendPQExpBufferStr(&output, "UNCLEAN_SHUTDOWN");
 			break;
 		case NODE_STATUS_UNKNOWN:
-			appendPQExpBuffer(&output, "UNKNOWN");
+			appendPQExpBufferStr(&output, "UNKNOWN");
 			break;
 	}
 
@@ -847,7 +847,7 @@ do_node_check(void)
 								  ",\"%s\"",
 								  cell->details);
 			}
-			appendPQExpBuffer(&output, "\n");
+			appendPQExpBufferChar(&output, '\n');
 		}
 	}
 	else
@@ -869,7 +869,7 @@ do_node_check(void)
 								  " (%s)",
 								  cell->details);
 			}
-			appendPQExpBuffer(&output, "\n");
+			appendPQExpBufferChar(&output, '\n');
 		}
 	}
 
@@ -899,12 +899,12 @@ do_node_check_replication_connection(void)
 
 
 	initPQExpBuffer(&output);
-	appendPQExpBuffer(&output,
+	appendPQExpBufferStr(&output,
-					  "--connection=");
+						 "--connection=");
 
 	if (runtime_options.remote_node_id == UNKNOWN_NODE_ID)
 	{
-		appendPQExpBuffer(&output, "UNKNOWN");
+		appendPQExpBufferStr(&output, "UNKNOWN");
 		printf("%s\n", output.data);
 		termPQExpBuffer(&output);
 		return;
@@ -918,7 +918,7 @@ do_node_check_replication_connection(void)
 
 	if (record_status != RECORD_FOUND)
 	{
-		appendPQExpBuffer(&output, "UNKNOWN");
+		appendPQExpBufferStr(&output, "UNKNOWN");
 		printf("%s\n", output.data);
 		termPQExpBuffer(&output);
 		return;
@@ -938,7 +938,7 @@ do_node_check_replication_connection(void)
 
 	if (PQstatus(repl_conn) != CONNECTION_OK)
 	{
-		appendPQExpBuffer(&output, "BAD");
+		appendPQExpBufferStr(&output, "BAD");
 		printf("%s\n", output.data);
 		termPQExpBuffer(&output);
 		return;
@@ -946,7 +946,7 @@ do_node_check_replication_connection(void)
 
 	PQfinish(repl_conn);
 
-	appendPQExpBuffer(&output, "OK");
+	appendPQExpBufferStr(&output, "OK");
 	printf("%s\n", output.data);
 	termPQExpBuffer(&output);
 
@@ -1042,9 +1042,8 @@ do_node_check_archive_ready(PGconn *conn, OutputMode mode, CheckStatusList *list
 				break;
 			case OM_NAGIOS:
 			case OM_TEXT:
-				appendPQExpBuffer(
+				appendPQExpBufferStr(&details,
-								  &details,
+									 "unable to check archive_status directory");
-								  "unable to check archive_status directory");
 				break;
 
 			default:
@@ -1172,8 +1171,8 @@ do_node_check_downstream(PGconn *conn, OutputMode mode, CheckStatusList *list_ou
 	if (missing_nodes_count == 0)
 	{
 		if (expected_nodes_count == 0)
-			appendPQExpBuffer(&details,
+			appendPQExpBufferStr(&details,
-							  "this node has no downstream nodes");
+								 "this node has no downstream nodes");
 		else
 			appendPQExpBuffer(&details,
 							  "%i of %i downstream nodes attached",
@@ -1194,20 +1193,18 @@ do_node_check_downstream(PGconn *conn, OutputMode mode, CheckStatusList *list_ou
 
 		if (mode != OM_NAGIOS)
 		{
-			appendPQExpBuffer(&details, "; missing: ");
+			appendPQExpBufferStr(&details, "; missing: ");
 
 			for (missing_cell = missing_nodes.head; missing_cell; missing_cell = missing_cell->next)
 			{
 				if (first == false)
-					appendPQExpBuffer(&details,
+					appendPQExpBufferStr(&details,
-									  ", ");
+										 ", ");
 				else
 					first = false;
 
 				if (first == false)
-					appendPQExpBuffer(
+					appendPQExpBufferStr(&details, missing_cell->string);
-									  &details,
-									  "%s", missing_cell->string);
 			}
 		}
 	}
@@ -1307,8 +1304,8 @@ do_node_check_replication_lag(PGconn *conn, OutputMode mode, t_node_info *node_i
 		switch (mode)
 		{
 			case OM_OPTFORMAT:
-				appendPQExpBuffer(&details,
+				appendPQExpBufferStr(&details,
-								  "--lag=0");
+									 "--lag=0");
 				break;
 			case OM_NAGIOS:
 				appendPQExpBuffer(&details,
@@ -1319,13 +1316,13 @@ do_node_check_replication_lag(PGconn *conn, OutputMode mode, t_node_info *node_i
 			case OM_TEXT:
 				if (node_info->type == WITNESS)
 				{
-					appendPQExpBuffer(&details,
+					appendPQExpBufferStr(&details,
-									  "N/A - node is witness");
+										 "N/A - node is witness");
 				}
 				else
 				{
-					appendPQExpBuffer(&details,
+					appendPQExpBufferStr(&details,
-									  "N/A - node is primary");
+										 "N/A - node is primary");
 				}
 				break;
 			default:
@@ -1406,9 +1403,8 @@ do_node_check_replication_lag(PGconn *conn, OutputMode mode, t_node_info *node_i
 					break;
 				case OM_NAGIOS:
 				case OM_TEXT:
-					appendPQExpBuffer(
+					appendPQExpBufferStr(&details,
-									  &details,
+										 "unable to query replication lag");
-									  "unable to query replication lag");
 					break;
 
 				default:
@@ -1508,39 +1504,39 @@ do_node_check_role(PGconn *conn, OutputMode mode, t_node_info *node_info, CheckS
 			if (recovery_type == RECTYPE_STANDBY)
 			{
 				status = CHECK_STATUS_CRITICAL;
-				appendPQExpBuffer(&details,
+				appendPQExpBufferStr(&details,
-								  _("node is registered as primary but running as standby"));
+									 _("node is registered as primary but running as standby"));
 			}
 			else
 			{
-				appendPQExpBuffer(&details,
+				appendPQExpBufferStr(&details,
-								  _("node is primary"));
+									 _("node is primary"));
 			}
 			break;
 		case STANDBY:
 			if (recovery_type == RECTYPE_PRIMARY)
 			{
 				status = CHECK_STATUS_CRITICAL;
-				appendPQExpBuffer(&details,
+				appendPQExpBufferStr(&details,
-								  _("node is registered as standby but running as primary"));
+									 _("node is registered as standby but running as primary"));
 			}
 			else
 			{
-				appendPQExpBuffer(&details,
+				appendPQExpBufferStr(&details,
-								  _("node is standby"));
+									 _("node is standby"));
 			}
 			break;
 		case WITNESS:
 			if (recovery_type == RECTYPE_STANDBY)
 			{
 				status = CHECK_STATUS_CRITICAL;
-				appendPQExpBuffer(&details,
+				appendPQExpBufferStr(&details,
-								  _("node is registered as witness but running as standby"));
+									 _("node is registered as witness but running as standby"));
 			}
 			else
 			{
-				appendPQExpBuffer(&details,
+				appendPQExpBufferStr(&details,
-								  _("node is witness"));
+									 _("node is witness"));
 			}
 			break;
 		case BDR:
@@ -1551,8 +1547,8 @@ do_node_check_role(PGconn *conn, OutputMode mode, t_node_info *node_info, CheckS
 				if (is_bdr_db(conn, &output) == false)
 				{
 					status = CHECK_STATUS_CRITICAL;
-					appendPQExpBuffer(&details,
+					appendPQExpBufferStr(&details,
-									  "%s", output.data);
+										 output.data);
 				}
 				termPQExpBuffer(&output);
 
@@ -1561,13 +1557,13 @@ do_node_check_role(PGconn *conn, OutputMode mode, t_node_info *node_info, CheckS
 					if (is_active_bdr_node(conn, node_info->node_name) == false)
 					{
 						status = CHECK_STATUS_CRITICAL;
-						appendPQExpBuffer(&details,
+						appendPQExpBufferStr(&details,
-										  _("node is not an active BDR node"));
+											 _("node is not an active BDR node"));
 					}
 					else
 					{
-						appendPQExpBuffer(&details,
+						appendPQExpBufferStr(&details,
-										  _("node is an active BDR node"));
+											 _("node is an active BDR node"));
 					}
 				}
 			}
@@ -1624,13 +1620,13 @@ do_node_check_slots(PGconn *conn, OutputMode mode, t_node_info *node_info, Check
 
 	if (server_version_num < 90400)
 	{
-		appendPQExpBuffer(&details,
+		appendPQExpBufferStr(&details,
-						  _("replication slots not available for this PostgreSQL version"));
+							 _("replication slots not available for this PostgreSQL version"));
 	}
 	else if (node_info->total_replication_slots == 0)
 	{
-		appendPQExpBuffer(&details,
+		appendPQExpBufferStr(&details,
-						  _("node has no replication slots"));
+							 _("node has no replication slots"));
 	}
 	else if (node_info->inactive_replication_slots == 0)
 	{
@@ -1700,8 +1696,8 @@ do_node_check_missing_slots(PGconn *conn, OutputMode mode, t_node_info *node_inf
 
 	if (server_version_num < 90400)
 	{
-		appendPQExpBuffer(&details,
+		appendPQExpBufferStr(&details,
-						  _("replication slots not available for this PostgreSQL version"));
+							 _("replication slots not available for this PostgreSQL version"));
 	}
 	else
 	{
@@ -1711,9 +1707,8 @@ do_node_check_missing_slots(PGconn *conn, OutputMode mode, t_node_info *node_inf
 
 		if (missing_slots.node_count == 0)
 		{
-			appendPQExpBuffer(&details,
+			appendPQExpBufferStr(&details,
-						  _("node has no missing replication slots"));
+								 _("node has no missing replication slots"));
-
 		}
 		else
 		{
@@ -1728,7 +1723,7 @@ do_node_check_missing_slots(PGconn *conn, OutputMode mode, t_node_info *node_inf
 
 			if (missing_slots.node_count)
 			{
-				appendPQExpBuffer(&details, ": ");
+				appendPQExpBufferStr(&details, ": ");
 
 				for (missing_slot_cell = missing_slots.head; missing_slot_cell; missing_slot_cell = missing_slot_cell->next)
 				{
@@ -1738,10 +1733,10 @@ do_node_check_missing_slots(PGconn *conn, OutputMode mode, t_node_info *node_inf
 					}
 					else
 					{
-						appendPQExpBuffer(&details, ", ");
+						appendPQExpBufferStr(&details, ", ");
 					}
 
-					appendPQExpBuffer(&details, "%s", missing_slot_cell->node_info->slot_name);
+					appendPQExpBufferStr(&details, missing_slot_cell->node_info->slot_name);
 				}
 			}
 		}
@@ -1784,7 +1779,7 @@ do_node_check_missing_slots(PGconn *conn, OutputMode mode, t_node_info *node_inf
 			if (list_output != NULL)
 			{
 				check_status_list_set(list_output,
-									  "Replication slots",
+									  "Missing replication slots",
 									  status,
 									  details.data);
 			}
@@ -2137,8 +2132,8 @@ do_node_rejoin(void)
 			exit(ERR_BAD_CONFIG);
 		}
 
-		appendPQExpBuffer(&msg,
+		appendPQExpBufferStr(&msg,
-						  _("prerequisites for using pg_rewind are met"));
+							 _("prerequisites for using pg_rewind are met"));
 
 		if (runtime_options.dry_run == true)
 		{

repmgr-action-primary.c

@@ -170,8 +170,8 @@ do_primary_register(void)
 											&node_info);
 		if (record_created == true)
 		{
-			appendPQExpBuffer(&event_description,
+			appendPQExpBufferStr(&event_description,
-							  "existing primary record updated");
+								 "existing primary record updated");
 		}
 		else
 		{

repmgr-action-standby.c

@@ -731,16 +731,16 @@ do_standby_clone(void)
 					  runtime_options.host,
 					  runtime_options.port);
 
-	appendPQExpBuffer(&event_details,
+	appendPQExpBufferStr(&event_details,
-					  _("; backup method: "));
+						 _("; backup method: "));
 
 	switch (mode)
 	{
 		case pg_basebackup:
-			appendPQExpBuffer(&event_details, "pg_basebackup");
+			appendPQExpBufferStr(&event_details, "pg_basebackup");
 			break;
 		case barman:
-			appendPQExpBuffer(&event_details, "barman");
+			appendPQExpBufferStr(&event_details, "barman");
 			break;
 	}
 
@@ -1672,12 +1672,12 @@ do_standby_register(void)
 
 	if (record_created == false)
 	{
-		appendPQExpBuffer(&details,
+		appendPQExpBufferStr(&details,
-						  "standby registration failed");
+							 "standby registration failed");
 
 		if (runtime_options.force == true)
-			appendPQExpBuffer(&details,
+			appendPQExpBufferStr(&details,
-							  " (-F/--force option was used)");
+								 " (-F/--force option was used)");
 
 		create_event_notification_extended(
 			primary_conn,
@@ -1697,12 +1697,12 @@ do_standby_register(void)
 		exit(ERR_BAD_CONFIG);
 	}
 
-	appendPQExpBuffer(&details,
+	appendPQExpBufferStr(&details,
-					  "standby registration succeeded");
+						 "standby registration succeeded");
 
 	if (runtime_options.force == true)
-		appendPQExpBuffer(&details,
+		appendPQExpBufferStr(&details,
-						  " (-F/--force option was used)");
+							 " (-F/--force option was used)");
 
 
 	/* Log the event */
@@ -2771,8 +2771,8 @@ do_standby_follow_internal(PGconn *primary_conn, t_node_info *primary_node_recor
 								  primary_node_record->node_id,
 								  true) == false)
 	{
-		appendPQExpBuffer(output,
+		appendPQExpBufferStr(output,
-						  _("unable to update upstream node"));
+							 _("unable to update upstream node"));
 		return false;
 	}
 
@@ -2788,15 +2788,13 @@ do_standby_follow_internal(PGconn *primary_conn, t_node_info *primary_node_recor
 
 /*
  * Perform a switchover by:
+ *
  *  - stopping current primary node
  *  - promoting this standby node to primary
- *  - forcing previous primary node to follow this node
+ *  - forcing the previous primary node to follow this node
  *
- * Caveat:
+ * Where running and not already paused, repmgrd will be paused (and
- *  - repmgrd must not be running, otherwise it may
+ * subsequently unpaused), unless --repmgrd-no-pause provided.
- *    attempt a failover
- *    (TODO: find some way of notifying repmgrd of planned
- *     activity like this)
  *
  * TODO:
  *  - make connection test timeouts/intervals configurable (see below)
@@ -2854,6 +2852,11 @@ do_standby_switchover(void)
 
 	t_event_info event_info = T_EVENT_INFO_INITIALIZER;
 
+	/* used for handling repmgrd pause/unpause */
+	NodeInfoList all_nodes = T_NODE_INFO_LIST_INITIALIZER;
+	RepmgrdInfo **repmgrd_info = NULL;
+	int			repmgrd_running_count = 0;
+
 	/*
 	 * SANITY CHECKS
 	 *
@@ -2924,7 +2927,7 @@ do_standby_switchover(void)
 
 	if (record_status != RECORD_FOUND)
 	{
-		log_error(_("unable to retrieve node record for node %i"),
+		log_error(_("unable to retrieve node record for currentr primary (node %i)"),
 				  remote_node_id);
 
 		PQfinish(local_conn);
@@ -2934,11 +2937,11 @@ do_standby_switchover(void)
 	}
 
 	/*
-	 * Check that there's no exclusive backups running on the primary.
+	 * Check that there are no exclusive backups running on the primary.
 	 * We don't want to end up damaging the backup and also leaving the server in an
 	 * state where there's control data saying it's in backup mode but there's no
 	 * backup_label in PGDATA.
-	 * If the DBA wants to do the switchover anyway, he should first stop the
+	 * If the user wants to do the switchover anyway, they should first stop the
 	 * backup that's running.
 	 */
 	if (server_in_exclusive_backup_mode(remote_conn) != BACKUP_STATE_NO_BACKUP)
@@ -2980,6 +2983,7 @@ do_standby_switchover(void)
 	{
 		min_required_free_slots++;
 	}
+
 	/*
 	 * If --force-rewind specified, check pg_rewind can be used, and
 	 * pre-emptively fetch the list of configuration files which should be
@@ -3006,8 +3010,8 @@ do_standby_switchover(void)
 		termPQExpBuffer(&reason);
 
 		initPQExpBuffer(&msg);
-		appendPQExpBuffer(&msg,
+		appendPQExpBufferStr(&msg,
-						  _("prerequisites for using pg_rewind are met"));
+							 _("prerequisites for using pg_rewind are met"));
 
 		if (runtime_options.dry_run == true)
 		{
@@ -3065,7 +3069,7 @@ do_standby_switchover(void)
 	initPQExpBuffer(&remote_command_str);
 	make_remote_repmgr_path(&remote_command_str, &remote_node_record);
 
-	appendPQExpBuffer(&remote_command_str, "--version 2>/dev/null && echo \"1\" || echo \"0\"");
+	appendPQExpBufferStr(&remote_command_str, "--version 2>/dev/null && echo \"1\" || echo \"0\"");
 	initPQExpBuffer(&command_output);
 	command_success = remote_command(remote_host,
 									 runtime_options.remote_user,
@@ -3087,8 +3091,8 @@ do_standby_switchover(void)
 		termPQExpBuffer(&command_output);
 
 		initPQExpBuffer(&hint);
-		appendPQExpBuffer(&hint,
+		appendPQExpBufferStr(&hint,
-						  _("check \"pg_bindir\" is set to the correct path in \"repmgr.conf\"; current value: "));
+							 _("check \"pg_bindir\" is set to the correct path in \"repmgr.conf\"; current value: "));
 
 		if (strlen(config_file_options.pg_bindir))
 		{
@@ -3097,8 +3101,8 @@ do_standby_switchover(void)
 		}
 		else
 		{
-			appendPQExpBuffer(&hint,
+			appendPQExpBufferStr(&hint,
-							  "(not set)");
+								 "(not set)");
 		}
 
 
@@ -3345,8 +3349,8 @@ do_standby_switchover(void)
 
 			initPQExpBuffer(&remote_command_str);
 			make_remote_repmgr_path(&remote_command_str, &remote_node_record);
-			appendPQExpBuffer(&remote_command_str,
+			appendPQExpBufferStr(&remote_command_str,
-							  "node check --terse -LERROR --archive-ready --optformat");
+								 "node check --terse -LERROR --archive-ready --optformat");
 
 			initPQExpBuffer(&command_output);
 
@@ -3544,8 +3548,8 @@ do_standby_switchover(void)
 
 		log_debug("minimum of %i free slots (%i for siblings) required; %i available",
 				  min_required_free_slots,
-				  reachable_sibling_nodes_with_slot_count
+				  reachable_sibling_nodes_with_slot_count,
-				  , available_slots);
+				  available_slots);
 
 		if (available_slots < min_required_free_slots)
 		{
@@ -3575,6 +3579,156 @@ do_standby_switchover(void)
 		}
 	}
 
+	/*
+	 * Attempt to pause all repmgrd instances, unless user explicitly
+	 * specifies not to.
+	 */
+	if (runtime_options.repmgrd_no_pause == false)
+	{
+		NodeInfoListCell *cell = NULL;
+		ItemList repmgrd_connection_errors = {NULL, NULL};
+		int i = 0;
+		int unreachable_node_count = 0;
+
+		get_all_node_records(local_conn, &all_nodes);
+
+		repmgrd_info = (RepmgrdInfo **) pg_malloc0(sizeof(RepmgrdInfo *) * all_nodes.node_count);
+
+		for (cell = all_nodes.head; cell; cell = cell->next)
+		{
+			cell->node_info->conn = establish_db_connection_quiet(cell->node_info->conninfo);
+
+			repmgrd_info[i] = pg_malloc0(sizeof(RepmgrdInfo));
+			repmgrd_info[i]->node_id = cell->node_info->node_id;
+			repmgrd_info[i]->pid = UNKNOWN_PID;
+			repmgrd_info[i]->paused = false;
+			repmgrd_info[i]->running = false;
+
+			if (PQstatus(cell->node_info->conn) != CONNECTION_OK)
+			{
+				/*
+				 * unable to connect; treat this as an error
+				 */
+
+				repmgrd_info[i]->pg_running = false;
+
+				item_list_append_format(&repmgrd_connection_errors,
+										_("unable to connect to node \"%s\" (ID %i)"),
+										cell->node_info->node_name,
+										cell->node_info->node_id);
+
+				unreachable_node_count++;
+				continue;
+			}
+
+			repmgrd_info[i]->running = repmgrd_is_running(cell->node_info->conn);
+			repmgrd_info[i]->pid = repmgrd_get_pid(cell->node_info->conn);
+			repmgrd_info[i]->paused = repmgrd_is_paused(cell->node_info->conn);
+
+			if (repmgrd_info[i]->running == true)
+				repmgrd_running_count++;
+
+			i++;
+		}
+
+		if (unreachable_node_count > 0)
+		{
+			PQExpBufferData msg;
+			PQExpBufferData detail;
+			ItemListCell *cell;
+
+			initPQExpBuffer(&msg);
+			appendPQExpBuffer(&msg,
+							  _("unable to connect to %i node(s), unable to pause all repmgrd instances"),
+							  unreachable_node_count);
+
+			initPQExpBuffer(&detail);
+
+			for (cell = repmgrd_connection_errors.head; cell; cell = cell->next)
+			{
+				appendPQExpBuffer(&detail,
+								  "  %s\n",
+								  cell->string);
+			}
+
+
+			if (runtime_options.force == false)
+			{
+				log_error("%s", msg.data);
+			}
+			else
+			{
+				log_warning("%s", msg.data);
+			}
+
+			log_detail(_("following node(s) unreachable:\n%s"), detail.data);
+
+			termPQExpBuffer(&msg);
+			termPQExpBuffer(&detail);
+
+			/* tell user about footgun */
+			if (runtime_options.force == false)
+			{
+				log_hint(_("use -F/--force to continue anyway"));
+
+				clear_node_info_list(&sibling_nodes);
+				clear_node_info_list(&all_nodes);
+
+				exit(ERR_SWITCHOVER_FAIL);
+			}
+
+		}
+
+		if (repmgrd_running_count > 0)
+		{
+			i = 0;
+			for (cell = all_nodes.head; cell; cell = cell->next)
+			{
+				/*
+				 * Skip if node is already paused. Note we won't unpause these, to
+				 * leave the repmgrd instances in the cluster in the same state they
+				 * were before the switchover.
+				 */
+				if (repmgrd_info[i]->paused == true)
+				{
+					PQfinish(cell->node_info->conn);
+					cell->node_info->conn = NULL;
+					i++;
+					continue;
+				}
+
+				if (runtime_options.dry_run == true)
+				{
+					log_info(_("would pause repmgrd on node %s (ID %i)"),
+							 cell->node_info->node_name,
+							 cell->node_info->node_id);
+				}
+				else
+				{
+					/* XXX check result  */
+					log_debug("pausing repmgrd on node %s (ID %i)",
+							 cell->node_info->node_name,
+							 cell->node_info->node_id);
+
+					(void) repmgrd_pause(cell->node_info->conn, true);
+				}
+
+				PQfinish(cell->node_info->conn);
+				cell->node_info->conn = NULL;
+				i++;
+			}
+		}
+		else
+		{
+			/* close all connections - we'll reestablish later */
+			for (cell = all_nodes.head; cell; cell = cell->next)
+			{
+				PQfinish(cell->node_info->conn);
+				cell->node_info->conn = NULL;
+			}
+		}
+	}
+
 
 	/*
 	 * Sanity checks completed - prepare for the switchover
@@ -3615,8 +3769,8 @@ do_standby_switchover(void)
 
 	if (runtime_options.dry_run == true)
 	{
-		appendPQExpBuffer(&remote_command_str,
+		appendPQExpBufferStr(&remote_command_str,
-						  "node service --terse -LERROR --list-actions --action=stop");
+							 "node service --terse -LERROR --list-actions --action=stop");
 
 	}
 	else
@@ -3624,8 +3778,8 @@ do_standby_switchover(void)
 		log_notice(_("stopping current primary node \"%s\" (ID: %i)"),
 				   remote_node_record.node_name,
 				   remote_node_record.node_id);
-		appendPQExpBuffer(&remote_command_str,
+		appendPQExpBufferStr(&remote_command_str,
-						  "node service --action=stop --checkpoint");
+							 "node service --action=stop --checkpoint");
 	}
 
 	/* XXX handle failure */
@@ -3656,6 +3810,7 @@ do_standby_switchover(void)
 				 shutdown_command);
 
 		clear_node_info_list(&sibling_nodes);
+
 		key_value_list_free(&remote_config_files);
 
 		return;
@@ -3666,13 +3821,14 @@ do_standby_switchover(void)
 
 	/* loop for timeout waiting for current primary to stop */
 
-	for (i = 0; i < config_file_options.reconnect_attempts; i++)
+	for (i = 0; i < config_file_options.shutdown_check_timeout; i++)
 	{
 		/* Check whether primary is available */
 		PGPing		ping_res;
 
-		log_info(_("checking primary status; %i of %i attempts"),
+		log_info(_("checking for primary shutdown; %i of %i attempts (\"shutdown_check_timeout\")"),
-				 i + 1, config_file_options.reconnect_attempts);
+				 i + 1, config_file_options.shutdown_check_timeout);
+
 		ping_res = PQping(remote_conninfo);
 
 		log_debug("ping status is: %s", print_pqping_status(ping_res));
@@ -3691,8 +3847,8 @@ do_standby_switchover(void)
 
 			initPQExpBuffer(&remote_command_str);
 			make_remote_repmgr_path(&remote_command_str, &remote_node_record);
-			appendPQExpBuffer(&remote_command_str,
+			appendPQExpBufferStr(&remote_command_str,
-							  "node status --is-shutdown-cleanly");
+								 "node status --is-shutdown-cleanly");
 
 			initPQExpBuffer(&command_output);
 
@@ -3741,9 +3897,8 @@ do_standby_switchover(void)
 			termPQExpBuffer(&command_output);
 		}
 
-		log_debug("sleeping %i seconds (\"reconnect_interval\") until next check",
+		log_debug("sleeping 1 second until next check");
-				  config_file_options.reconnect_interval);
+		sleep(1);
-		sleep(config_file_options.reconnect_interval);
 	}
 
 	if (shutdown_success == false)
@@ -3793,7 +3948,7 @@ do_standby_switchover(void)
 
 
 	/*
-	 * if pg_rewind is requested, issue a checkpoint immediately after promoting
+	 * If pg_rewind is requested, issue a checkpoint immediately after promoting
 	 * the local node, as pg_rewind compares timelines on the basis of the value
 	 * in pg_control, which is written at the first checkpoint, which might not
 	 * occur immediately.
@@ -3805,7 +3960,7 @@ do_standby_switchover(void)
 	}
 
 	/*
-	 * Execute `repmgr node rejoin` to create recovery.conf and start the
+	 * Execute "repmgr node rejoin" to create recovery.conf and start the
 	 * remote server. Additionally execute "pg_rewind", if required and
 	 * requested.
 	 */
@@ -3819,12 +3974,13 @@ do_standby_switchover(void)
 		{
 			log_error(_("new primary diverges from former primary and --force-rewind not provided"));
 			log_hint(_("the former primary will need to be restored manually, or use \"repmgr node rejoin\""));
+
 			termPQExpBuffer(&node_rejoin_options);
 			PQfinish(local_conn);
 			exit(ERR_SWITCHOVER_FAIL);
 		}
 
-		appendPQExpBuffer(&node_rejoin_options,
+		appendPQExpBufferStr(&node_rejoin_options,
 						  " --force-rewind");
 
 		if (runtime_options.force_rewind_path[0] != '\0')
@@ -3833,20 +3989,20 @@ do_standby_switchover(void)
 							  "=%s",
 							  runtime_options.force_rewind_path);
 		}
-		appendPQExpBuffer(&node_rejoin_options,
+		appendPQExpBufferStr(&node_rejoin_options,
-						  " --config-files=");
+							 " --config-files=");
 
 		for (cell = remote_config_files.head; cell; cell = cell->next)
 		{
 			if (first_entry == false)
-				appendPQExpBuffer(&node_rejoin_options, ",");
+				appendPQExpBufferChar(&node_rejoin_options, ',');
 			else
 				first_entry = false;
 
-			appendPQExpBuffer(&node_rejoin_options, "%s", cell->key);
+			appendPQExpBufferStr(&node_rejoin_options, cell->key);
 		}
 
-		appendPQExpBuffer(&node_rejoin_options, " ");
+		appendPQExpBufferChar(&node_rejoin_options, ' ');
 	}
 
 	key_value_list_free(&remote_config_files);
@@ -3875,7 +4031,7 @@ do_standby_switchover(void)
 
 	if (command_success == false)
 	{
-		log_error(_("rejoin failed %i"), r);
+		log_error(_("rejoin failed with error code %i"), r);
 
 		create_event_notification_extended(local_conn,
 										   &config_file_options,
@@ -3940,14 +4096,30 @@ do_standby_switchover(void)
 
 			if (sibling_node_record.type == WITNESS)
 			{
+				/* TODO: create "repmgr witness resync" or similar */
 				appendPQExpBuffer(&remote_command_str,
 								  "witness register -d \\'%s\\' --force 2>/dev/null && echo \"1\" || echo \"0\"",
 								  local_node_record.conninfo);
+
+				/*
+				 * Notify the witness repmgrd about the new primary, as at this point it will be assuming
+				 * a failover situation is in place. It will detect the new primary at some point, this
+				 * just speeds up the process.
+				 *
+				 * In the unlikely event repmgrd is not running or not in use, this will have no effect.
+				 */
+				cell->node_info->conn = establish_db_connection_quiet(cell->node_info->conninfo);
+
+				if (PQstatus(cell->node_info->conn) == CONNECTION_OK)
+				{
+					notify_follow_primary(cell->node_info->conn, local_node_record.node_id);
+				}
+				PQfinish(cell->node_info->conn);
 			}
 			else
 			{
-				appendPQExpBuffer(&remote_command_str,
+				appendPQExpBufferStr(&remote_command_str,
-								  "standby follow 2>/dev/null && echo \"1\" || echo \"0\"");
+									 "standby follow 2>/dev/null && echo \"1\" || echo \"0\"");
 			}
 			get_conninfo_value(cell->node_info->conninfo, "host", host);
 			log_debug("executing:\n  %s", remote_command_str.data);
@@ -4000,8 +4172,8 @@ do_standby_switchover(void)
 	PQfinish(local_conn);
 
 	/*
-	 * Clean up remote node. It's possible that the standby is still starting up,
+	 * Clean up remote node (primary demoted to standby). It's possible that the node is
-	 * so poll for a while until we get a connection.
+	 * still starting up, so poll for a while until we get a connection.
 	 */
 
 	for (i = 0; i < config_file_options.standby_reconnect_timeout; i++)
@@ -4053,6 +4225,84 @@ do_standby_switchover(void)
 
 	PQfinish(remote_conn);
 
+	/*
+	 * Attempt to unpause all paused repmgrd instances, unless user explicitly
+	 * specifies not to.
+	 */
+	if (runtime_options.repmgrd_no_pause == false)
+	{
+		if (repmgrd_running_count > 0)
+		{
+			ItemList repmgrd_unpause_errors = {NULL, NULL};
+			NodeInfoListCell *cell = NULL;
+			int i = 0;
+			int error_node_count = 0;
+
+			for (cell = all_nodes.head; cell; cell = cell->next)
+			{
+
+				if (repmgrd_info[i]->paused == true)
+				{
+					log_debug("repmgrd on node %s (ID %i) paused before switchover, not unpausing",
+							  cell->node_info->node_name,
+							  cell->node_info->node_id);
+
+					i++;
+					continue;
+				}
+
+				log_debug("unpausing repmgrd on node %s (ID %i)",
+						  cell->node_info->node_name,
+						  cell->node_info->node_id);
+
+				cell->node_info->conn = establish_db_connection_quiet(cell->node_info->conninfo);
+
+				if (PQstatus(cell->node_info->conn) == CONNECTION_OK)
+				{
+					if (repmgrd_pause(cell->node_info->conn, false) == false)
+					{
+						item_list_append_format(&repmgrd_unpause_errors,
+												_("unable to unpause node \"%s\" (ID %i)"),
+												cell->node_info->node_name,
+												cell->node_info->node_id);
+						error_node_count++;
+					}
+				}
+				else
+				{
+					item_list_append_format(&repmgrd_unpause_errors,
+											_("unable to connect to node \"%s\" (ID %i)"),
+											cell->node_info->node_name,
+											cell->node_info->node_id);
+					error_node_count++;
+				}
+
+				i++;
+			}
+
+			if (error_node_count > 0)
+			{
+				PQExpBufferData detail;
+				ItemListCell *cell;
+
+				for (cell = repmgrd_unpause_errors.head; cell; cell = cell->next)
+				{
+					appendPQExpBuffer(&detail,
+									  "  %s\n",
+									  cell->string);
+				}
+
+				log_warning(_("unable to unpause repmgrd on %i node(s)"),
+							error_node_count);
+				log_detail(_("errors encountered for following node(s):\n%s"), detail.data);
+				log_hint(_("check node connection and status; unpause manually with \"repmgr daemon unpause\""));
+
+				termPQExpBuffer(&detail);
+			}
+		}
+
+		clear_node_info_list(&all_nodes);
+	}
 
 	if (switchover_success == true)
 	{
@@ -5025,7 +5275,7 @@ run_basebackup(t_node_info *node_record)
 
 	if (runtime_options.fast_checkpoint)
 	{
-		appendPQExpBuffer(&params, " -c fast");
+		appendPQExpBufferStr(&params, " -c fast");
 	}
 
 	if (config_file_options.tablespace_mapping.head != NULL)
@@ -5047,7 +5297,7 @@ run_basebackup(t_node_info *node_record)
 	 */
 	if (!strlen(backup_options.xlog_method))
 	{
-		appendPQExpBuffer(&params, " -X stream");
+		appendPQExpBufferStr(&params, " -X stream");
 	}
 
 	/*
@@ -6087,8 +6337,8 @@ create_recovery_file(t_node_info *node_record, t_conninfo_param_list *recovery_c
 	initPQExpBuffer(&recovery_file_buf);
 
 	/* standby_mode = 'on' */
-	appendPQExpBuffer(&recovery_file_buf,
+	appendPQExpBufferStr(&recovery_file_buf,
-					  "standby_mode = 'on'\n");
+						 "standby_mode = 'on'\n");
 
 	/* primary_conninfo = '...' */
 
@@ -6116,8 +6366,8 @@ create_recovery_file(t_node_info *node_record, t_conninfo_param_list *recovery_c
 	}
 
 	/* recovery_target_timeline = 'latest' */
-	appendPQExpBuffer(&recovery_file_buf,
+	appendPQExpBufferStr(&recovery_file_buf,
-					  "recovery_target_timeline = 'latest'\n");
+						 "recovery_target_timeline = 'latest'\n");
 
 
 	/* recovery_min_apply_delay = ... (optional) */
@@ -6250,12 +6500,12 @@ write_primary_conninfo(PQExpBufferData *dest, t_conninfo_param_list *param_list)
 	{
 		if (strlen(config_file_options.node_name))
 		{
-			appendPQExpBuffer(&conninfo_buf, " application_name=");
+			appendPQExpBufferStr(&conninfo_buf, " application_name=");
 			appendConnStrVal(&conninfo_buf, config_file_options.node_name);
 		}
 		else
 		{
-			appendPQExpBuffer(&conninfo_buf, " application_name=repmgr");
+			appendPQExpBufferStr(&conninfo_buf, " application_name=repmgr");
 		}
 	}
 
@@ -6268,7 +6518,7 @@ write_primary_conninfo(PQExpBufferData *dest, t_conninfo_param_list *param_list)
 
 			if (password != NULL)
 			{
-				appendPQExpBuffer(&conninfo_buf, " password=");
+				appendPQExpBufferStr(&conninfo_buf, " password=");
 				appendConnStrVal(&conninfo_buf, password);
 			}
 		}
@@ -6280,7 +6530,7 @@ write_primary_conninfo(PQExpBufferData *dest, t_conninfo_param_list *param_list)
 		/* check if the libpq we're using supports "passfile=" */
 		if (has_passfile() == true)
 		{
-			appendPQExpBuffer(&conninfo_buf, " passfile=");
+			appendPQExpBufferStr(&conninfo_buf, " passfile=");
 			appendConnStrVal(&conninfo_buf, config_file_options.passfile);
 		}
 	}
@@ -6602,6 +6852,7 @@ do_standby_help(void)
 	printf(_("                                        (9.3 and 9.4 - provide \"pg_rewind\" path)\n"));
 
 	printf(_("  -R, --remote-user=USERNAME          database server username for SSH operations (default: \"%s\")\n"), runtime_options.username);
+	printf(_("  --repmgrd-no-pause                  don't pause repmgrd\n"));
 	printf(_("  --siblings-follow                   have other standbys follow new primary\n"));
 
 	puts("");

repmgr-action-witness.c

@@ -218,7 +218,13 @@ do_witness_register(void)
 	 * if repmgr.nodes contains entries, delete if -F/--force provided,
 	 * otherwise exit with error
 	 */
-	get_all_node_records(witness_conn, &nodes);
+	if (get_all_node_records(witness_conn, &nodes) == false)
+	{
+		/* get_all_node_records() will display the error */
+		PQfinish(witness_conn);
+		PQfinish(primary_conn);
+		exit(ERR_BAD_CONFIG);
+	}
 
 	log_verbose(LOG_DEBUG, "%i node records found", nodes.node_count);
 

repmgr-client-global.h

@@ -97,6 +97,7 @@ typedef struct
 	bool		force_rewind_used;
 	char		force_rewind_path[MAXPGPATH];
 	bool		siblings_follow;
+	bool		repmgrd_no_pause;
 
 	/* "node status" options */
 	bool		is_shutdown_cleanly;
@@ -156,7 +157,7 @@ typedef struct
 		/* "standby register" options */ \
 		false, -1, DEFAULT_WAIT_START,   \
 		/* "standby switchover" options */ \
-		false, false, "", false,		   \
+		false, false, "", false, false,	\
 		/* "node status" options */ \
 		false, \
 		/* "node check" options */ \
@@ -193,6 +194,14 @@ typedef enum
 } t_server_action;
 
 
+typedef struct ColHeader
+{
+	char		title[MAXLEN];
+	int			max_length;
+	int			cur_length;
+} ColHeader;
+
+
 
 /* global configuration structures */
 extern t_runtime_options runtime_options;
@@ -228,7 +237,10 @@ extern void get_superuser_connection(PGconn **conn, PGconn **superuser_conn, PGc
 extern bool remote_command(const char *host, const char *user, const char *command, PQExpBufferData *outputbuf);
 
 extern void make_remote_repmgr_path(PQExpBufferData *outputbuf, t_node_info *remote_node_record);
+
+/* display functions */
 extern void print_help_header(void);
+extern void print_status_header(int cols, ColHeader *headers);
 
 /* server control functions */
 extern void get_server_action(t_server_action action, char *script, char *data_dir);

repmgr-client.c

@@ -30,10 +30,15 @@
  * NODE STATUS
  * NODE CHECK
  *
+ * DAEMON STATUS
+ * DAEMON PAUSE
+ * DAEMON UNPAUSE
+ *
  * For internal use:
  * NODE REJOIN
  * NODE SERVICE
  *
+ *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
  * the Free Software Foundation, either version 3 of the License, or
@@ -62,6 +67,7 @@
 #include "repmgr-action-bdr.h"
 #include "repmgr-action-node.h"
 #include "repmgr-action-cluster.h"
+#include "repmgr-action-daemon.h"
 
 #include <storage/fd.h>			/* for PG_TEMP_FILE_PREFIX */
 
@@ -438,6 +444,10 @@ main(int argc, char **argv)
 				runtime_options.siblings_follow = true;
 				break;
 
+			case OPT_REPMGRD_NO_PAUSE:
+				runtime_options.repmgrd_no_pause = true;
+				break;
+
 				/*----------------------
 				 * "node status" options
 				 *----------------------
@@ -900,6 +910,21 @@ main(int argc, char **argv)
 			else if (strcasecmp(repmgr_action, "CLEANUP") == 0)
 				action = CLUSTER_CLEANUP;
 		}
+		else if (strcasecmp(repmgr_command, "DAEMON") == 0)
+		{
+			if (help_option == true)
+			{
+				do_daemon_help();
+				exit(SUCCESS);
+			}
+
+			if (strcasecmp(repmgr_action, "STATUS") == 0)
+				action = DAEMON_STATUS;
+			else if (strcasecmp(repmgr_action, "PAUSE") == 0)
+				action = DAEMON_PAUSE;
+			else if (strcasecmp(repmgr_action, "UNPAUSE") == 0)
+				action = DAEMON_UNPAUSE;
+		}
 		else
 		{
 			valid_repmgr_command_found = false;
@@ -1298,6 +1323,17 @@ main(int argc, char **argv)
 			do_cluster_cleanup();
 			break;
 
+			/* DAEMON */
+		case DAEMON_STATUS:
+			do_daemon_status();
+			break;
+		case DAEMON_PAUSE:
+			do_daemon_pause();
+			break;
+		case DAEMON_UNPAUSE:
+			do_daemon_unpause();
+			break;
+
 		default:
 			/* An action will have been determined by this point  */
 			break;
@@ -1520,6 +1556,7 @@ check_cli_parameters(const int action)
 			case PRIMARY_UNREGISTER:
 			case STANDBY_UNREGISTER:
 			case WITNESS_UNREGISTER:
+			case CLUSTER_CLEANUP:
 			case CLUSTER_EVENT:
 			case CLUSTER_MATRIX:
 			case CLUSTER_CROSSCHECK:
@@ -1743,6 +1780,18 @@ check_cli_parameters(const int action)
 		}
 	}
 
+	if (runtime_options.repmgrd_no_pause == true)
+	{
+		switch (action)
+		{
+			case STANDBY_SWITCHOVER:
+				break;
+			default:
+				item_list_append_format(&cli_warnings,
+										_("--repmgrd-no-pause will be ignored when executing %s"),
+										action_name(action));
+		}
+	}
 
 	if (runtime_options.config_files[0] != '\0')
 	{
@@ -1771,6 +1820,8 @@ check_cli_parameters(const int action)
 			case WITNESS_UNREGISTER:
 			case NODE_REJOIN:
 			case NODE_SERVICE:
+			case DAEMON_PAUSE:
+			case DAEMON_UNPAUSE:
 				break;
 			default:
 				item_list_append_format(&cli_warnings,
@@ -1850,6 +1901,14 @@ action_name(const int action)
 			return "CLUSTER MATRIX";
 		case CLUSTER_CROSSCHECK:
 			return "CLUSTER CROSSCHECK";
+
+		case DAEMON_STATUS:
+			return "DAEMON STATUS";
+		case DAEMON_PAUSE:
+			return "DAEMON PAUSE";
+		case DAEMON_UNPAUSE:
+			return "DAEMON UNPAUSE";
+
 	}
 
 	return "UNKNOWN ACTION";
@@ -1877,6 +1936,42 @@ print_error_list(ItemList *error_list, int log_level)
 }
 
 
+void
+print_status_header(int cols, ColHeader *headers)
+{
+	int i;
+
+	for (i = 0; i < cols; i++)
+	{
+		if (i == 0)
+			printf(" ");
+		else
+			printf(" | ");
+
+		printf("%-*s",
+			   headers[i].max_length,
+			   headers[i].title);
+	}
+	printf("\n");
+	printf("-");
+
+	for (i = 0; i < cols; i++)
+	{
+		int			j;
+
+		for (j = 0; j < headers[i].max_length; j++)
+			printf("-");
+
+		if (i < (cols - 1))
+			printf("-+-");
+		else
+			printf("-");
+	}
+
+	printf("\n");
+}
+
+
 void
 print_help_header(void)
 {
@@ -1905,6 +2000,7 @@ do_help(void)
 	printf(_("    %s [OPTIONS] node    {status|check|rejoin|service}\n"), progname());
 	printf(_("    %s [OPTIONS] cluster {show|event|matrix|crosscheck|cleanup}\n"), progname());
 	printf(_("    %s [OPTIONS] witness {register|unregister}\n"), progname());
+	printf(_("    %s [OPTIONS] daemon  {status|pause|unpause}\n"), progname());
 
 	puts("");
 
@@ -2576,11 +2672,29 @@ remote_command(const char *host, const char *user, const char *command, PQExpBuf
 void
 make_remote_repmgr_path(PQExpBufferData *output_buf, t_node_info *remote_node_record)
 {
+	if (config_file_options.repmgr_bindir[0] != '\0')
+	{
+		int			len = strlen(config_file_options.repmgr_bindir);
+
+		appendPQExpBufferStr(output_buf,
+							 config_file_options.repmgr_bindir);
+
+		/* Add trailing slash */
+		if (config_file_options.repmgr_bindir[len - 1] != '/')
+		{
+			appendPQExpBufferChar(output_buf, '/');
+		}
+	}
+	else if (pg_bindir[0] != '\0')
+	{
+		appendPQExpBufferStr(output_buf,
+							 pg_bindir);
+	}
+
 	appendPQExpBuffer(output_buf,
 					  "%s -f %s ",
-					  make_pg_path(progname()),
+					  progname(),
 					  remote_node_record->config_file);
-
 }
 
 
@@ -3020,4 +3134,3 @@ drop_replication_slot_if_exists(PGconn *conn, int node_id, char *slot_name)
 		}
 	}
 }
-

repmgr-client.h

@@ -45,6 +45,9 @@
 #define CLUSTER_MATRIX		   19
 #define CLUSTER_CROSSCHECK	   20
 #define CLUSTER_EVENT		   21
+#define DAEMON_STATUS		   22
+#define DAEMON_PAUSE		   23
+#define DAEMON_UNPAUSE		   24
 
 /* command line options without short versions */
 #define OPT_HELP						   1001
@@ -88,6 +91,7 @@
 #define OPT_RECOVERY_CONF_ONLY             1039
 #define OPT_NO_WAIT                        1040
 #define OPT_MISSING_SLOTS                  1041
+#define OPT_REPMGRD_NO_PAUSE               1042
 
 /* deprecated since 3.3 */
 #define OPT_DATA_DIR						999
@@ -156,6 +160,7 @@ static struct option long_options[] =
  */
 	{"always-promote", no_argument, NULL, OPT_ALWAYS_PROMOTE},
 	{"siblings-follow", no_argument, NULL, OPT_SIBLINGS_FOLLOW},
+	{"repmgrd-no-pause", no_argument, NULL, OPT_REPMGRD_NO_PAUSE},
 
 /* "node status" options */
 	{"is-shutdown-cleanly", no_argument, NULL, OPT_IS_SHUTDOWN_CLEANLY},

repmgr.c

@@ -26,6 +26,7 @@
 #include "access/xlog.h"
 #include "miscadmin.h"
 #include "replication/walreceiver.h"
+#include "storage/fd.h"
 #include "storage/ipc.h"
 #include "storage/lwlock.h"
 #include "storage/procarray.h"
@@ -43,14 +44,21 @@
 #include "lib/stringinfo.h"
 #include "access/xact.h"
 #include "utils/snapmgr.h"
-#include "pgstat.h"
 
+#if (PG_VERSION_NUM >= 90400)
+#include "pgstat.h"
+#else
+#define PGSTAT_STAT_PERMANENT_DIRECTORY             "pg_stat"
+#endif
 
 #include "voting.h"
 
 #define UNKNOWN_NODE_ID		-1
+#define UNKNOWN_PID			-1
 
 #define TRANCHE_NAME "repmgrd"
+#define REPMGRD_STATE_FILE PGSTAT_STAT_PERMANENT_DIRECTORY "/repmgrd_state.txt"
+
 
 PG_MODULE_MAGIC;
 
@@ -66,6 +74,9 @@ typedef struct repmgrdSharedState
 	LWLockId	lock;			/* protects search/modification */
 	TimestampTz last_updated;
 	int			local_node_id;
+	int			repmgrd_pid;
+	char		repmgrd_pidfile[MAXPGPATH];
+	bool		repmgrd_paused;
 	/* streaming failover */
 	NodeVotingStatus voting_status;
 	int			current_electoral_term;
@@ -112,6 +123,25 @@ PG_FUNCTION_INFO_V1(am_bdr_failover_handler);
 Datum		unset_bdr_failover_handler(PG_FUNCTION_ARGS);
 PG_FUNCTION_INFO_V1(unset_bdr_failover_handler);
 
+Datum		set_repmgrd_pid(PG_FUNCTION_ARGS);
+PG_FUNCTION_INFO_V1(set_repmgrd_pid);
+
+Datum		get_repmgrd_pid(PG_FUNCTION_ARGS);
+PG_FUNCTION_INFO_V1(get_repmgrd_pid);
+
+Datum		get_repmgrd_pidfile(PG_FUNCTION_ARGS);
+PG_FUNCTION_INFO_V1(get_repmgrd_pidfile);
+
+Datum		repmgrd_is_running(PG_FUNCTION_ARGS);
+PG_FUNCTION_INFO_V1(repmgrd_is_running);
+
+Datum		repmgrd_pause(PG_FUNCTION_ARGS);
+PG_FUNCTION_INFO_V1(repmgrd_pause);
+
+Datum		repmgrd_is_paused(PG_FUNCTION_ARGS);
+PG_FUNCTION_INFO_V1(repmgrd_is_paused);
+
+
 
 /*
  * Module load callback
@@ -185,6 +215,9 @@ repmgr_shmem_startup(void)
 #endif
 
 		shared_state->local_node_id = UNKNOWN_NODE_ID;
+		shared_state->repmgrd_pid = UNKNOWN_PID;
+		memset(shared_state->repmgrd_pidfile, 0, MAXPGPATH);
+		shared_state->repmgrd_paused = false;
 		shared_state->current_electoral_term = 0;
 		shared_state->voting_status = VS_NO_VOTE;
 		shared_state->candidate_node_id = UNKNOWN_NODE_ID;
@@ -204,6 +237,8 @@ Datum
 set_local_node_id(PG_FUNCTION_ARGS)
 {
 	int			local_node_id = UNKNOWN_NODE_ID;
+	int			stored_node_id = UNKNOWN_NODE_ID;
+	int			paused = -1;
 
 	if (!shared_state)
 		PG_RETURN_NULL();
@@ -213,6 +248,34 @@ set_local_node_id(PG_FUNCTION_ARGS)
 
 	local_node_id = PG_GETARG_INT32(0);
 
+	/* read state file and if exists/valid, update "repmgrd_paused" */
+	{
+		FILE	   *file = NULL;
+
+		file = AllocateFile(REPMGRD_STATE_FILE, PG_BINARY_R);
+
+		if (file != NULL)
+		{
+			int			buffer_size = 128;
+			char		buffer[buffer_size];
+
+			if (fgets(buffer, buffer_size, file) != NULL)
+			{
+				if (sscanf(buffer, "%i:%i", &stored_node_id, &paused) != 2)
+				{
+					elog(WARNING, "unable to parse repmgrd state file");
+				}
+				else
+				{
+					elog(DEBUG1, "node_id: %i; paused: %i", stored_node_id, paused);
+				}
+			}
+
+			FreeFile(file);
+		}
+
+	}
+
 	LWLockAcquire(shared_state->lock, LW_EXCLUSIVE);
 
 	/* only set local_node_id once, as it should never change */
@@ -221,6 +284,19 @@ set_local_node_id(PG_FUNCTION_ARGS)
 		shared_state->local_node_id = local_node_id;
 	}
 
+	/* only update if state file valid */
+	if (stored_node_id == shared_state->local_node_id)
+	{
+		if (paused == 0)
+		{
+			shared_state->repmgrd_paused = false;
+		}
+		else if (paused == 1)
+		{
+			shared_state->repmgrd_paused = true;
+		}
+	}
+
 	LWLockRelease(shared_state->lock);
 
 	PG_RETURN_VOID();
@@ -422,3 +498,185 @@ unset_bdr_failover_handler(PG_FUNCTION_ARGS)
 
 	PG_RETURN_VOID();
 }
+
+
+/*
+ * Returns the repmgrd pid; or NULL if none set; or -1 if set but repmgrd
+ * process not running (TODO!)
+ */
+Datum
+get_repmgrd_pid(PG_FUNCTION_ARGS)
+{
+	int repmgrd_pid = UNKNOWN_PID;
+
+	if (!shared_state)
+		PG_RETURN_NULL();
+
+	LWLockAcquire(shared_state->lock, LW_SHARED);
+	repmgrd_pid = shared_state->repmgrd_pid;
+	LWLockRelease(shared_state->lock);
+
+	PG_RETURN_INT32(repmgrd_pid);
+}
+
+
+/*
+ * Returns the repmgrd pidfile
+ */
+Datum
+get_repmgrd_pidfile(PG_FUNCTION_ARGS)
+{
+	char repmgrd_pidfile[MAXPGPATH];
+
+	if (!shared_state)
+		PG_RETURN_NULL();
+
+	memset(repmgrd_pidfile, 0, MAXPGPATH);
+
+	LWLockAcquire(shared_state->lock, LW_SHARED);
+	strncpy(repmgrd_pidfile, shared_state->repmgrd_pidfile, MAXPGPATH);
+	LWLockRelease(shared_state->lock);
+
+	if (repmgrd_pidfile[0] == '\0')
+		PG_RETURN_NULL();
+
+	PG_RETURN_TEXT_P(cstring_to_text(repmgrd_pidfile));
+}
+
+Datum
+set_repmgrd_pid(PG_FUNCTION_ARGS)
+{
+	int repmgrd_pid = UNKNOWN_PID;
+	char *repmgrd_pidfile = NULL;
+
+	if (!shared_state)
+		PG_RETURN_VOID();
+
+	if (PG_ARGISNULL(0))
+	{
+		repmgrd_pid = UNKNOWN_PID;
+	}
+	else
+	{
+		repmgrd_pid = PG_GETARG_INT32(0);
+	}
+
+	elog(DEBUG3, "set_repmgrd_pid(): provided pid is %i", repmgrd_pid);
+
+	if (repmgrd_pid != UNKNOWN_PID && !PG_ARGISNULL(1))
+	{
+		repmgrd_pidfile = text_to_cstring(PG_GETARG_TEXT_PP(1));
+		elog(INFO, "set_repmgrd_pid(): provided pidfile is %s", repmgrd_pidfile);
+	}
+
+	LWLockAcquire(shared_state->lock, LW_EXCLUSIVE);
+
+	shared_state->repmgrd_pid = repmgrd_pid;
+	memset(shared_state->repmgrd_pidfile, 0, MAXPGPATH);
+
+	if(repmgrd_pidfile != NULL)
+	{
+		strncpy(shared_state->repmgrd_pidfile, repmgrd_pidfile, MAXPGPATH);
+	}
+
+	LWLockRelease(shared_state->lock);
+	PG_RETURN_VOID();
+}
+
+
+Datum
+repmgrd_is_running(PG_FUNCTION_ARGS)
+{
+	int repmgrd_pid = UNKNOWN_PID;
+	int kill_ret;
+
+	if (!shared_state)
+		PG_RETURN_NULL();
+
+	LWLockAcquire(shared_state->lock, LW_SHARED);
+	repmgrd_pid = shared_state->repmgrd_pid;
+	LWLockRelease(shared_state->lock);
+
+	/* No PID registered - assume not running */
+	if (repmgrd_pid == UNKNOWN_PID)
+	{
+		PG_RETURN_BOOL(false);
+	}
+
+	kill_ret = kill(repmgrd_pid, 0);
+
+	if (kill_ret == 0)
+	{
+		PG_RETURN_BOOL(true);
+	}
+
+	PG_RETURN_BOOL(false);
+}
+
+
+Datum
+repmgrd_pause(PG_FUNCTION_ARGS)
+{
+	bool		pause;
+	FILE	   *file = NULL;
+	StringInfoData buf;
+
+	if (!shared_state)
+		PG_RETURN_NULL();
+
+	if (PG_ARGISNULL(0))
+		PG_RETURN_NULL();
+
+	pause = PG_GETARG_BOOL(0);
+
+	LWLockAcquire(shared_state->lock, LW_EXCLUSIVE);
+	shared_state->repmgrd_paused = pause;
+	LWLockRelease(shared_state->lock);
+
+	/* write state to file */
+	file = AllocateFile(REPMGRD_STATE_FILE, PG_BINARY_W);
+
+	if (file == NULL)
+	{
+		elog(DEBUG1, "unable to allocate %s", REPMGRD_STATE_FILE);
+
+		// XXX anything else we can do? log?
+		PG_RETURN_VOID();
+	}
+
+	elog(DEBUG1, "allocated");
+
+	initStringInfo(&buf);
+
+	LWLockAcquire(shared_state->lock, LW_SHARED);
+
+	appendStringInfo(&buf, "%i:%i",
+					 shared_state->local_node_id,
+					 pause ? 1 : 0);
+	LWLockRelease(shared_state->lock);
+
+	// XXX check success
+	fwrite(buf.data, strlen(buf.data) + 1, 1, file);
+
+
+	resetStringInfo(&buf);
+	FreeFile(file);
+
+	PG_RETURN_VOID();
+}
+
+
+Datum
+repmgrd_is_paused(PG_FUNCTION_ARGS)
+{
+	bool is_paused;
+
+	if (!shared_state)
+		PG_RETURN_NULL();
+
+	LWLockAcquire(shared_state->lock, LW_SHARED);
+	is_paused = shared_state->repmgrd_paused;
+	LWLockRelease(shared_state->lock);
+
+	PG_RETURN_BOOL(is_paused);
+}

repmgr.conf.sample

@@ -147,7 +147,11 @@
 					# *NOTE* "pg_bindir" is only used when repmgr directly
 					# executes PostgreSQL binaries; any user-defined scripts
 					# *must* be specified with the full path
-					#
+
+#repmgr_bindir=''			# Path to repmgr binary directory (location of the repmgr
+					# binary. Only needed if the repmgr executable is not in
+                                        # the system $PATH or the path defined in "pg_bindir".
+
 #use_primary_conninfo_password=false	# explicitly set "password" in recovery.conf's
 					# "primary_conninfo" parameter using the value contained
 					# in the environment variable PGPASSWORD
@@ -231,6 +235,8 @@ ssh_options='-q -o ConnectTimeout=10'	# Options to append to "ssh"
 # These settings apply when switching roles between a primary and a standby
 # ("repmgr standby switchover").
 
+#shutdown_check_timeout=60		# The max length of time (in seconds) to wait for the demotion
+					# candidate (current primary) to shut down
 #standby_reconnect_timeout=60		# The max length of time (in seconds) to wait
 					# for the demoted standby to reconnect to the promoted
 					# primary (note: this value should be equal to or greater

repmgr.control

@@ -1,6 +1,6 @@
 # repmgr extension
 comment = 'Replication manager for PostgreSQL'
-default_version = '4.1'
+default_version = '4.2'
 module_pathname = '$libdir/repmgr'
 relocatable = false
 schema = repmgr

repmgr.h

@@ -53,6 +53,7 @@
 
 #define UNKNOWN_TIMELINE_ID -1
 #define UNKNOWN_SYSTEM_IDENTIFIER 0
+#define UNKNOWN_PID			-1
 
 #define NODE_NOT_FOUND		-1
 #define NO_UPSTREAM_NODE	-1
@@ -84,6 +85,7 @@
 #define DEFAULT_WAIT_START                   30  /* seconds */
 #define DEFAULT_PROMOTE_CHECK_TIMEOUT        60  /* seconds */
 #define DEFAULT_PROMOTE_CHECK_INTERVAL       1   /* seconds */
+#define DEFAULT_SHUTDOWN_CHECK_TIMEOUT       60  /* seconds */
 #define DEFAULT_STANDBY_RECONNECT_TIMEOUT    60  /* seconds */
 #define DEFAULT_NODE_REJOIN_TIMEOUT          60  /* seconds */
 

repmgr_version.h.in

@@ -1,2 +1,2 @@
 #define REPMGR_VERSION_DATE ""
-#define REPMGR_VERSION "4.1.2"
+#define REPMGR_VERSION "4.2"

repmgrd-bdr.c

@@ -150,7 +150,13 @@ monitor_bdr(void)
 	 * retrieve list of all nodes - we'll need these if the DB connection goes
 	 * away
 	 */
-	get_all_node_records(local_conn, &nodes);
+	if (get_all_node_records(local_conn, &nodes) == false)
+	{
+		/* get_all_node_records() will display the error */
+		PQfinish(local_conn);
+		exit(ERR_BAD_CONFIG);
+	}
+
 
 	/* we're expecting all (both) nodes to be up */
 	for (cell = nodes.head; cell; cell = cell->next)

repmgrd-physical.c

@@ -71,6 +71,8 @@ static void notify_followers(NodeInfoList *standby_nodes, int follow_node_id);
 
 static void check_connection(t_node_info *node_info, PGconn **conn);
 
+static bool check_primary_status(int degraded_monitoring_elapsed);
+
 static bool wait_primary_notification(int *new_primary_id);
 static FailoverState follow_new_primary(int new_primary_id);
 static FailoverState witness_follow_new_primary(int new_primary_id);
@@ -106,12 +108,13 @@ handle_sigint_physical(SIGNAL_ARGS)
 	else
 		writeable_conn = primary_conn;
 
-	create_event_notification(writeable_conn,
+	if (PQstatus(writeable_conn) == CONNECTION_OK)
-							  &config_file_options,
+		create_event_notification(writeable_conn,
-							  config_file_options.node_id,
+								  &config_file_options,
-							  "repmgrd_shutdown",
+								  config_file_options.node_id,
-							  true,
+								  "repmgrd_shutdown",
-							  event_details.data);
+								  true,
+								  event_details.data);
 
 	termPQExpBuffer(&event_details);
 
@@ -145,7 +148,6 @@ do_physical_node_check(void)
 			case FAILOVER_AUTOMATIC:
 				log_error(_("this node is marked as inactive and cannot be used as a failover target"));
 				log_hint(_("%s"), hint);
-				close_connection(&local_conn);
 
 				create_event_notification(NULL,
 										  &config_file_options,
@@ -206,8 +208,7 @@ do_physical_node_check(void)
 		if (required_param_missing == true)
 		{
 			log_hint(_("add the missing configuration parameter(s) and start repmgrd again"));
-			close_connection(&local_conn);
+			terminate(ERR_BAD_CONFIG);
-			exit(ERR_BAD_CONFIG);
 		}
 	}
 }
@@ -286,8 +287,8 @@ monitor_streaming_primary(void)
 
 				initPQExpBuffer(&event_details);
 
-				appendPQExpBuffer(&event_details,
+				appendPQExpBufferStr(&event_details,
-								  _("unable to connect to local node"));
+									 _("unable to connect to local node"));
 
 				log_warning("%s", event_details.data);
 
@@ -339,8 +340,16 @@ monitor_streaming_primary(void)
 					if (stored_local_node_id == UNKNOWN_NODE_ID)
 					{
 						repmgrd_set_local_node_id(local_conn, config_file_options.node_id);
+						repmgrd_set_pid(local_conn, getpid(), pid_file);
 					}
 
+					/*
+					 * check that the local node is still primary, otherwise switch
+					 * to standby monitoring
+					 */
+					if (check_primary_status(-1) == false)
+						return;
+
 					goto loop;
 				}
 
@@ -393,143 +402,10 @@ monitor_streaming_primary(void)
 				{
 					local_node_info.node_status = NODE_STATUS_UP;
 
-					/* check to see if the node has been restored as a standby */
+					if (check_primary_status(degraded_monitoring_elapsed) == false)
-					if (get_recovery_type(local_conn) == RECTYPE_STANDBY)
+						return;
-					{
-						PGconn *new_primary_conn;
 
-						initPQExpBuffer(&event_details);
+					goto loop;
-
-						appendPQExpBuffer(&event_details,
-										  _("reconnected to node after %i seconds, node is now a standby, switching to standby monitoring"),
-										  degraded_monitoring_elapsed);
-						log_notice("%s", event_details.data);
-						termPQExpBuffer(&event_details);
-
-						primary_node_id = UNKNOWN_NODE_ID;
-
-						new_primary_conn = get_primary_connection_quiet(local_conn, &primary_node_id, NULL);
-
-						if (PQstatus(new_primary_conn) != CONNECTION_OK)
-						{
-							close_connection(&new_primary_conn);
-							log_warning(_("unable to connect to new primary node %i"), primary_node_id);
-						}
-						else
-						{
-							RecordStatus record_status;
-
-							log_debug("primary node id is now %i", primary_node_id);
-
-							record_status = get_node_record(new_primary_conn, config_file_options.node_id, &local_node_info);
-
-							if (record_status == RECORD_FOUND)
-							{
-								bool resume_monitoring = true;
-
-								log_debug("node %i is registered with type = %s",
-										  config_file_options.node_id,
-										  get_node_type_string(local_node_info.type));
-
-								/*
-								 * node has recovered but metadata not updated - we can do that ourselves,
-								 */
-								if (local_node_info.type == PRIMARY)
-								{
-									log_notice(_("node \"%s\" (ID: %i) still registered as primary, setting to standby"),
-											   config_file_options.node_name,
-											   config_file_options.node_id);
-
-									if (update_node_record_set_active_standby(new_primary_conn, config_file_options.node_id) == false)
-									{
-										resume_monitoring = false;
-									}
-									else
-									{
-										record_status = get_node_record(new_primary_conn, config_file_options.node_id, &local_node_info);
-
-										if (record_status != RECORD_FOUND)
-										{
-											resume_monitoring = false;
-										}
-									}
-								}
-
-								if (resume_monitoring == true)
-								{
-									monitoring_state = MS_NORMAL;
-									log_notice(_("former primary has been restored as standby after %i seconds, updating node record and resuming monitoring"),
-											   degraded_monitoring_elapsed);
-
-									initPQExpBuffer(&event_details);
-
-									appendPQExpBuffer(&event_details,
-													  _("node restored as standby after %i seconds, monitoring connection to upstream node %i"),
-													  degraded_monitoring_elapsed,
-													  local_node_info.upstream_node_id);
-
-									create_event_notification(new_primary_conn,
-															  &config_file_options,
-															  config_file_options.node_id,
-															  "repmgrd_standby_reconnect",
-															  true,
-															  event_details.data);
-
-
-									termPQExpBuffer(&event_details);
-
-									close_connection(&new_primary_conn);
-
-									/* restart monitoring as standby */
-									return;
-								}
-							}
-							else if (record_status == RECORD_NOT_FOUND)
-							{
-								PQExpBufferData event_details;
-								initPQExpBuffer(&event_details);
-
-								appendPQExpBuffer(&event_details,
-												  _("no metadata record found for this node on current primary %i"),
-												  primary_node_id);
-
-								log_error("%s", event_details.data);
-								log_hint(_("check that 'repmgr (primary|standby) register' was executed for this node"));
-
-								close_connection(&new_primary_conn);
-
-								create_event_notification(NULL,
-														  &config_file_options,
-														  config_file_options.node_id,
-														  "repmgrd_shutdown",
-														  false,
-														  event_details.data);
-								termPQExpBuffer(&event_details);
-
-								terminate(ERR_BAD_CONFIG);
-							}
-						}
-					}
-					else
-					{
-						monitoring_state = MS_NORMAL;
-
-						initPQExpBuffer(&event_details);
-						appendPQExpBuffer(&event_details,
-										  _("reconnected to primary node after %i seconds, resuming monitoring"),
-										  degraded_monitoring_elapsed);
-
-						create_event_notification(local_conn,
-												  &config_file_options,
-												  config_file_options.node_id,
-												  "repmgrd_local_reconnect",
-												  true,
-												  event_details.data);
-
-						log_notice("%s", event_details.data);
-						termPQExpBuffer(&event_details);
-						goto loop;
-					}
 				}
 			}
 
@@ -541,6 +417,11 @@ monitor_streaming_primary(void)
 			 */
 		}
 loop:
+
+		/* check node is still primary, if not restart monitoring */
+		if (check_primary_status(-1) == false)
+			return;
+
 		/* emit "still alive" log message at regular intervals, if requested */
 		if (config_file_options.log_status_interval > 0)
 		{
@@ -575,6 +456,181 @@ loop:
 }
 
 
+/*
+ * If monitoring a primary, it's possible that after an outage of the local node
+ * (due to e.g. a switchover), the node has come back as a standby. We therefore
+ * need to verify its status and if everything looks OK, restart monitoring in
+ * standby mode.
+ */
+bool
+check_primary_status(int degraded_monitoring_elapsed)
+{
+	PQExpBufferData event_details;
+
+	/* check to see if the node has been restored as a standby */
+	if (get_recovery_type(local_conn) == RECTYPE_STANDBY)
+	{
+		PGconn *new_primary_conn;
+
+		initPQExpBuffer(&event_details);
+
+		if (degraded_monitoring_elapsed > 0)
+		{
+			appendPQExpBuffer(&event_details,
+							  _("reconnected to node after %i seconds, node is now a standby, switching to standby monitoring"),
+							  degraded_monitoring_elapsed);
+		}
+		else
+		{
+			appendPQExpBufferStr(&event_details,
+								 _("node is now a standby, switching to standby monitoring"));
+		}
+
+		log_notice("%s", event_details.data);
+		termPQExpBuffer(&event_details);
+
+		primary_node_id = UNKNOWN_NODE_ID;
+
+		new_primary_conn = get_primary_connection_quiet(local_conn, &primary_node_id, NULL);
+
+		if (PQstatus(new_primary_conn) != CONNECTION_OK)
+		{
+			close_connection(&new_primary_conn);
+			log_warning(_("unable to connect to new primary node %i"), primary_node_id);
+		}
+		else
+		{
+			RecordStatus record_status;
+
+			log_debug("primary node id is now %i", primary_node_id);
+
+			record_status = get_node_record(new_primary_conn, config_file_options.node_id, &local_node_info);
+
+			if (record_status == RECORD_FOUND)
+			{
+				bool resume_monitoring = true;
+
+				log_debug("node %i is registered with type = %s",
+						  config_file_options.node_id,
+						  get_node_type_string(local_node_info.type));
+
+				/*
+				 * node has recovered but metadata not updated - we can do that ourselves,
+				 */
+				if (local_node_info.type == PRIMARY)
+				{
+					log_notice(_("node \"%s\" (ID: %i) still registered as primary, setting to standby"),
+											   config_file_options.node_name,
+											   config_file_options.node_id);
+
+					if (update_node_record_set_active_standby(new_primary_conn, config_file_options.node_id) == false)
+					{
+						resume_monitoring = false;
+					}
+					else
+					{
+						record_status = get_node_record(new_primary_conn, config_file_options.node_id, &local_node_info);
+
+						if (record_status != RECORD_FOUND)
+						{
+							resume_monitoring = false;
+						}
+					}
+				}
+
+				if (resume_monitoring == true)
+				{
+					initPQExpBuffer(&event_details);
+
+					if (degraded_monitoring_elapsed > 0)
+					{
+						monitoring_state = MS_NORMAL;
+
+						log_notice(_("former primary has been restored as standby after %i seconds, updating node record and resuming monitoring"),
+								   degraded_monitoring_elapsed);
+
+
+						appendPQExpBuffer(&event_details,
+										  _("node restored as standby after %i seconds, monitoring connection to upstream node %i"),
+										  degraded_monitoring_elapsed,
+										  local_node_info.upstream_node_id);
+					}
+					else
+					{
+						appendPQExpBuffer(&event_details,
+										  _("node has become a standby, monitoring connection to upstream node %i"),
+										  local_node_info.upstream_node_id);
+					}
+
+					create_event_notification(new_primary_conn,
+											  &config_file_options,
+											  config_file_options.node_id,
+											  "repmgrd_standby_reconnect",
+											  true,
+											  event_details.data);
+
+
+					termPQExpBuffer(&event_details);
+
+					close_connection(&new_primary_conn);
+
+					/* restart monitoring as standby */
+					return false;
+				}
+			}
+			else if (record_status == RECORD_NOT_FOUND)
+			{
+				initPQExpBuffer(&event_details);
+
+				appendPQExpBuffer(&event_details,
+								  _("no metadata record found for this node on current primary %i"),
+								  primary_node_id);
+
+				log_error("%s", event_details.data);
+				log_hint(_("check that 'repmgr (primary|standby) register' was executed for this node"));
+
+				close_connection(&new_primary_conn);
+
+				create_event_notification(NULL,
+										  &config_file_options,
+										  config_file_options.node_id,
+										  "repmgrd_shutdown",
+										  false,
+										  event_details.data);
+				termPQExpBuffer(&event_details);
+
+				terminate(ERR_BAD_CONFIG);
+			}
+		}
+	}
+	else
+	{
+		if (degraded_monitoring_elapsed > 0)
+		{
+			monitoring_state = MS_NORMAL;
+
+			initPQExpBuffer(&event_details);
+			appendPQExpBuffer(&event_details,
+							  _("reconnected to primary node after %i seconds, resuming monitoring"),
+							  degraded_monitoring_elapsed);
+
+			create_event_notification(local_conn,
+									  &config_file_options,
+									  config_file_options.node_id,
+									  "repmgrd_local_reconnect",
+									  true,
+									  event_details.data);
+
+			log_notice("%s", event_details.data);
+			termPQExpBuffer(&event_details);
+		}
+	}
+
+	return true;
+}
+
+
+
 void
 monitor_streaming_standby(void)
 {
@@ -595,7 +651,7 @@ monitor_streaming_standby(void)
 	 */
 	if (local_node_info.upstream_node_id == UNKNOWN_NODE_ID)
 	{
-		local_node_info.upstream_node_id = get_primary_node_id(local_conn);
+		upstream_conn = get_primary_connection(local_conn, &local_node_info.upstream_node_id, NULL);
 
 		/*
 		 * Terminate if there doesn't appear to be an active cluster primary.
@@ -606,37 +662,42 @@ monitor_streaming_standby(void)
 		if (local_node_info.upstream_node_id == NODE_NOT_FOUND)
 		{
 			log_error(_("unable to determine an active primary for this cluster, terminating"));
-			close_connection(&local_conn);
+			terminate(ERR_BAD_CONFIG);
-			exit(ERR_BAD_CONFIG);
 		}
+
+		(void) get_node_record(upstream_conn, local_node_info.upstream_node_id, &upstream_node_info);
+
 	}
-
+	else
-	record_status = get_node_record(local_conn, local_node_info.upstream_node_id, &upstream_node_info);
-
-	/*
-	 * Terminate if we can't find the record for the node we're supposed to
-	 * monitor. This is a "fix-the-config" situation, not a lot else we can
-	 * do.
-	 */
-	if (record_status == RECORD_NOT_FOUND)
 	{
-		log_error(_("no record found for upstream node (ID: %i), terminating"),
+		record_status = get_node_record(local_conn, local_node_info.upstream_node_id, &upstream_node_info);
-				  local_node_info.upstream_node_id);
+
-		log_hint(_("ensure the upstream node is registered correctly"));
+		/*
-		close_connection(&local_conn);
+		 * Terminate if we can't find the record for the node we're supposed to
-		exit(ERR_DB_CONN);
+		 * monitor. This is a "fix-the-config" situation, not a lot else we can
-	}
+		 * do.
-	else if (record_status == RECORD_ERROR)
+		 */
-	{
+		if (record_status == RECORD_NOT_FOUND)
-		log_error(_("unable to retrieve record for upstream node (ID: %i), terminating"),
+		{
-				  local_node_info.upstream_node_id);
+			log_error(_("no record found for upstream node (ID: %i), terminating"),
-		close_connection(&local_conn);
+					  local_node_info.upstream_node_id);
-		exit(ERR_DB_CONN);
+			log_hint(_("ensure the upstream node is registered correctly"));
+
+			terminate(ERR_DB_CONN);
+		}
+		else if (record_status == RECORD_ERROR)
+		{
+			log_error(_("unable to retrieve record for upstream node (ID: %i), terminating"),
+					  local_node_info.upstream_node_id);
+
+			terminate(ERR_DB_CONN);
+		}
+
+		log_debug("connecting to upstream node %i: \"%s\"", upstream_node_info.node_id, upstream_node_info.conninfo);
+
+		upstream_conn = establish_db_connection(upstream_node_info.conninfo, false);
 	}
 
-	log_debug("connecting to upstream node %i: \"%s\"", upstream_node_info.node_id, upstream_node_info.conninfo);
-
-	upstream_conn = establish_db_connection(upstream_node_info.conninfo, false);
 
 	/*
 	 * Upstream node must be running at repmgrd startup.
@@ -650,8 +711,16 @@ monitor_streaming_standby(void)
 				  local_node_info.upstream_node_id);
 		log_hint(_("upstream node must be running before repmgrd can start"));
 
-		close_connection(&local_conn);
+		terminate(ERR_DB_CONN);
-		exit(ERR_DB_CONN);
+	}
+
+	record_status = get_node_record(upstream_conn, local_node_info.node_id, &local_node_info);
+
+	if (upstream_node_info.node_id == local_node_info.node_id)
+	{
+		PQfinish(upstream_conn);
+		upstream_conn = NULL;
+		return;
 	}
 
 	/*
@@ -673,7 +742,8 @@ monitor_streaming_standby(void)
 		{
 			log_error(_("unable to connect to primary node"));
 			log_hint(_("ensure the primary node is reachable from this node"));
-			exit(ERR_DB_CONN);
+
+			terminate(ERR_DB_CONN);
 		}
 
 		log_verbose(LOG_DEBUG, "connected to primary");
@@ -683,6 +753,23 @@ monitor_streaming_standby(void)
 		primary_conn = upstream_conn;
 	}
 
+	/*
+	 * It's possible monitoring has been restarted after some outage which
+	 * resulted in the local node being marked as inactive; if so mark it
+	 * as active again.
+	 */
+	if (local_node_info.active == false)
+	{
+		if (update_node_record_set_active(primary_conn, local_node_info.node_id, true) == true)
+		{
+			PQExpBufferData event_details;
+
+			initPQExpBuffer(&event_details);
+
+			local_node_info.active = true;
+		}
+	}
+
 	primary_node_id = get_primary_node_id(primary_conn);
 
 	/* Log startup event */
@@ -767,6 +854,7 @@ monitor_streaming_standby(void)
 				if (PQstatus(local_conn) != CONNECTION_OK)
 				{
 					check_connection(&local_node_info, &local_conn);
+					log_debug("YYY here");
 				}
 
 				try_reconnect(&upstream_conn, &upstream_node_info);
@@ -779,6 +867,43 @@ monitor_streaming_standby(void)
 					if (upstream_node_info.type == PRIMARY)
 					{
 						primary_conn = upstream_conn;
+
+						if (get_recovery_type(primary_conn) == RECTYPE_STANDBY)
+						{
+							ExecStatusType ping_result;
+
+							log_notice(_("current upstream node \"%s\" (node ID: %i) is not primary, restarting monitoring"),
+									   upstream_node_info.node_name, upstream_node_info.node_id);
+							PQfinish(upstream_conn);
+							upstream_conn = NULL;
+							termPQExpBuffer(&event_details);
+							local_node_info.upstream_node_id = UNKNOWN_NODE_ID;
+
+							/* check local connection */
+							ping_result = connection_ping(local_conn);
+
+							if (ping_result != PGRES_TUPLES_OK)
+							{
+								int i;
+
+								PQfinish(local_conn);
+
+								for (i = 0; i < config_file_options.repmgrd_standby_startup_timeout; i++)
+								{
+									local_conn = establish_db_connection(local_node_info.conninfo, false);
+
+									if (PQstatus(local_conn) == CONNECTION_OK)
+										break;
+
+									log_debug("sleeping 1 second; %i of %i attempts to reconnect to local node",
+											  i + 1,
+											  config_file_options.repmgrd_standby_startup_timeout);
+									sleep(1);
+								}
+							}
+
+							return;
+						}
 					}
 
 					initPQExpBuffer(&event_details);
@@ -799,28 +924,40 @@ monitor_streaming_standby(void)
 					goto loop;
 				}
 
-				/* still down after reconnect attempt(s) */
+
+				/* upstream is still down after reconnect attempt(s) */
 				if (upstream_node_info.node_status == NODE_STATUS_DOWN)
 				{
 					bool		failover_done = false;
 
-					if (upstream_node_info.type == PRIMARY)
+					if (PQstatus(local_conn) == CONNECTION_OK && repmgrd_is_paused(local_conn))
 					{
-						failover_done = do_primary_failover();
+						log_notice(_("repmgrd on this node is paused"));
+						log_detail(_("no failover will be carried out"));
+						log_hint(_("execute \"repmgr daemon unpause\" to resume normal failover mode"));
+						monitoring_state = MS_DEGRADED;
+						INSTR_TIME_SET_CURRENT(degraded_monitoring_start);
 					}
-					else if (upstream_node_info.type == STANDBY)
+					else
 					{
-						failover_done = do_upstream_standby_failover();
+						if (upstream_node_info.type == PRIMARY)
-					}
+						{
+							failover_done = do_primary_failover();
+						}
+						else if (upstream_node_info.type == STANDBY)
+						{
+							failover_done = do_upstream_standby_failover();
+						}
 
-					/*
+						/*
-					 * XXX it's possible it will make sense to return in all
+						 * XXX it's possible it will make sense to return in all
-					 * cases to restart monitoring
+						 * cases to restart monitoring
-					 */
+						 */
-					if (failover_done == true)
+						if (failover_done == true)
-					{
+						{
-						primary_node_id = get_primary_node_id(local_conn);
+							primary_node_id = get_primary_node_id(local_conn);
-						return;
+							return;
+						}
 					}
 				}
 			}
@@ -940,8 +1077,8 @@ monitor_streaming_standby(void)
 								   degraded_monitoring_elapsed);
 
 						initPQExpBuffer(&event_details);
-						appendPQExpBuffer(&event_details,
+						appendPQExpBufferStr(&event_details,
-										  "promotion command failed but promotion completed successfully");
+											 _("promotion command failed but promotion completed successfully"));
 						create_event_notification(local_conn,
 												  &config_file_options,
 												  local_node_info.node_id,
@@ -990,7 +1127,7 @@ monitor_streaming_standby(void)
 				}
 
 
-				if (config_file_options.failover == FAILOVER_AUTOMATIC)
+				if (config_file_options.failover == FAILOVER_AUTOMATIC && repmgrd_is_paused(local_conn) == false)
 				{
 					get_active_sibling_node_records(local_conn,
 													local_node_info.node_id,
@@ -1066,15 +1203,23 @@ loop:
 
 				if (config_file_options.failover == FAILOVER_MANUAL)
 				{
-					appendPQExpBuffer(&monitoring_summary,
+					appendPQExpBufferStr(&monitoring_summary,
-									  _(" (automatic failover disabled)"));
+										 _(" (automatic failover disabled)"));
 				}
 
 				log_info("%s", monitoring_summary.data);
 				termPQExpBuffer(&monitoring_summary);
 				if (monitoring_state == MS_DEGRADED && config_file_options.failover == FAILOVER_AUTOMATIC)
 				{
-					log_detail(_("waiting for upstream or another primary to reappear"));
+					if (PQstatus(local_conn) == CONNECTION_OK && repmgrd_is_paused(local_conn))
+					{
+						log_detail(_("repmgrd paused by administrator"));
+						log_hint(_("execute \"repmgr daemon unpause\" to resume normal failover mode"));
+					}
+					else
+					{
+						log_detail(_("waiting for upstream or another primary to reappear"));
+					}
 				}
 				else if (config_file_options.monitoring_history == true)
 				{
@@ -1121,6 +1266,7 @@ loop:
 
 		check_connection(&local_node_info, &local_conn);
 
+
 		if (PQstatus(local_conn) != CONNECTION_OK)
 		{
 			if (local_node_info.active == true)
@@ -1161,11 +1307,37 @@ loop:
 		}
 		else
 		{
+			int stored_local_node_id = repmgrd_get_local_node_id(local_conn);
+
+			/*
+			 * If the local node was restarted, we'll need to reinitialise values
+			 * stored in shared memory.
+			 */
+
+			if (stored_local_node_id == UNKNOWN_NODE_ID)
+			{
+				repmgrd_set_local_node_id(local_conn, config_file_options.node_id);
+				repmgrd_set_pid(local_conn, getpid(), pid_file);
+			}
+
+			if (PQstatus(primary_conn) == CONNECTION_OK)
+			{
+				if (get_recovery_type(primary_conn) == RECTYPE_STANDBY)
+				{
+					log_notice(_("current upstream node \"%s\" (node ID: %i) is not primary, restarting monitoring"),
+							   upstream_node_info.node_name, upstream_node_info.node_id);
+					PQfinish(primary_conn);
+					primary_conn = NULL;
+					termPQExpBuffer(&event_details);
+
+					local_node_info.upstream_node_id = UNKNOWN_NODE_ID;
+					return;
+				}
+			}
+
 			/* we've reconnected to the local node after an outage */
 			if (local_node_info.active == false)
 			{
-				int stored_local_node_id = UNKNOWN_NODE_ID;
-
 				if (PQstatus(primary_conn) == CONNECTION_OK)
 				{
 					if (update_node_record_set_active(primary_conn, local_node_info.node_id, true) == true)
@@ -1175,7 +1347,6 @@ loop:
 						initPQExpBuffer(&event_details);
 
 						local_node_info.active = true;
-
 						appendPQExpBuffer(&event_details,
 										  _("reconnected to local node \"%s\" (ID: %i), marking active"),
 										  local_node_info.node_name,
@@ -1193,17 +1364,6 @@ loop:
 						termPQExpBuffer(&event_details);
 					}
 				}
-
-				/*
-				 * If the local node was restarted, we'll need to reinitialise values
-				 * stored in shared memory.
-				 */
-
-				stored_local_node_id = repmgrd_get_local_node_id(local_conn);
-				if (stored_local_node_id == UNKNOWN_NODE_ID)
-				{
-					repmgrd_set_local_node_id(local_conn, config_file_options.node_id);
-				}
 			}
 		}
 
@@ -1239,7 +1399,7 @@ monitor_streaming_witness(void)
 	/*
 	 * At this point we can't trust the local copy of "repmgr.nodes", as
 	 * it may not have been updated. We'll scan the cluster for the current
-[''	 * primary and refresh the copy from that before proceeding further.
+	 * primary and refresh the copy from that before proceeding further.
 	 */
 	primary_conn = get_primary_connection_quiet(local_conn, &primary_node_id, NULL);
 
@@ -1255,8 +1415,7 @@ monitor_streaming_witness(void)
 				  upstream_node_info.node_id);
 		log_hint(_("primary node must be running before repmgrd can start"));
 
-		close_connection(&local_conn);
+		terminate(ERR_DB_CONN);
-		exit(ERR_DB_CONN);
 	}
 
 	/* synchronise local copy of "repmgr.nodes", in case it was stale */
@@ -1280,12 +1439,13 @@ monitor_streaming_witness(void)
 		log_warning(_("unable to retrieve node record from primary"));
 	}
 
-
-	/* Log startup event */
-	if (startup_event_logged == false)
 	{
 		PQExpBufferData event_details;
 
+		char *event_type = startup_event_logged == false
+			? "repmgrd_start"
+			: "repmgrd_upstream_reconnect";
+
 		initPQExpBuffer(&event_details);
 
 		appendPQExpBuffer(&event_details,
@@ -1293,16 +1453,17 @@ monitor_streaming_witness(void)
 						  upstream_node_info.node_name,
 						  upstream_node_info.node_id);
 
+		log_info("%s", event_details.data);
+
 		create_event_notification(primary_conn,
 								  &config_file_options,
 								  config_file_options.node_id,
-								  "repmgrd_start",
+								  event_type,
 								  true,
 								  event_details.data);
 
-		startup_event_logged = true;
+		if (startup_event_logged == false)
-
+			startup_event_logged = true;
-		log_info("%s", event_details.data);
 
 		termPQExpBuffer(&event_details);
 	}
@@ -1352,6 +1513,17 @@ monitor_streaming_witness(void)
 									  upstream_node_unreachable_elapsed);
 					log_notice("%s", event_details.data);
 
+					/* check upstream is still primary */
+					if (get_recovery_type(primary_conn) != RECTYPE_PRIMARY)
+					{
+						log_notice(_("current upstream node \"%s\" (node ID: %i) is not primary, restarting monitoring"),
+								   upstream_node_info.node_name, upstream_node_info.node_id);
+						PQfinish(primary_conn);
+						primary_conn = NULL;
+						termPQExpBuffer(&event_details);
+						return;
+					}
+
 					create_event_notification(primary_conn,
 											  &config_file_options,
 											  config_file_options.node_id,
@@ -1409,14 +1581,25 @@ monitor_streaming_witness(void)
 									  upstream_node_info.node_id,
 									  degraded_monitoring_elapsed);
 
+					log_notice("%s", event_details.data);
+
+					/* check upstream is still primary */
+					if (get_recovery_type(primary_conn) != RECTYPE_PRIMARY)
+					{
+						log_notice(_("current upstream node \"%s\" (node ID: %i) is not primary, restarting monitoring"),
+								   upstream_node_info.node_name, upstream_node_info.node_id);
+						PQfinish(primary_conn);
+						primary_conn = NULL;
+						termPQExpBuffer(&event_details);
+						return;
+					}
+
 					create_event_notification(primary_conn,
 											  &config_file_options,
 											  config_file_options.node_id,
 											  "repmgrd_upstream_reconnect",
 											  true,
 											  event_details.data);
-
-					log_notice("%s", event_details.data);
 					termPQExpBuffer(&event_details);
 
 					goto loop;
@@ -1448,6 +1631,12 @@ monitor_streaming_witness(void)
 							continue;
 						}
 
+						/* skip node if configured as a witness node - we can't possibly "follow" that */
+						if (cell->node_info->type == WITNESS)
+						{
+							continue;
+						}
+
 						cell->node_info->conn = establish_db_connection(cell->node_info->conninfo, false);
 
 						if (PQstatus(cell->node_info->conn) != CONNECTION_OK)
@@ -1569,19 +1758,34 @@ loop:
 				if (stored_local_node_id == UNKNOWN_NODE_ID)
 				{
 					repmgrd_set_local_node_id(local_conn, config_file_options.node_id);
+					repmgrd_set_pid(local_conn, getpid(), pid_file);
 				}
 			}
 		}
 
 
-		/* refresh repmgr.nodes after "witness_sync_interval" seconds */
+		/*
+		 * Refresh repmgr.nodes after "witness_sync_interval" seconds, and check if primary
+		 * has changed
+		 */
 
 		{
 			int witness_sync_interval_elapsed = calculate_elapsed(witness_sync_interval_start);
+
 			if (witness_sync_interval_elapsed >= config_file_options.witness_sync_interval)
 			{
+				if (get_recovery_type(primary_conn) != RECTYPE_PRIMARY)
+				{
+					log_notice(_("current upstream node \"%s\" (node ID: %i) is not primary, restarting monitoring"),
+							   upstream_node_info.node_name, upstream_node_info.node_id);
+					PQfinish(primary_conn);
+					primary_conn = NULL;
+					return;
+				}
+
 				log_debug("synchronising witness node records");
 				witness_copy_node_records(primary_conn, local_conn);
+
 				INSTR_TIME_SET_CURRENT(witness_sync_interval_start);
 			}
 		}
@@ -1672,7 +1876,19 @@ do_primary_failover(void)
 	}
 	else if (election_result == ELECTION_LOST || election_result == ELECTION_NOT_CANDIDATE)
 	{
-		log_info(_("follower node awaiting notification from the candidate node"));
+		/*
+		 * if the node couldn't be promoted as it's not in the same location as the primary,
+		 * add an explanatory notice
+		 */
+		if (election_result == ELECTION_NOT_CANDIDATE && strncmp(upstream_node_info.location, local_node_info.location, MAXLEN) != 0)
+		{
+			log_notice(_("this node's location (\"%s\") is not the primary node location (\"%s\"), so node cannot be promoted"),
+					   local_node_info.location,
+					   upstream_node_info.location);
+		}
+
+		log_info(_("follower node awaiting notification from a candidate node"));
+
 		failover_state = FAILOVER_STATE_WAITING_NEW_PRIMARY;
 	}
 
@@ -2090,6 +2306,7 @@ do_upstream_standby_failover(void)
 
 	/* refresh shared memory settings which will have been zapped by the restart */
 	repmgrd_set_local_node_id(local_conn, config_file_options.node_id);
+	repmgrd_set_pid(local_conn, getpid(), pid_file);
 
 	/*
 	 *
@@ -2486,8 +2703,8 @@ follow_new_primary(int new_primary_id)
 			if (upstream_recovery_type == RECTYPE_PRIMARY)
 			{
 				initPQExpBuffer(&event_details);
-				appendPQExpBuffer(&event_details,
+				appendPQExpBufferStr(&event_details,
-								  _("original primary reappeared - no action taken"));
+									 _("original primary reappeared - no action taken"));
 
 				log_notice("%s", event_details.data);
 
@@ -2562,6 +2779,7 @@ follow_new_primary(int new_primary_id)
 
 	/* refresh shared memory settings which will have been zapped by the restart */
 	repmgrd_set_local_node_id(local_conn, config_file_options.node_id);
+	repmgrd_set_pid(local_conn, getpid(), pid_file);
 
 	initPQExpBuffer(&event_details);
 	appendPQExpBuffer(&event_details,
@@ -2776,7 +2994,9 @@ do_election(void)
 
 	total_nodes = sibling_nodes.node_count + 1;
 
-	log_debug("do_election(): primary location is %s", upstream_node_info.location);
+	log_debug("do_election(): primary location is \"%s\", standby location is \"%s\"",
+			  upstream_node_info.location,
+			  local_node_info.location);
 
 	local_node_info.last_wal_receive_lsn = InvalidXLogRecPtr;
 
@@ -3056,6 +3276,8 @@ check_connection(t_node_info *node_info, PGconn **conn)
 	if (is_server_available(node_info->conninfo) == false)
 	{
 		log_warning(_("connection to node %i lost"), node_info->node_id);
+		PQfinish(*conn);
+		*conn = NULL;
 	}
 
 	if (PQstatus(*conn) != CONNECTION_OK)
@@ -3084,6 +3306,7 @@ check_connection(t_node_info *node_info, PGconn **conn)
 			if (stored_local_node_id == UNKNOWN_NODE_ID)
 			{
 				repmgrd_set_local_node_id(*conn, config_file_options.node_id);
+				repmgrd_set_pid(local_conn, getpid(), pid_file);
 			}
 
 		}

repmgrd.c

@@ -35,7 +35,7 @@
 
 static char *config_file = NULL;
 static bool verbose = false;
-static char pid_file[MAXPGPATH];
+char pid_file[MAXPGPATH];
 static bool daemonize = true;
 static bool show_pid_file = false;
 static bool no_pid_file = false;
@@ -101,7 +101,8 @@ main(int argc, char **argv)
 		{"config-file", required_argument, NULL, 'f'},
 
 /* daemon options */
-		{"daemonize", optional_argument, NULL, 'd'},
+		{"daemonize-short", optional_argument, NULL, 'd'},
+		{"daemonize", optional_argument, NULL, OPT_DAEMONIZE},
 		{"pid-file", required_argument, NULL, 'p'},
 		{"show-pid-file", no_argument, NULL, 's'},
 		{"no-pid-file", no_argument, NULL, OPT_NO_PID_FILE},
@@ -175,10 +176,11 @@ main(int argc, char **argv)
 				/* daemon options */
 
 			case 'd':
-				if (optarg != NULL)
+				daemonize = true;
-				{
+				break;
-					daemonize = parse_bool(optarg, "-d/--daemonize", &cli_errors);
+
-				}
+			case OPT_DAEMONIZE:
+				daemonize = parse_bool(optarg, "-d/--daemonize", &cli_errors);
 				break;
 
 			case 'p':
@@ -488,6 +490,9 @@ main(int argc, char **argv)
 		check_and_create_pid_file(pid_file);
 	}
 
+	repmgrd_set_pid(local_conn, getpid(), pid_file);
+
+
 #ifndef WIN32
 	setup_event_handlers();
 #endif
@@ -760,7 +765,8 @@ show_help(void)
 	puts("");
 
 	printf(_("Daemon configuration options:\n"));
-	printf(_("  -d, --daemonize[=true/false]\n"));
+	printf(_("  -d\n"));
+	printf(_("  --daemonize[=true/false]\n"));
 	printf(_("                            detach process from foreground (default: true)\n"));
 	printf(_("  -p, --pid-file=PATH       use the specified PID file\n"));
 	printf(_("  -s, --show-pid-file       show PID file which would be used by the current configuration\n"));
@@ -901,6 +907,9 @@ print_monitoring_state(MonitoringState monitoring_state)
 void
 terminate(int retval)
 {
+	if (PQstatus(local_conn)  == CONNECTION_OK)
+		repmgrd_set_pid(local_conn, UNKNOWN_PID, NULL);
+
 	logger_shutdown();
 
 	if (pid_file[0] != '\0')

repmgrd.h

@@ -11,6 +11,7 @@
 #include "portability/instr_time.h"
 
 #define OPT_NO_PID_FILE                  1000
+#define OPT_DAEMONIZE                    1001
 
 extern volatile sig_atomic_t got_SIGHUP;
 extern MonitoringState monitoring_state;
@@ -20,6 +21,7 @@ extern t_configuration_options config_file_options;
 extern t_node_info local_node_info;
 extern PGconn *local_conn;
 extern bool startup_event_logged;
+extern char pid_file[MAXPGPATH];
 
 void		try_reconnect(PGconn **conn, t_node_info *node_info);
 

strutil.c

@@ -87,17 +87,17 @@ append_where_clause(PQExpBufferData *where_clause, const char *format,...)
 
 	if (where_clause->data[0] == '\0')
 	{
-		appendPQExpBuffer(where_clause,
+		appendPQExpBufferStr(where_clause,
-						  " WHERE ");
+							 " WHERE ");
 	}
 	else
 	{
-		appendPQExpBuffer(where_clause,
+		appendPQExpBufferStr(where_clause,
-						  " AND ");
+							 " AND ");
 	}
 
-	appendPQExpBuffer(where_clause,
+	appendPQExpBufferStr(where_clause,
-					  "%s", stringbuf);
+						 stringbuf);
 
 }