doc: add a link to the current documentation from the contents page

Per user report on mailing list.

HISTORY

@@ -1,21 +1,65 @@
+4.1.1   2018-09-05
+        logging: explicitly log the text of failed queries as ERRORs to
+          assist logfile analysis; GitHub #498
+        repmgr: truncate version string, if necessary; GitHub #490 (Ian)
+        repmgr: improve messages emitted during "standby promote" (Ian)
+        repmgr: "standby clone" - don't copy external config files in --dry-run
+          mode; GitHub #491 (Ian)
+        repmgr: add "cluster_cleanup" event; GitHub #492 (Ian)
+        repmgr: (standby switchover) improve detection of free walsenders;
+          GitHub #495 (Ian)
+        repmgr: (node rejoin) improve replication slot handling; GitHub #499 (Ian)
+        repmgrd: ensure that sending SIGHUP always results in the log file
+          being reopened; GitHub #485 (Ian)
+        repmgrd: report version number *after* logger initialisation; GitHub #487 (Ian)
+        repmgrd: fix startup on witness node when local data is stale; GitHub #488/#489 (Ian)
+        repmgrd: improve cascaded standby failover handling; GitHub #480 (Ian)
+        repmgrd: improve reconnection handling (Ian)
+
+4.1.0   2018-07-31
+        repmgr: change default log_level to INFO, add documentation; GitHub #470 (Ian)
+        repmgr: add "--missing-slots" check to "repmgr node check" (Ian)
+        repmgr: improve command line error handling; GitHub #464 (Ian)
+        repmgr: fix "standby register --wait-sync" when no timeout provided (Ian)
+        repmgr: "cluster show" returns non-zero value if an issue encountered;
+          GitHub #456 (Ian)
+        repmgr: "node check" and "node status" returns non-zero value if an issue
+           encountered (Ian)
+        repmgr: add CSV output mode to "cluster event"; GitHub #471 (Ian)
+        repmgr: add -q/--quiet option to suppress non-error output; GitHub #468 (Ian)
+        repmgr: "node status" returns non-zero value if an issue encountered (Ian)
+        repmgr: enable "recovery_min_apply_delay" to be 0; GitHub #448 (Ian)
+        repmgr: "cluster cleanup" - add missing help options; GitHub #461/#462 (gclough)
+        repmgr: ensure witness node follows new primary after switchover;
+          GitHub #453 (Ian)
+        repmgr: fix witness node handling in "node check"/"node status";
+          GitHub #451 (Ian)
+        repmgr: fix "primary_slot_name" when using "standby clone" with --recovery-conf-only;
+          GitHub #474 (Ian)
+        repmgr: don't perform a switchover if an exclusive backup is running;
+          GitHub #476 (Martín)
+        repmgr: enable "witness unregister" to be run on any node; GitHub #472 (Ian)
+        repmgrd: create a PID file by default; GitHub #457 (Ian)
+        repmgrd: daemonize process by default; GitHub #458 (Ian)
+
 4.0.6   2018-06-14
         repmgr: (witness register) prevent registration of a witness server with the
- 		  same name as an existing node (Ian)
+          same name as an existing node (Ian)
-		repmgr: (standby follow) check node has actually connected to new primary
+        repmgr: (standby follow) check node has actually connected to new primary
-		  before reporting success; GitHub #444 (Ian)
+          before reporting success; GitHub #444 (Ian)
-		repmgr: (standby clone) improve handling of external configuration file copying,
+        repmgr: (standby clone) improve handling of external configuration file copying,
-		  including consideration in --dry-run check; GitHub #443 (Ian)
+          including consideration in --dry-run check; GitHub #443 (Ian)
-		repmgr: (standby clone) don't require presence of "user" parameter in
+        repmgr: (standby clone) don't require presence of "user" parameter in
-		  conninfo string; GitHub #437 (Ian)
+          conninfo string; GitHub #437 (Ian)
-		repmgr: (standby clone) improve documentation of --recovery-conf-only
+        repmgr: (standby clone) improve documentation of --recovery-conf-only
-		  mode; GitHub #438 (Ian)
+          mode; GitHub #438 (Ian)
-		repmgr: (node rejoin) fix bug when parsing --config-files parameter;
+        repmgr: (node rejoin) fix bug when parsing --config-files parameter;
-		  GitHub #442 (Ian)
+          GitHub #442 (Ian)
-		repmgr: when using --dry-run, force log level to INFO to ensure output
+        repmgr: when using --dry-run, force log level to INFO to ensure output
-		  will always be displayed; GitHub #441 (Ian)
+          will always be displayed; GitHub #441 (Ian)
         repmgr: (cluster matrix/crosscheck) return non-zero exit code if node
            connection issues detected; GitHub #447 (Ian)
-		repmgrd: ensure local node is counted as quorum member; GitHub #439 (Ian)
+        repmgrd: ensure local node is counted as quorum member; GitHub #439 (Ian)
 
 4.0.5   2018-05-02
         repmgr: poll demoted primary after restart as a standby during a

Makefile.in

@@ -11,7 +11,10 @@ EXTENSION = repmgr
 
 DATA = \
   repmgr--unpackaged--4.0.sql \
-  repmgr--4.0.sql
+  repmgr--4.0.sql \
+  repmgr--4.0--4.1.sql \
+  repmgr--4.1.sql
+
 
 REGRESS = repmgr_extension
 

configfile.c

@@ -28,10 +28,8 @@ char		config_file_path[MAXPGPATH] = "";
 static bool config_file_provided = false;
 bool		config_file_found = false;
 
+static void parse_config(t_configuration_options *options, bool terse);
 static void _parse_config(t_configuration_options *options, ItemList *error_list, ItemList *warning_list);
-static bool parse_bool(const char *s,
-		   const char *config_item,
-		   ItemList *error_list);
 
 static void _parse_line(char *buf, char *name, char *value);
 static void parse_event_notifications_list(t_configuration_options *options, const char *arg);
@@ -241,7 +239,7 @@ end_search:
 }
 
 
-void
+static void
 parse_config(t_configuration_options *options, bool terse)
 {
 	/* Collate configuration file errors here for friendlier reporting */
@@ -333,6 +331,12 @@ _parse_config(t_configuration_options *options, ItemList *error_list, ItemList *
 	options->primary_follow_timeout = DEFAULT_PRIMARY_FOLLOW_TIMEOUT;
 	options->standby_follow_timeout = DEFAULT_STANDBY_FOLLOW_TIMEOUT;
 
+	/*------------------------
+	 * standby switchover settings
+	 *------------------------
+	 */
+	options->standby_reconnect_timeout = DEFAULT_STANDBY_RECONNECT_TIMEOUT;
+
 	/*-----------------
 	 * repmgrd settings
 	 *-----------------
@@ -352,7 +356,8 @@ _parse_config(t_configuration_options *options, ItemList *error_list, ItemList *
 	options->degraded_monitoring_timeout = -1;
 	options->async_query_timeout = DEFAULT_ASYNC_QUERY_TIMEOUT;
 	options->primary_notification_timeout = DEFAULT_PRIMARY_NOTIFICATION_TIMEOUT;
-	options->standby_reconnect_timeout = DEFAULT_STANDBY_RECONNECT_TIMEOUT;
+	options->repmgrd_standby_startup_timeout = -1; /* defaults to "standby_reconnect_timeout" if not set */
+	memset(options->repmgrd_pid_file, 0, sizeof(options->repmgrd_pid_file));
 
 	/*-------------
 	 * witness settings
@@ -539,6 +544,14 @@ _parse_config(t_configuration_options *options, ItemList *error_list, ItemList *
 		else if (strcmp(name, "standby_follow_timeout") == 0)
 			options->standby_follow_timeout = repmgr_atoi(value, name, error_list, 0);
 
+		/* standby switchover settings */
+		else if (strcmp(name, "standby_reconnect_timeout") == 0)
+			options->standby_reconnect_timeout = repmgr_atoi(value, name, error_list, 0);
+
+		/* node rejoin settings */
+		else if (strcmp(name, "node_rejoin_timeout") == 0)
+			options->node_rejoin_timeout = repmgr_atoi(value, name, error_list, 0);
+
 		/* node check settings */
 		else if (strcmp(name, "archive_ready_warning") == 0)
 			options->archive_ready_warning = repmgr_atoi(value, name, error_list, 1);
@@ -588,8 +601,10 @@ _parse_config(t_configuration_options *options, ItemList *error_list, ItemList *
 			options->async_query_timeout = repmgr_atoi(value, name, error_list, 0);
 		else if (strcmp(name, "primary_notification_timeout") == 0)
 			options->primary_notification_timeout = repmgr_atoi(value, name, error_list, 0);
-		else if (strcmp(name, "standby_reconnect_timeout") == 0)
+		else if (strcmp(name, "repmgrd_standby_startup_timeout") == 0)
-			options->standby_reconnect_timeout = repmgr_atoi(value, name, error_list, 0);
+			options->repmgrd_standby_startup_timeout = repmgr_atoi(value, name, error_list, 0);
+		else if (strcmp(name, "repmgrd_pid_file") == 0)
+			strncpy(options->repmgrd_pid_file, value, MAXPGPATH);
 
 		/* witness settings */
 		else if (strcmp(name, "witness_sync_interval") == 0)
@@ -771,6 +786,17 @@ _parse_config(t_configuration_options *options, ItemList *error_list, ItemList *
 		PQconninfoFree(conninfo_options);
 	}
 
+	/* set values for parameters which default to other parameters */
+
+	/*
+	 * From 4.1, "repmgrd_standby_startup_timeout" replaces "standby_reconnect_timeout"
+	 * in repmgrd; fall back to "standby_reconnect_timeout" if no value explicitly provided
+	 */
+	if (options->repmgrd_standby_startup_timeout == -1)
+	{
+		options->repmgrd_standby_startup_timeout = options->standby_reconnect_timeout;
+	}
+
 	/* add warning about changed "barman_" parameter meanings */
 	if ((options->barman_host[0] == '\0' && options->barman_server[0] != '\0') ||
 		(options->barman_host[0] != '\0' && options->barman_server[0] == '\0'))
@@ -795,6 +821,12 @@ _parse_config(t_configuration_options *options, ItemList *error_list, ItemList *
 		item_list_append(error_list,
 						 _("\replication_lag_critical\" must be greater than  \"replication_lag_warning\""));
 	}
+
+	if (options->standby_reconnect_timeout < options->node_rejoin_timeout)
+	{
+		item_list_append(error_list,
+						 _("\"standby_reconnect_timeout\" must be equal to or greater than \"node_rejoin_timeout\""));
+	}
 }
 
 
@@ -959,12 +991,11 @@ parse_time_unit_parameter(const char *name, const char *value, char *dest, ItemL
 	char	   *ptr = NULL;
 	int			targ = strtol(value, &ptr, 10);
 
-	if (targ < 1)
+	if (targ < 0)
 	{
 		if (errors != NULL)
 		{
-			item_list_append_format(
+			item_list_append_format(errors,
-									errors,
 									_("invalid value provided for \"%s\""),
 									name);
 		}
@@ -1018,13 +1049,16 @@ parse_time_unit_parameter(const char *name, const char *value, char *dest, ItemL
  * - promote_delay
  * - reconnect_attempts
  * - reconnect_interval
+ * - repmgrd_standby_startup_timeout
  * - retry_promote_interval_secs
  *
- * non-changeable options
+ * non-changeable options (repmgrd references these from the "repmgr.nodes"
+ * table, not the configuration file)
  *
  * - node_id
  * - node_name
  * - data_directory
+ * - location
  * - priority
  * - replication_type
  *
@@ -1033,7 +1067,7 @@ parse_time_unit_parameter(const char *name, const char *value, char *dest, ItemL
 
  */
 bool
-reload_config(t_configuration_options *orig_options)
+reload_config(t_configuration_options *orig_options, t_server_type server_type)
 {
 	PGconn	   *conn;
 	t_configuration_options new_options = T_CONFIGURATION_OPTIONS_INITIALIZER;
@@ -1043,17 +1077,50 @@ reload_config(t_configuration_options *orig_options)
 	static ItemList config_errors = {NULL, NULL};
 	static ItemList config_warnings = {NULL, NULL};
 
+	PQExpBufferData errors;
+
 	log_info(_("reloading configuration file"));
 
 	_parse_config(&new_options, &config_errors, &config_warnings);
 
+
+	if (server_type == PRIMARY || server_type == STANDBY)
+	{
+		if (new_options.promote_command[0] == '\0')
+		{
+			item_list_append(&config_errors, _("\"promote_command\": required parameter was not found"));
+		}
+
+		if (new_options.follow_command[0] == '\0')
+		{
+			item_list_append(&config_errors, _("\"follow_command\": required parameter was not found"));
+		}
+	}
+
 	if (config_errors.head != NULL)
 	{
-		/* XXX dump errors to log */
+		ItemListCell *cell = NULL;
+
 		log_warning(_("unable to parse new configuration, retaining current configuration"));
+
+		initPQExpBuffer(&errors);
+
+		appendPQExpBuffer(&errors,
+						  "following errors were detected:\n");
+
+		for (cell = config_errors.head; cell; cell = cell->next)
+		{
+			appendPQExpBuffer(&errors,
+							  "  %s\n", cell->string);
+		}
+
+		log_detail("%s", errors.data);
+		termPQExpBuffer(&errors);
 		return false;
 	}
 
+
+
 	/* The following options cannot be changed */
 
 	if (new_options.node_id != orig_options->node_id)
@@ -1207,7 +1274,7 @@ reload_config(t_configuration_options *orig_options)
 		config_changed = true;
 	}
 
-	/* promote_delay */
+	/* promote_delay (for testing use only; not documented */
 	if (orig_options->promote_delay != new_options.promote_delay)
 	{
 		orig_options->promote_delay = new_options.promote_delay;
@@ -1234,6 +1301,15 @@ reload_config(t_configuration_options *orig_options)
 		config_changed = true;
 	}
 
+	/* repmgrd_standby_startup_timeout */
+	if (orig_options->repmgrd_standby_startup_timeout != new_options.repmgrd_standby_startup_timeout)
+	{
+		orig_options->repmgrd_standby_startup_timeout = new_options.repmgrd_standby_startup_timeout;
+		log_info(_("\"repmgrd_standby_startup_timeout\" is now \"%i\""), new_options.repmgrd_standby_startup_timeout);
+
+		config_changed = true;
+	}
+
 	/*
 	 * Handle changes to logging configuration
 	 */
@@ -1326,13 +1402,23 @@ exit_with_config_file_errors(ItemList *config_errors, ItemList *config_warnings,
 
 
 void
-exit_with_cli_errors(ItemList *error_list)
+exit_with_cli_errors(ItemList *error_list, const char *repmgr_command)
 {
 	fprintf(stderr, _("The following command line errors were encountered:\n"));
 
 	print_item_list(error_list);
 
-	fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname());
+	if (repmgr_command != NULL)
+	{
+		fprintf(stderr, _("Try \"%s --help\" or \"%s %s --help\" for more information.\n"),
+				progname(),
+				progname(),
+				repmgr_command);
+	}
+	else
+	{
+		fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname());
+	}
 
 	exit(ERR_BAD_CONFIG);
 }
@@ -1437,7 +1523,7 @@ repmgr_atoi(const char *value, const char *config_item, ItemList *error_list, in
  *
  *   https://www.postgresql.org/docs/current/static/config-setting.html
  */
-static bool
+bool
 parse_bool(const char *s, const char *config_item, ItemList *error_list)
 {
 	PQExpBufferData errors;
@@ -1723,6 +1809,9 @@ free_parsed_argv(char ***argv_array)
 }
 
 
+
+
+
 bool
 parse_pg_basebackup_options(const char *pg_basebackup_options, t_basebackup_options *backup_options, int server_version_num, ItemList *error_list)
 {

configfile.h

@@ -102,6 +102,12 @@ typedef struct
 	int			primary_follow_timeout;
 	int			standby_follow_timeout;
 
+	/* standby switchover settings */
+	int			standby_reconnect_timeout;
+
+	/* node rejoin settings */
+	int			node_rejoin_timeout;
+
 	/* node check settings */
 	int			archive_ready_warning;
 	int			archive_ready_critical;
@@ -124,7 +130,8 @@ typedef struct
 	int			degraded_monitoring_timeout;
 	int			async_query_timeout;
 	int			primary_notification_timeout;
-	int			standby_reconnect_timeout;
+	int			repmgrd_standby_startup_timeout;
+	char		repmgrd_pid_file[MAXPGPATH];
 
 	/* BDR settings */
 	bool		bdr_local_monitoring_only;
@@ -173,6 +180,10 @@ typedef struct
 		/* standby follow settings */ \
 		DEFAULT_PRIMARY_FOLLOW_TIMEOUT,	\
 		DEFAULT_STANDBY_FOLLOW_TIMEOUT,	\
+		/* standby switchover settings */ \
+		DEFAULT_STANDBY_RECONNECT_TIMEOUT, \
+		/* node rejoin settings */ \
+		DEFAULT_NODE_REJOIN_TIMEOUT, \
 		/* node check settings */ \
 		DEFAULT_ARCHIVE_READY_WARNING, DEFAULT_ARCHIVE_READY_CRITICAL, \
 		DEFAULT_REPLICATION_LAG_WARNING, DEFAULT_REPLICATION_LAG_CRITICAL, \
@@ -186,7 +197,7 @@ typedef struct
         false, -1, \
 		DEFAULT_ASYNC_QUERY_TIMEOUT, \
 		DEFAULT_PRIMARY_NOTIFICATION_TIMEOUT,	\
-		DEFAULT_STANDBY_RECONNECT_TIMEOUT,	\
+		-1, "", \
 		/* BDR settings */ \
 		false, DEFAULT_BDR_RECOVERY_TIMEOUT, \
 		/* service settings */ \
@@ -262,16 +273,20 @@ typedef struct
 	"", "", "", "" \
 }
 
+#include "dbutils.h"
 
 void		set_progname(const char *argv0);
 const char *progname(void);
 
 void		load_config(const char *config_file, bool verbose, bool terse, t_configuration_options *options, char *argv0);
-void		parse_config(t_configuration_options *options, bool terse);
+bool		reload_config(t_configuration_options *orig_options, t_server_type server_type);
-bool		reload_config(t_configuration_options *orig_options);
 
 bool		parse_recovery_conf(const char *data_dir, t_recovery_conf *conf);
 
+bool		parse_bool(const char *s,
+					   const char *config_item,
+					   ItemList *error_list);
+
 int repmgr_atoi(const char *s,
 			const char *config_item,
 			ItemList *error_list,
@@ -287,7 +302,7 @@ void free_parsed_argv(char ***argv_array);
 
 
 /* called by repmgr-client and repmgrd */
-void		exit_with_cli_errors(ItemList *error_list);
+void		exit_with_cli_errors(ItemList *error_list, const char *repmgr_command);
 void		print_item_list(ItemList *item_list);
 
 #endif							/* _REPMGR_CONFIGFILE_H_ */

configure

@@ -1,6 +1,6 @@
 #! /bin/sh
 # Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.69 for repmgr 4.0.5.
+# Generated by GNU Autoconf 2.69 for repmgr 4.1.2.
 #
 # Report bugs to <pgsql-bugs@postgresql.org>.
 #
@@ -582,8 +582,8 @@ MAKEFLAGS=
 # Identity of this package.
 PACKAGE_NAME='repmgr'
 PACKAGE_TARNAME='repmgr'
-PACKAGE_VERSION='4.0.5'
+PACKAGE_VERSION='4.1.2'
-PACKAGE_STRING='repmgr 4.0.5'
+PACKAGE_STRING='repmgr 4.1.2'
 PACKAGE_BUGREPORT='pgsql-bugs@postgresql.org'
 PACKAGE_URL='https://2ndquadrant.com/en/resources/repmgr/'
 
@@ -1178,7 +1178,7 @@ if test "$ac_init_help" = "long"; then
   # Omit some internal or obsolete options to make the list less imposing.
   # This message is too long to be a string in the A/UX 3.1 sh.
   cat <<_ACEOF
-\`configure' configures repmgr 4.0.5 to adapt to many kinds of systems.
+\`configure' configures repmgr 4.1.2 to adapt to many kinds of systems.
 
 Usage: $0 [OPTION]... [VAR=VALUE]...
 
@@ -1239,7 +1239,7 @@ fi
 
 if test -n "$ac_init_help"; then
   case $ac_init_help in
-     short | recursive ) echo "Configuration of repmgr 4.0.5:";;
+     short | recursive ) echo "Configuration of repmgr 4.1.2:";;
    esac
   cat <<\_ACEOF
 
@@ -1313,7 +1313,7 @@ fi
 test -n "$ac_init_help" && exit $ac_status
 if $ac_init_version; then
   cat <<\_ACEOF
-repmgr configure 4.0.5
+repmgr configure 4.1.2
 generated by GNU Autoconf 2.69
 
 Copyright (C) 2012 Free Software Foundation, Inc.
@@ -1332,7 +1332,7 @@ cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.
 
-It was created by repmgr $as_me 4.0.5, which was
+It was created by repmgr $as_me 4.1.2, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   $ $0 $@
@@ -2359,7 +2359,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 # report actual input values of CONFIG_FILES etc. instead of their
 # values after options handling.
 ac_log="
-This file was extended by repmgr $as_me 4.0.5, which was
+This file was extended by repmgr $as_me 4.1.2, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   CONFIG_FILES    = $CONFIG_FILES
@@ -2422,7 +2422,7 @@ _ACEOF
 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
 ac_cs_version="\\
-repmgr config.status 4.0.5
+repmgr config.status 4.1.2
 configured by $0, generated by GNU Autoconf 2.69,
   with options \\"\$ac_cs_config\\"
 

configure.in

@@ -1,4 +1,4 @@
-AC_INIT([repmgr], [4.0.6], [pgsql-bugs@postgresql.org], [repmgr], [https://2ndquadrant.com/en/resources/repmgr/])
+AC_INIT([repmgr], [4.1.2], [pgsql-bugs@postgresql.org], [repmgr], [https://2ndquadrant.com/en/resources/repmgr/])
 
 AC_COPYRIGHT([Copyright (c) 2010-2018, 2ndQuadrant Ltd.])
 

controldata.c

@@ -227,7 +227,15 @@ get_controlfile(const char *DataDir)
 
 	control_file_info->control_file_processed = true;
 
-	if (version_num >= 90500)
+	if (version_num >= 110000)
+	{
+		ControlFileData11 *ptr = (struct ControlFileData11 *)ControlFileDataPtr;
+		control_file_info->system_identifier = ptr->system_identifier;
+		control_file_info->state = ptr->state;
+		control_file_info->checkPoint = ptr->checkPoint;
+		control_file_info->data_checksum_version = ptr->data_checksum_version;
+	}
+	else if (version_num >= 90500)
 	{
 		ControlFileData95 *ptr = (struct ControlFileData95 *)ControlFileDataPtr;
 		control_file_info->system_identifier = ptr->system_identifier;

controldata.h

@@ -265,6 +265,71 @@ typedef struct ControlFileData95
 
 } ControlFileData95;
 
+/*
+ * Following field removed in 11:
+ *
+ *  XLogRecPtr	prevCheckPoint;
+ *
+ * In 10, following field appended *after* "data_checksum_version":
+ *
+ * 	char		mock_authentication_nonce[MOCK_AUTH_NONCE_LEN];
+ *
+ * (but we don't care about that)
+ */
+
+typedef struct ControlFileData11
+{
+	uint64		system_identifier;
+
+	uint32		pg_control_version;		/* PG_CONTROL_VERSION */
+	uint32		catalog_version_no;		/* see catversion.h */
+
+	DBState		state;			/* see enum above */
+	pg_time_t	time;			/* time stamp of last pg_control update */
+	XLogRecPtr	checkPoint;		/* last check point record ptr */
+
+	CheckPoint95	checkPointCopy; /* copy of last check point record */
+
+	XLogRecPtr	unloggedLSN;	/* current fake LSN value, for unlogged rels */
+
+	XLogRecPtr	minRecoveryPoint;
+	TimeLineID	minRecoveryPointTLI;
+	XLogRecPtr	backupStartPoint;
+	XLogRecPtr	backupEndPoint;
+	bool		backupEndRequired;
+
+	int			wal_level;
+	bool		wal_log_hints;
+	int			MaxConnections;
+	int			max_worker_processes;
+	int			max_prepared_xacts;
+	int			max_locks_per_xact;
+	bool		track_commit_timestamp;
+
+	uint32		maxAlign;		/* alignment requirement for tuples */
+	double		floatFormat;	/* constant 1234567.0 */
+
+	uint32		blcksz;			/* data block size for this DB */
+	uint32		relseg_size;	/* blocks per segment of large relation */
+
+	uint32		xlog_blcksz;	/* block size within WAL files */
+	uint32		xlog_seg_size;	/* size of each WAL segment */
+
+	uint32		nameDataLen;	/* catalog name field width */
+	uint32		indexMaxKeys;	/* max number of columns in an index */
+
+	uint32		toast_max_chunk_size;	/* chunk size in TOAST tables */
+	uint32		loblksize;		/* chunk size in pg_largeobject */
+
+	bool		enableIntTimes; /* int64 storage enabled? */
+
+	bool		float4ByVal;	/* float4 pass-by-value? */
+	bool		float8ByVal;	/* float8, int8, etc pass-by-value? */
+
+	uint32		data_checksum_version;
+
+} ControlFileData11;
+
 
 
 extern DBState get_db_state(const char *data_directory);

dbutils.h

@@ -29,7 +29,9 @@
 #include "voting.h"
 
 #define REPMGR_NODES_COLUMNS "n.node_id, n.type, n.upstream_node_id, n.node_name, n.conninfo, n.repluser, n.slot_name, n.location, n.priority, n.active, n.config_file, '' AS upstream_node_name "
-#define BDR_NODES_COLUMNS "node_sysid, node_timeline, node_dboid, node_status, node_name, node_local_dsn, node_init_from_dsn, node_read_only, node_seq_id"
+#define BDR2_NODES_COLUMNS "node_sysid, node_timeline, node_dboid, node_name, node_local_dsn, ''"
+#define BDR3_NODES_COLUMNS "ns.node_id, 0, 0, ns.node_name, ns.interface_connstr, ns.peer_state_name"
+
 
 #define ERRBUFF_SIZE 512
 
@@ -94,6 +96,14 @@ typedef enum
 	SLOT_ACTIVE
 } ReplSlotStatus;
 
+typedef enum
+{
+	BACKUP_STATE_UNKNOWN = -1,
+	BACKUP_STATE_IN_BACKUP,
+	BACKUP_STATE_NO_BACKUP
+} BackupState;
+
+
 /*
  * Struct to store node information
  */
@@ -237,18 +247,14 @@ typedef struct s_bdr_node_info
 	char		node_sysid[MAXLEN];
 	uint32		node_timeline;
 	uint32		node_dboid;
-	char		node_status;
 	char		node_name[MAXLEN];
 	char		node_local_dsn[MAXLEN];
-	char		node_init_from_dsn[MAXLEN];
+	char		peer_state_name[MAXLEN];
-	bool		read_only;
-	uint32		node_seq_id;
 } t_bdr_node_info;
 
 #define T_BDR_NODE_INFO_INITIALIZER { \
 	"", InvalidOid, InvalidOid, \
-	'?', "", "", "", \
+    "", "", "" \
-    false, -1 \
 }
 
 
@@ -392,6 +398,7 @@ int			get_ready_archive_files(PGconn *conn, const char *data_directory);
 bool		identify_system(PGconn *repl_conn, t_system_identification *identification);
 bool		repmgrd_set_local_node_id(PGconn *conn, int local_node_id);
 int			repmgrd_get_local_node_id(PGconn *conn);
+BackupState	server_in_exclusive_backup_mode(PGconn *conn);
 
 /* extension functions */
 ExtensionStatus get_repmgr_extension_status(PGconn *conn);
@@ -468,7 +475,7 @@ int			wait_connection_availability(PGconn *conn, long long timeout);
 /* node availability functions */
 bool		is_server_available(const char *conninfo);
 bool		is_server_available_params(t_conninfo_param_list *param_list);
-void		connection_ping(PGconn *conn);
+ExecStatusType	connection_ping(PGconn *conn);
 
 /* monitoring functions  */
 void
@@ -507,12 +514,14 @@ void		get_node_replication_stats(PGconn *conn, int server_version_num, t_node_in
 bool		is_downstream_node_attached(PGconn *conn, char *node_name);
 
 /* BDR functions */
+int			get_bdr_version_num(void);
 void		get_all_bdr_node_records(PGconn *conn, BdrNodeInfoList *node_list);
 RecordStatus get_bdr_node_record_by_name(PGconn *conn, const char *node_name, t_bdr_node_info *node_info);
 bool		is_bdr_db(PGconn *conn, PQExpBufferData *output);
 bool		is_bdr_db_quiet(PGconn *conn);
 bool		is_active_bdr_node(PGconn *conn, const char *node_name);
 bool		is_bdr_repmgr(PGconn *conn);
+char	   *get_default_bdr_replication_set(PGconn *conn);
 bool		is_table_in_bdr_replication_set(PGconn *conn, const char *tablename, const char *set);
 bool		add_table_to_bdr_replication_set(PGconn *conn, const char *tablename, const char *set);
 void		add_extension_tables_to_bdr_replication_set(PGconn *conn);

doc/appendix-faq.sgml

@@ -108,6 +108,14 @@
      is not possible, contact your vendor for assistance.
    </para>
   </sect2>
+
+  <sect2 id="faq-old-packages">
+   <title>How can I obtain old versions of &repmgr; packages?</title>
+   <para>
+     See appendix <xref linkend="packages-old-versions"> for details.
+   </para>
+  </sect2>
+
  </sect1>
 
  <sect1 id="faq-repmgr" xreflabel="repmgr">
@@ -239,11 +247,22 @@
      Under some circumstances event notifications can be generated for servers
      which have not yet been registered; it's also useful to retain a record
      of events which includes servers removed from the replication cluster
-     which no longer have an entry in the <literal>repmrg.nodes</literal> table.
+     which no longer have an entry in the <literal>repmgr.nodes</literal> table.
    </para>
   </sect2>
 
-
+  <sect2 id="faq-repmgr-recovery-conf-quoted-values" xreflabel="Quoted values in recovery.conf">
+    <title>Why are some values in <filename>recovery.conf</filename> surrounded by pairs of single quotes?</title>
+    <para>
+      This is to ensure that user-supplied values which are written as parameter values in <filename>recovery.conf</filename>
+      are escaped correctly and do not cause errors when <filename>recovery.conf</filename> is parsed.
+    </para>
+    <para>
+      The escaping is performed by an internal PostgreSQL routine, which leaves strings consisting
+      of digits and alphabetical characters only as-is, but wraps everything else in pairs of single quotes,
+      even if the string does not contain any characters which need escaping.
+    </para>
+  </sect2>
 
 
  </sect1>
@@ -255,7 +274,7 @@
   <sect2 id="faq-repmgrd-prevent-promotion" xreflabel="Prevent standby from being promoted to primary">
    <title>How can I prevent a node from ever being promoted to primary?</title>
    <para>
-    In `repmgr.conf`, set its priority to a value of 0 or less; apply the changed setting with
+     In <filename>repmgr.conf</filename>, set its priority to a value of <literal>0</literal>; apply the changed setting with
     <command><link linkend="repmgr-standby-register">repmgr standby register --force</link></command>.
    </para>
    <para>
@@ -303,5 +322,36 @@
    </para>
   </sect2>
 
+  <sect2 id="faq-repmgrd-pg-bindir" xreflabel="repmgrd does not apply pg_bindir to promote_command or follow_command">
+    <title>
+      <application>repmgrd</application> ignores pg_bindir when executing <varname>promote_command</varname> or <varname>follow_command</varname>
+    </title>
+    <para>
+      <varname>promote_command</varname> or <varname>follow_command</varname> can be user-defined scripts,
+      so &repmgr; will not apply <option>pg_bindir</option> even if excuting &repmgr;. Always provide the full
+      path; see <xref linkend="repmgrd-automatic-failover-configuration"> for more details.
+    </para>
+  </sect2>
+
+  <sect2 id="faq-repmgrd-startup-no-upstream" xreflabel="repmgrd does not start if upstream node is not running">
+    <title>
+      <application>repmgrd</application> aborts startup with the error "<literal>upstream node must be running before repmgrd can start</literal>"
+    </title>
+    <para>
+      <application>repmgrd</application> does this to avoid starting up on a replication cluster
+      which is not in a healthy state. If the upstream is unavailable, <application>repmgrd</application>
+      may initiate a failover immediately after starting up, which could have unintended side-effects,
+      particularly if <application>repmgrd</application> is not running on other nodes.
+    </para>
+    <para>
+      In particular, it's possible that the node's local copy of the <literal>repmgr.nodes</literal> copy
+      is out-of-date, which may lead to incorrect failover behaviour.
+    </para>
+    <para>
+      The onus is therefore on the adminstrator to manually set the cluster to a stable, healthy state before
+      starting <application>repmgrd</application>.
+    </para>
+  </sect2>
+
  </sect1>
 </appendix>

doc/appendix-packages.sgml

@@ -53,11 +53,11 @@
           <tbody>
             <row>
               <entry>Repository URL:</entry>
-              <entry><ulink url="https://rpm.2ndquadrant.com/">https://rpm.2ndquadrant.com/</ulink></entry>
+              <entry><ulink url="https://dl.2ndquadrant.com/">https://dl.2ndquadrant.com/</ulink></entry>
             </row>
             <row>
               <entry>Repository documentation:</entry>
-              <entry><ulink url="https://repmgr.org/docs/4.0/installation-packages.html#INSTALLATION-PACKAGES-REDHAT-2NDQ">https://repmgr.org/docs/4.0/installation-packages.html#INSTALLATION-PACKAGES-REDHAT-2NDQ</ulink></entry>
+              <entry><ulink url="https://repmgr.org/docs/4.1/installation-packages.html#INSTALLATION-PACKAGES-REDHAT-2NDQ">https://repmgr.org/docs/4.1/installation-packages.html#INSTALLATION-PACKAGES-REDHAT-2NDQ</ulink></entry>
             </row>
           </tbody>
         </tgroup>
@@ -253,6 +253,23 @@
       </para>
 
 
+      <table id="apt-2ndquadrant-repository">
+        <title>2ndQuadrant public repository</title>
+        <tgroup cols="2">
+          <tbody>
+            <row>
+              <entry>Repository URL:</entry>
+              <entry><ulink url="https://dl.2ndquadrant.com/">https://dl.2ndquadrant.com/</ulink></entry>
+            </row>
+            <row>
+              <entry>Repository documentation:</entry>
+              <entry><ulink url="https://repmgr.org/docs/4.1/installation-packages.html#INSTALLATION-PACKAGES-DEBIAN">https://repmgr.org/docs/4.1/installation-packages.html#INSTALLATION-PACKAGES-DEBIAN</ulink></entry>
+            </row>
+          </tbody>
+        </tgroup>
+      </table>
+
+
       <table id="apt-repository">
         <title>PostgreSQL Community APT repository (PGDG)</title>
         <tgroup cols="2">
@@ -364,4 +381,169 @@
    </sect2>
 
   </sect1>
+
+  <sect1 id="packages-snapshot" xreflabel="Snapshot packages">
+    <title>Snapshot packages</title>
+    <indexterm>
+      <primary>snapshot packages</primary>
+    </indexterm>
+    <indexterm>
+      <primary>packages</primary>
+      <secondary>snaphots</secondary>
+    </indexterm>
+
+    <para>
+      For testing new features and bug fixes, from time to time 2ndQuadrant provides
+      so-called &quot;snapshot packages&quot; via its public repository. These packages
+      are built from the &repmgr; source at a particular point in time, and are not formal
+      releases.
+    </para>
+    <note>
+      <para>
+        We do not recommend installing these packages in a production environment
+        unless specifically advised.
+      </para>
+    </note>
+    <para>
+      To install a snapshot package, it's necessary to install the 2ndQuadrant public snapshot repository,
+      following the instructions here: <ulink url="https://dl.2ndquadrant.com/default/release/site/">https://dl.2ndquadrant.com/default/release/site/</ulink> but replace <literal>release</literal> with <literal>snapshot</literal>
+      in the appropriate URL.
+    </para>
+    <para>
+      For example, to install the snapshot RPM repository for PostgreSQL 9.6, execute (as <literal>root</literal>):
+      <programlisting>
+curl https://dl.2ndquadrant.com/default/snapshot/get/9.6/rpm | bash</programlisting>
+
+      or as a normal user with root sudo access:
+      <programlisting>
+curl https://dl.2ndquadrant.com/default/snapshot/get/9.6/rpm | sudo bash</programlisting>
+    </para>
+    <para>
+      Alternatively you can browse the repository here:
+      <ulink url="https://dl.2ndquadrant.com/default/snapshot/browse/">https://dl.2ndquadrant.com/default/snapshot/browse/</ulink>.
+    </para>
+    <para>
+      Once the repository is installed, installing or updating &repmgr; will result in the latest snapshot
+      package being installed.
+    </para>
+    <para>
+      The package name will be formatted like this:
+      <programlisting>
+repmgr96-4.1.1-0.0git320.g5113ab0.1.el7.x86_64.rpm</programlisting>
+      containg the snapshot build number (here: <literal>320</literal>) and the hash
+      of the <application>git</application> commit it was built from (here: <literal>g5113ab0</literal>).
+    </para>
+
+    <para>
+      Note that the next formal release (in the above example <literal>4.1.1</literal>), once available,
+      will install in place of any snapshot builds.
+    </para>
+
+  </sect1>
+
+  <sect1 id="packages-old-versions" xreflabel="Installing old package versions">
+    <title>Installing old package versions</title>
+    <indexterm>
+      <primary>old packages</primary>
+    </indexterm>
+    <indexterm>
+      <primary>packages</primary>
+      <secondary>old versions</secondary>
+    </indexterm>
+
+    <sect2 id="packages-old-versions-debian" xreflabel="old Debian package versions">
+      <title>Debian/Ubuntu</title>
+      <para>
+        An archive of old packages (<literal>3.3.2</literal> and later) for Debian/Ubuntu-based systems is available here:
+        <ulink url="http://atalia.postgresql.org/morgue/r/repmgr/">http://atalia.postgresql.org/morgue/r/repmgr/</ulink>
+      </para>
+    </sect2>
+
+    <sect2 id="packages-old-versions-rhel-centos" xreflabel="old RHEL/CentOS package versions">
+      <title>RHEL/CentOS</title>
+      <para>
+        Old RPM packages (<literal>3.2</literal> and later) can be retrieved from the
+        (deprecated) 2ndQuadrant repository at
+        <ulink url="http://packages.2ndquadrant.com/">http://packages.2ndquadrant.com/</ulink>
+        by installing the appropriate repository RPM:
+      </para>
+
+      <itemizedlist spacing="compact" mark="bullet">
+
+        <listitem>
+          <simpara>
+            <ulink url="http://packages.2ndquadrant.com/repmgr/yum-repo-rpms/repmgr-fedora-1.0-1.noarch.rpm">http://packages.2ndquadrant.com/repmgr/yum-repo-rpms/repmgr-fedora-1.0-1.noarch.rpm</ulink>
+          </simpara>
+        </listitem>
+
+        <listitem>
+          <simpara>
+            <ulink url="http://packages.2ndquadrant.com/repmgr/yum-repo-rpms/repmgr-rhel-1.0-1.noarch.rpm">http://packages.2ndquadrant.com/repmgr/yum-repo-rpms/repmgr-rhel-1.0-1.noarch.rpm</ulink>
+          </simpara>
+        </listitem>
+
+      </itemizedlist>
+
+      <para>
+        Old versions can be located with e.g.:
+        <programlisting>
+          yum --showduplicates list repmgr96</programlisting>
+        (substitute the appropriate package name; see <xref linkend="packages-centos">) and installed with:
+        <programlisting>
+          yum install {package_name}-{version}</programlisting>
+        where <literal>{package_name}</literal> is the base package name (e.g. <literal>repmgr96</literal>)
+        and <literal>{version}</literal> is the version listed by the
+        <command> yum --showduplicates list ...</command> command, e.g. <literal>4.0.6-1.rhel6</literal>.
+      </para>
+      <para>For example:
+        <programlisting>
+          yum install repmgr96-4.0.6-1.rhel6</programlisting>
+      </para>
+
+    </sect2>
+  </sect1>
+
+
+  <sect1 id="packages-packager-info" xreflabel="Information for packagers">
+    <title>Information for packagers</title>
+    <indexterm>
+      <primary>packages</primary>
+      <secondary>information for packagers</secondary>
+    </indexterm>
+    <para>
+      We recommend patching the following parameters when
+      building the package as built-in default values for user convenience.
+      These values can nevertheless be overridden by the user, if desired.
+    </para>
+    <itemizedlist>
+      <listitem>
+        <para>
+          Configuration file location: the default configuration file location
+          can be hard-coded by patching <varname>package_conf_file</varname>
+          in <filename>configfile.c</filename>:
+          <programlisting>
+		/* packagers: if feasible, patch configuration file path into "package_conf_file" */
+		char		package_conf_file[MAXPGPATH] = "";</programlisting>
+        </para>
+        <para>
+          See also: <xref linkend="configuration-file">
+        </para>
+      </listitem>
+
+      <listitem>
+        <para>
+          PID file location: the default <application>repmgrd</application> PID file
+          location can be hard-coded by patching <varname>package_pid_file</varname>
+          in <filename>repmgrd.c</filename>:
+          <programlisting>
+		/* packagers: if feasible, patch PID file path into "package_pid_file" */
+		char		package_pid_file[MAXPGPATH] = "";</programlisting>
+        </para>
+        <para>
+          See also: <xref linkend="repmgrd-pid-file">
+        </para>
+      </listitem>
+
+    </itemizedlist>
+  </sect1>
 </appendix>

doc/appendix-release-notes.sgml

@@ -15,9 +15,373 @@
     See also: <xref linkend="upgrading-repmgr">
   </para>
 
+  <sect1 id="release-4.1.1">
+    <title>Release 4.1.1</title>
+    <para><emphasis>Wed September 5, 2018</emphasis></para>
+    <para>
+      repmgr 4.1.1 contains a number of usability enhancements and bug fixes.
+    </para>
+    <para>
+	  We recommend upgrading to this version as soon as possible.
+	  This release can be installed as a simple package upgrade from repmgr 4.0 ~ 4.1.0;
+      <application>repmgrd</application> (if running) should be restarted.
+      See <xref linkend="upgrading-repmgr"> for more details.
+	</para>
+
+    <sect2>
+      <title>repmgr enhancements</title>
+      <para>
+        <itemizedlist>
+
+          <listitem>
+            <para>
+              <command><link linkend="repmgr-standby-switchover">repmgr standby switchover --dry-run</link></command>
+              no longer copies external configuration files to test they can be copied; this avoids making
+              any changes to the target system. (GitHub #491).
+            </para>
+          </listitem>
+
+          <listitem>
+            <para>
+              <command><link linkend="repmgr-cluster-cleanup">repmgr cluster cleanup</link></command>:
+              add <literal>cluster_cleanup</literal> event. (GitHub #492).
+            </para>
+          </listitem>
+
+          <listitem>
+            <para>
+              <command><link linkend="repmgr-standby-switchover">repmgr standby switchover</link></command>:
+              improve detection of free walsenders. (GitHub #495).
+            </para>
+          </listitem>
+
+          <listitem>
+            <para>
+              Improve messages emitted during
+              <command><link linkend="repmgr-standby-promote">repmgr standby promote</link></command>.
+            </para>
+          </listitem>
+
+        </itemizedlist>
+      </para>
+   </sect2>
+
+
+    <sect2>
+      <title>repmgrd enhancements</title>
+      <para>
+        <itemizedlist>
+
+          <listitem>
+            <para>
+              Always reopen the log file after
+              receiving <literal>SIGHUP</literal>. Previously this only happened if
+              a configuration file change was detected.
+              (GitHub #485).
+            </para>
+          </listitem>
+
+          <listitem>
+            <para>
+              Report version number <emphasis>after</emphasis>
+              logger initialisation. (GitHub #487).
+            </para>
+          </listitem>
+
+          <listitem>
+            <para>
+              Improve cascaded standby failover handling. (GitHub #480).
+            </para>
+          </listitem>
+
+          <listitem>
+            <para>
+              Improve reconnection handling after brief network outages; if
+              monitoring data being collected, this could lead to orphaned
+              sessions on the primary. (GitHub #480).
+            </para>
+          </listitem>
+
+          <listitem>
+            <para>
+              Check <varname>promote_command</varname> and <varname>follow_command</varname>
+              are defined when reloading configuration. These were checked on startup but
+              not reload by <application>repmgrd</application>, which made it possible to
+              make <application>repmgrd</application> with invalid values. It's unlikely
+              anyone would want to do this, but we should make it impossible anyway.
+              (GitHub #486).
+            </para>
+          </listitem>
+
+        </itemizedlist>
+      </para>
+    </sect2>
+
+    <sect2>
+      <title>Other</title>
+      <para>
+        <itemizedlist>
+
+          <listitem>
+            <para>
+              Text of any failed queries will now be logged as <literal>ERROR</literal> to assist
+              logfile analysis at log levels higher than <literal>DEBUG</literal>.
+              (GitHub #498).
+            </para>
+          </listitem>
+        </itemizedlist>
+      </para>
+    </sect2>
+
+    <sect2>
+      <title>Bug fixes</title>
+      <para>
+        <itemizedlist>
+
+          <listitem>
+            <para>
+              <command><link linkend="repmgr-node-rejoin">repmgr node rejoin</link></command>:
+              remove new upstream's replication slot if it still exists on the rejoined
+              standby. (GitHub #499).
+            </para>
+          </listitem>
+
+          <listitem>
+            <para>
+              <application>repmgrd</application>: fix startup on witness node when local data is stale. (GitHub #488, #489).
+            </para>
+          </listitem>
+
+          <listitem>
+            <para>
+              Truncate version string reported by PostgreSQL if necessary; some
+              distributions insert additional detail after the actual version.
+              (GitHub #490).
+            </para>
+          </listitem>
+
+
+        </itemizedlist>
+      </para>
+    </sect2>
+
+
+  </sect1>
+
+
+
+  <sect1 id="release-4.1.0">
+    <title>Release 4.1.0</title>
+    <para><emphasis>Tue July 31, 2018</emphasis></para>
+    <para>
+      &repmgr; 4.1.0 introduces some changes to <application>repmgrd</application>
+      behaviour and some additional configuration parameters.
+    </para>
+    <para>
+      This release can be installed as a simple package upgrade from repmgr 4.0 ~ 4.0.6.
+      The following post-upgrade steps must be carried out:
+
+       <itemizedlist>
+          <listitem>
+            <para>
+              Execute <command>ALTER EXTENSION repmgr UPDATE</command>
+              on the primary server in the database where &repmgr; is installed.
+            </para>
+          </listitem>
+          <listitem>
+            <para>
+              <application>repmgrd</application> must be restarted on all nodes where it is running.
+            </para>
+          </listitem>
+
+       </itemizedlist>
+
+       A restart of the PostgreSQL server is <emphasis>not</emphasis> required
+       for this release (unless upgrading from repmgr 3.x).
+    </para>
+    <para>
+       See <xref linkend="upgrading-repmgr-extension"> for more details.
+    </para>
+
+    <para>
+      Configuration changes are backwards-compatible and no changes to
+      <filename>repmgr.conf</filename> are required. However users should
+      review the changes listed below.
+    </para>
+
+    <note>
+      <para>
+        <emphasis>Repository changes</emphasis>
+      </para>
+      <para>
+        Coinciding with this release, the 2ndQuadrant repository structure has changed.
+        See section <xref linkend="installation-packages"> for details, particularly
+        if you are using a RPM-based system.
+      </para>
+    </note>
+
+    <sect2>
+      <title>Configuration file changes</title>
+
+      <para>
+        <itemizedlist>
+
+          <listitem>
+            <para>
+              Default for <xref linkend="repmgr-conf-log-level"> is now <option>INFO</option>.
+              This produces additional informative log output, without creating excessive additional
+              log file volume, and matches the setting assumed for examples in the documentation.
+              (GitHub #470).
+            </para>
+          </listitem>
+
+          <listitem>
+            <para>
+              <varname>recovery_min_apply_delay</varname> now accepts a minimum value
+              of <literal>zero</literal> (GitHub #448).
+            </para>
+          </listitem>
+
+        </itemizedlist>
+      </para>
+
+    </sect2>
+
+
+    <sect2>
+      <title>repmgr enhancements</title>
+      <para>
+        <itemizedlist>
+
+          <listitem>
+            <para>
+              <application>repmgr</application>: always exit with an error if an unrecognised
+              command line option is provided. This matches the behaviour of other PostgreSQL
+              utilities such as <application>psql</application>. (GitHub #464).
+            </para>
+          </listitem>
+
+          <listitem>
+            <para>
+              <application>repmgr</application>: add <option>-q/--quiet</option> option to suppress non-error
+              output. (GitHub #468).
+            </para>
+          </listitem>
+
+          <listitem>
+            <para>
+              <command><link linkend="repmgr-cluster-show">repmgr cluster show</link></command>,
+              <command><link linkend="repmgr-node-check">repmgr node check</link></command> and
+              <command><link linkend="repmgr-node-status">repmgr node status</link></command>
+              return non-zero exit code if node status issues detected. (GitHub #456).
+            </para>
+          </listitem>
+
+          <listitem>
+            <para>
+              Add <option>--csv</option> output option for
+              <command><link linkend="repmgr-cluster-event">repmgr cluster event</link></command>.
+              (GitHub #471).
+            </para>
+          </listitem>
+
+          <listitem>
+            <para>
+              <command><link linkend="repmgr-witness-unregister">repmgr witness unregister</link></command>
+			  can be run on any node, by providing the ID of the witness node with <option>--node-id</option>.
+			  (GitHub #472).
+			</para>
+		  </listitem>
+
+          <listitem>
+            <para>
+              <command><link linkend="repmgr-standby-switchover">repmgr standby switchover</link></command>
+              will refuse to run if an exclusive backup is taking place on the current primary.
+			  (GitHub #476).
+			</para>
+		  </listitem>
+
+        </itemizedlist>
+      </para>
+    </sect2>
+
+    <sect2>
+      <title>repmgrd enhancements</title>
+      <para>
+        <itemizedlist>
+
+          <listitem>
+            <para>
+              <application>repmgrd</application>: create a PID file by default
+              (GitHub #457). For details, see <xref linkend="repmgrd-pid-file">.
+            </para>
+          </listitem>
+
+          <listitem>
+            <para>
+              <application>repmgrd</application>: daemonize process by default.
+			  In case, for whatever reason, the user does not wish to daemonize the
+			  process, provide <option>--daemonize=false</option>.
+			  (GitHub #458).
+            </para>
+          </listitem>
+
+
+        </itemizedlist>
+      </para>
+    </sect2>
+
+
+    <sect2>
+      <title>Bug fixes</title>
+      <para>
+        <itemizedlist>
+
+          <listitem>
+            <para>
+              <command><link linkend="repmgr-standby-register">repmgr standby register --wait-sync</link></command>:
+              fix behaviour when no timeout provided.
+            </para>
+          </listitem>
+
+          <listitem>
+            <para>
+              <command><link linkend="repmgr-cluster-cleanup">repmgr cluster cleanup</link></command>:
+			  add missing help options. (GitHub #461/#462).
+			</para>
+		  </listitem>
+
+          <listitem>
+            <para>
+			  Ensure witness node follows new primary after switchover. (GitHub #453).
+			</para>
+		  </listitem>
+
+          <listitem>
+            <para>
+              <command><link linkend="repmgr-node-check">repmgr node check</link></command> and
+              <command><link linkend="repmgr-node-status">repmgr node status</link></command>:
+			  fix witness node handling. (GitHub #451).
+			</para>
+		  </listitem>
+
+          <listitem>
+            <para>
+              When using <command><link linkend="repmgr-standby-clone">repmgr standby clone</link></command>
+              with <option>--recovery-conf-only</option> and replication slots, ensure
+              <varname>primary_slot_name</varname> is set correctly. (GitHub #474).
+            </para>
+          </listitem>
+
+        </itemizedlist>
+      </para>
+    </sect2>
+
+  </sect1>
+
   <sect1 id="release-4.0.6">
     <title>Release 4.0.6</title>
-    <para><emphasis>June 14, 2018</emphasis></para>
+    <para><emphasis>Thu June 14, 2018</emphasis></para>
     <para>
 	  &repmgr; 4.0.6 contains a number of bug fixes and usability enhancements.
     </para>
@@ -140,7 +504,7 @@
           <listitem>
             <para>
               Various documentation improvements, with particular emphasis on
-              the importance of setting appropriate <link linkend="configuration-service-commands">service commands</link>
+              the importance of setting appropriate <link linkend="configuration-file-service-commands">service commands</link>
               instead of relying on <application>pg_ctl</application>.
             </para>
           </listitem>

doc/appendix-signatures.sgml

@@ -5,14 +5,14 @@
    <title>repmgr source code signing key</title>
    <para>
      The signing key ID used for <application>repmgr</application> source code bundles is:
-     <ulink url="http://packages.2ndquadrant.com/repmgr/SOURCE-GPG-KEY-repmgr">
+     <ulink url="https://repmgr.org/download/SOURCE-GPG-KEY-repmgr">
        <literal>0x297F1DCC</literal></ulink>.
    </para>
 
    <para>
      To download the <application>repmgr</application> source key to your computer:
      <programlisting>
-       curl -s http://packages.2ndquadrant.com/repmgr/SOURCE-GPG-KEY-repmgr | gpg --import
+       curl -s https://repmgr.org/download/SOURCE-GPG-KEY-repmgr | gpg --import
        gpg --fingerprint 0x297F1DCC
      </programlisting>
      then verify that the fingerprint is the expected value:

doc/configuration-file-log-settings.sgml

@@ -0,0 +1,107 @@
+<sect1 id="configuration-file-log-settings" xreflabel="log settings">
+  <indexterm>
+    <primary>repmgr.conf</primary>
+    <secondary>log settings</secondary>
+  </indexterm>
+  <indexterm>
+    <primary>log settings</primary>
+    <secondary>configuration in repmgr.conf</secondary>
+  </indexterm>
+  <title>Log settings</title>
+
+  <para>
+    By default, &repmgr; and <application>repmgrd</application> write log output to
+    <literal>STDERR</literal>. An alternative log destination can be specified
+    (either a file or <literal>syslog</literal>).
+  </para>
+
+  <note>
+    <para>
+      The &repmgr; application itself will continue to write log output to <literal>STDERR</literal>
+      even if another log destination is configured, as otherwise any output resulting from a command
+      line operation will "disappear" into the log.
+    </para>
+    <para>
+      This behaviour can be overriden with the command line option <option>--log-to-file</option>,
+      which will redirect all logging output to the configured log destination. This is recommended
+      when &repmgr; is executed by another application, particularly <application>repmgrd</application>,
+      to enable log output generated by the &repmgr; application to be stored for later reference.
+    </para>
+  </note>
+
+  <variablelist>
+
+   <varlistentry id="repmgr-conf-log-level" xreflabel="log_level">
+    <term><varname>log_level</varname> (<type>string</type>)
+     <indexterm>
+      <primary><varname>log_level</varname> configuration file parameter</primary>
+     </indexterm>
+    </term>
+    <listitem>
+     <para>
+       One of <option>DEBUG</option>, <option>INFO</option>, <option>NOTICE</option>,
+       <option>WARNING</option>, <option>ERROR</option>, <option>ALERT</option>, <option>CRIT</option>
+       or <option>EMERG</option>.
+     </para>
+     <para>
+       Default is <option>INFO</option>.
+     </para>
+     <para>
+       Note that <option>DEBUG</option> will produce a substantial amount of log output
+       and should not be enabled in normal use.
+     </para>
+    </listitem>
+   </varlistentry>
+
+   <varlistentry id="repmgr-conf-log-facility" xreflabel="log_facility">
+    <term><varname>log_facility</varname> (<type>string</type>)
+     <indexterm>
+      <primary><varname>log_facility</varname> configuration file parameter</primary>
+     </indexterm>
+    </term>
+    <listitem>
+     <para>
+       Logging facility: possible values are <option>STDERR</option> (default), or for
+       syslog integration, one of <option>LOCAL0</option>, <option>LOCAL1</option>, <option>...</option>,
+       <option>LOCAL7</option>, <option>USER</option>.
+     </para>
+    </listitem>
+   </varlistentry>
+
+   <varlistentry id="repmgr-conf-log-file" xreflabel="log_file">
+    <term><varname>log_file</varname> (<type>string</type>)
+     <indexterm>
+      <primary><varname>log_file</varname> configuration file parameter</primary>
+     </indexterm>
+    </term>
+    <listitem>
+     <para>
+       If <xref linkend="repmgr-conf-log-facility"> is set to <option>STDERR</option>, log output
+       can be redirected to the specified file.
+     </para>
+     <para>
+       See <xref linkend="repmgrd-log-rotation"> for information on configuring log rotation.
+     </para>
+    </listitem>
+   </varlistentry>
+
+   <varlistentry id="repmgr-conf-log-status-interval" xreflabel="log_status_interval">
+    <term><varname>log_status_interval</varname> (<type>integer</type>)
+     <indexterm>
+      <primary><varname>log_status_interval</varname> configuration file parameter</primary>
+     </indexterm>
+    </term>
+    <listitem>
+     <para>
+       This setting causes <application>repmgrd</application> to emit a status log
+       line at the specified interval (in seconds, default <literal>300</literal>)
+       describing <application>repmgrd</application>'s current state, e.g.:
+     </para>
+     <programlisting>
+      [2018-07-12 00:47:32] [INFO] monitoring connection to upstream node "node1" (node ID: 1)</programlisting>
+    </listitem>
+   </varlistentry>
+
+  </variablelist>
+
+</sect1>

doc/configuration-file-required-settings.sgml

@@ -1,10 +1,10 @@
-<sect1 id="configuration-file-settings" xreflabel="configuration file settings">
+<sect1 id="configuration-file-settings" xreflabel="required configuration file settings">
   <indexterm>
     <primary>repmgr.conf</primary>
-    <secondary>basic settings</secondary>
+    <secondary>required settings</secondary>
   </indexterm>
 
- <title>Basic configuration file settings</title>
+ <title>Required configuration file settings</title>
  <para>
    Each <filename>repmgr.conf</filename> file must contain the following parameters:
  </para>

doc/configuration-file-service-commands.sgml

@@ -1,4 +1,4 @@
-<sect1 id="configuration-service-commands" xreflabel="service command settings">
+<sect1 id="configuration-file-service-commands" xreflabel="service command settings">
   <indexterm>
     <primary>repmgr.conf</primary>
     <secondary>service command settings</secondary>
@@ -17,9 +17,9 @@
     <link linkend="repmgr-node-rejoin"><command>repmgr node rejoin</command></link>.
   </para>
   <para>
-    By default, &repmgr; will use PostgreSQL's <command>pg_ctl</command> to control the PostgreSQL
+    By default, &repmgr; will use PostgreSQL's <command>pg_ctl</command> utility to control the PostgreSQL
     server. However this can lead to various problems, particularly when PostgreSQL has been
-    installed from packages, and expecially so if <application>systemd</application> is in use.
+    installed from packages, and especially so if <application>systemd</application> is in use.
   </para>
 
 
@@ -47,6 +47,14 @@
     service_restart_command
     service_reload_command</programlisting>
   </para>
+
+  <note>
+    <para>
+      &repmgr; will not apply <option>pg_bindir</option> when executing any of these commands;
+      these can be user-defined scripts so must always be specified with the full path.
+    </para>
+  </note>
+
   <note>
     <para>
       It's also possible to specify a <varname>service_promote_command</varname>.
@@ -56,7 +64,7 @@
     </para>
     <para>
       If your packaging system does not provide such a command, it can be left empty,
-      and &repmgr; will generate the appropriate <command>pg_ctl ... promote</command> command.
+      and &repmgr; will generate the appropriate `pg_ctl ... promote` command.
     </para>
     <para>
       Do not confuse this with <varname>promote_command</varname>, which is used
@@ -64,7 +72,6 @@
     </para>
   </note>
 
-
   <para>
     To confirm which command &repmgr; will execute for each action, use
     <command>repmgr node service --list --action=...</command>, e.g.:
@@ -92,7 +99,7 @@
       Defaults:postgres !requiretty
       postgres ALL = NOPASSWD: /usr/bin/systemctl stop postgresql-9.6, \
         /usr/bin/systemctl start postgresql-9.6, \
-        /usr/bin/systemctl restart postgresql-9.6 \
+        /usr/bin/systemctl restart postgresql-9.6, \
         /usr/bin/systemctl reload postgresql-9.6</programlisting>
   </para>
 

doc/configuration.sgml

@@ -2,16 +2,17 @@
   <title>repmgr configuration</title>
 
   &configuration-file;
-  &configuration-file-settings;
+  &configuration-file-required-settings;
-  &configuration-service-commands;
+  &configuration-file-log-settings;
+  &configuration-file-service-commands;
 
-  <sect1 id="configuration-permissions" xreflabel="User permissions">
+  <sect1 id="configuration-permissions" xreflabel="Database user permissions">
     <indexterm>
       <primary>configuration</primary>
-      <secondary>user permissions</secondary>
+      <secondary>database user permissions</secondary>
     </indexterm>
 
-    <title>repmgr user permissions</title>
+    <title>repmgr database user permissions</title>
     <para>
       &repmgr; will create an extension database containing objects
       for administering &repmgr; metadata. The user defined in the <varname>conninfo</varname>

doc/configuring-witness-server.sgml

@@ -16,15 +16,22 @@
  <para>
    A typical use case for a witness server is a two-node streaming replication
    setup, where the primary and standby are in different locations (data centres).
-   By creating a witness server in the same location as the primary, if the primary
+   By creating a witness server in the same location (data centre) as the primary,
-   becomes unavailable  it's possible for the standby to decide whether it can
+   if the primary becomes unavailable it's possible for the standby to decide whether
-   promote itself without risking a "split brain" scenario: if it can't see either the
+   it can promote itself without risking a "split brain" scenario: if it can't see either the
    witness or the primary server, it's likely there's a network-level interruption
    and it should not promote itself. If it can seen the witness but not the primary,
    this proves there is no network interruption and the primary itself is unavailable,
    and it can therefore promote itself (and ideally take action to fence the
    former primary).
  </para>
+ <note>
+   <para>
+     <emphasis>Never</emphasis> install a witness server on the same physical host
+     as another node in the replication cluster managed by &repmgr; - it's essential
+     the witness is not affected in any way by failure of another node.
+   </para>
+ </note>
  <para>
    For more complex replication scenarios,e.g. with multiple datacentres, it may
    be preferable to use location-based failover, which ensures that only nodes

doc/event-notifications.sgml

@@ -147,34 +147,104 @@
  <para>
   By default, all notification types will be passed to the designated script;
   the notification types can be filtered to explicitly named ones using the
-  <varname>event_notifications</varname> parameter:
+  <varname>event_notifications</varname> parameter.
+ </para>
+
+ <para>
+   Events generated by the &repmgr; command:
 
   <itemizedlist spacing="compact" mark="bullet">
 
    <listitem>
-    <simpara><literal>primary_register</literal></simpara>
+     <simpara><literal><link linkend="repmgr-primary-register-events">cluster_created</link></literal></simpara>
    </listitem>
    <listitem>
-    <simpara><literal>primary_unregister</literal></simpara>
+     <simpara><literal><link linkend="repmgr-primary-register-events">primary_register</link></literal></simpara>
    </listitem>
    <listitem>
-    <simpara><literal>standby_register</literal></simpara>
+     <simpara><literal><link linkend="repmgr-primary-unregister-events">primary_unregister</link></literal></simpara>
+   </listitem>
+
+   <listitem>
+    <simpara><literal><link linkend="repmgr-standby-clone-events">standby_clone</link></literal></simpara>
    </listitem>
    <listitem>
-    <simpara><literal>standby_register_sync</literal></simpara>
+    <simpara><literal><link linkend="repmgr-standby-register-events">standby_register</link></literal></simpara>
    </listitem>
    <listitem>
-    <simpara><literal>standby_unregister</literal></simpara>
+    <simpara><literal><link linkend="repmgr-standby-register-events">standby_register_sync</link></literal></simpara>
    </listitem>
    <listitem>
-    <simpara><literal>standby_clone</literal></simpara>
+    <simpara><literal><link linkend="repmgr-standby-unregister-events">standby_unregister</link></literal></simpara>
+   </listitem>
+
+   <listitem>
+    <simpara><literal><link linkend="repmgr-standby-promote-events">standby_promote</link></literal></simpara>
    </listitem>
    <listitem>
-    <simpara><literal>standby_promote</literal></simpara>
+    <simpara><literal><link linkend="repmgr-standby-follow-events">standby_follow</link></literal></simpara>
    </listitem>
    <listitem>
-    <simpara><literal>standby_follow</literal></simpara>
+     <simpara><literal><link linkend="repmgr-standby-switchover-events">standby_switchover</link></literal></simpara>
    </listitem>
+
+   <listitem>
+     <simpara><literal><link linkend="repmgr-witness-register-events">witness_register</link></literal></simpara>
+   </listitem>
+   <listitem>
+    <simpara><literal><link linkend="repmgr-witness-unregister-events">witness_unregister</link></literal></simpara>
+   </listitem>
+   <listitem>
+    <simpara><literal><link linkend="repmgr-node-rejoin-events">node_rejoin</link></literal></simpara>
+   </listitem>
+   <listitem>
+    <simpara><literal><link linkend="repmgr-cluster-cleanup-events">cluster_cleanup</link></literal></simpara>
+   </listitem>
+
+  </itemizedlist>
+ </para>
+
+ <para>
+   Events generated by <application>repmgrd</application> (streaming replication mode):
+
+   <itemizedlist spacing="compact" mark="bullet">
+   <listitem>
+    <simpara><literal>repmgrd_start</literal></simpara>
+   </listitem>
+   <listitem>
+    <simpara><literal>repmgrd_shutdown</literal></simpara>
+   </listitem>
+   <listitem>
+    <simpara><literal>repmgrd_reload</literal></simpara>
+   </listitem>
+   <listitem>
+    <simpara><literal>repmgrd_failover_promote</literal></simpara>
+   </listitem>
+   <listitem>
+    <simpara><literal>repmgrd_failover_follow</literal></simpara>
+   </listitem>
+   <listitem>
+    <simpara><literal>repmgrd_failover_aborted</literal></simpara>
+   </listitem>
+   <listitem>
+    <simpara><literal>repmgrd_standby_reconnect</literal></simpara>
+   </listitem>
+   <listitem>
+    <simpara><literal>repmgrd_promote_error</literal></simpara>
+   </listitem>
+   <listitem>
+    <simpara><literal>repmgrd_local_disconnect</literal></simpara>
+   </listitem>
+   <listitem>
+    <simpara><literal>repmgrd_local_reconnect</literal></simpara>
+   </listitem>
+   <listitem>
+    <simpara><literal>repmgrd_upstream_disconnect</literal></simpara>
+   </listitem>
+   <listitem>
+    <simpara><literal>repmgrd_upstream_reconnect</literal></simpara>
+   </listitem>
+
    <listitem>
     <simpara><literal>standby_disconnect_manual</literal></simpara>
    </listitem>
@@ -184,39 +254,13 @@
    <listitem>
     <simpara><literal>standby_recovery</literal></simpara>
    </listitem>
-   <listitem>
+
-    <simpara><literal>witness_register</literal></simpara>
+   </itemizedlist>
-   </listitem>
+ </para>
-   <listitem>
+
-    <simpara><literal>witness_unregister</literal></simpara>
+  <para>
-   </listitem>
+   Events generated by <application>repmgrd</application> (BDR mode):
-   <listitem>
+   <itemizedlist spacing="compact" mark="bullet">
-    <simpara><literal>node_rejoin</literal></simpara>
-   </listitem>
-   <listitem>
-    <simpara><literal>repmgrd_start</literal></simpara>
-   </listitem>
-   <listitem>
-    <simpara><literal>repmgrd_shutdown</literal></simpara>
-   </listitem>
-   <listitem>
-    <simpara><literal>repmgrd_failover_promote</literal></simpara>
-   </listitem>
-   <listitem>
-    <simpara><literal>repmgrd_failover_follow</literal></simpara>
-   </listitem>
-   <listitem>
-     <simpara><literal>repmgrd_failover_aborted</literal></simpara>
-   </listitem>
-   <listitem>
-    <simpara><literal>repmgrd_upstream_disconnect</literal></simpara>
-   </listitem>
-   <listitem>
-    <simpara><literal>repmgrd_upstream_reconnect</literal></simpara>
-   </listitem>
-   <listitem>
-    <simpara><literal>repmgrd_promote_error</literal></simpara>
-   </listitem>
    <listitem>
     <simpara><literal>bdr_failover</literal></simpara>
    </listitem>

doc/filelist.sgml

@@ -38,8 +38,9 @@
 <!ENTITY quickstart      SYSTEM "quickstart.sgml">
 <!ENTITY configuration      SYSTEM "configuration.sgml">
 <!ENTITY configuration-file      SYSTEM "configuration-file.sgml">
-<!ENTITY configuration-file-settings      SYSTEM "configuration-file-settings.sgml">
+<!ENTITY configuration-file-required-settings      SYSTEM "configuration-file-required-settings.sgml">
-<!ENTITY configuration-service-commands   SYSTEM "configuration-service-commands.sgml">
+<!ENTITY configuration-file-log-settings      SYSTEM "configuration-file-log-settings.sgml">
+<!ENTITY configuration-file-service-commands   SYSTEM "configuration-file-service-commands.sgml">
 <!ENTITY cloning-standbys  SYSTEM "cloning-standbys.sgml">
 <!ENTITY promoting-standby  SYSTEM "promoting-standby.sgml">
 <!ENTITY follow-new-primary  SYSTEM "follow-new-primary.sgml">

doc/install-packages.sgml

@@ -16,7 +16,7 @@
   <para>
 	&repmgr; RPM packages for RedHat/CentOS variants and Fedora are available from the
 	<ulink url="https://2ndquadrant.com">2ndQuadrant</ulink>
-	<ulink url="https://rpm.2ndquadrant.com/">public RPM repository</ulink>; see following
+	<ulink url="https://dl.2ndquadrant.com/">public repository</ulink>; see following
 	section for details.
   </para>
   <para>
@@ -38,7 +38,7 @@
 
   <para>
     For more information on the package contents, including details of installation
-    paths and relevant <link linkend="configuration-service-commands">service commands</link>,
+    paths and relevant <link linkend="configuration-file-service-commands">service commands</link>,
     see the appendix section <xref linkend="packages-centos">.
   </para>
 
@@ -46,26 +46,15 @@
   <sect3 id="installation-packages-redhat-2ndq">
     <title>2ndQuadrant public RPM yum repository</title>
 
-	<note>
-	  <para>
-		<ulink url="https://2ndquadrant.com">2ndQuadrant</ulink> previously provided a dedicated
-        &repmgr; repository at
-        <ulink url="http://packages.2ndquadrant.com/repmgr/">http://packages.2ndquadrant.com/repmgr/</ulink>.
-		This repository will be deprecated in a future release as it is now replaced by
-		the <ulink url="https://rpm.2ndquadrant.com/">public RPM repository</ulink>
-		documented below.
-	  </para>
-	</note>
-
     <para>
-      Beginning with <ulink url="https://repmgr.org/docs/4.0/release-4.0.5.html">repmgr 4.0.5</ulink>,
+      Beginning with <ulink url="https://repmgr.org/docs/4.1/release-4.0.5.html">repmgr 4.0.5</ulink>,
       <ulink url="https://2ndquadrant.com/">2ndQuadrant</ulink> provides a dedicated <literal>yum</literal>
-	  <ulink url="https://rpm.2ndquadrant.com/">public RPM repository</ulink> for 2ndQuadrant software,
+	  <ulink url="https://dl.2ndquadrant.com/">public repository</ulink> for 2ndQuadrant software,
 	  including &repmgr;. We recommend using this for all future &repmgr; releases.
 	</para>
 	<para>
 	  General instructions for using this repository can be found on its
-	  <ulink url="https://rpm.2ndquadrant.com/">homepage</ulink>. Specific instructions
+	  <ulink url="https://dl.2ndquadrant.com/">homepage</ulink>. Specific instructions
 	  for installing &repmgr; follow below.
 	</para>
     <para>
@@ -75,20 +64,19 @@
 		<listitem>
 		  <para>
 			Locate the repository RPM for your PostgreSQL version from the list at:
-			<ulink url="https://rpm.2ndquadrant.com/">https://rpm.2ndquadrant.com/</ulink>
+			<ulink url="https://dl.2ndquadrant.com/">https://dl.2ndquadrant.com/</ulink>
 		  </para>
 		</listitem>
 
         <listitem>
           <para>
-            Install the repository RPM for your distribution and PostgreSQL version
+            Install the repository definition for your distribution and PostgreSQL version
 			(this enables the 2ndQuadrant repository as a source of &repmgr; packages).
 		  </para>
 		  <para>
 			For example, for PostgreSQL 10 on CentOS, execute:
 			<programlisting>
-sudo yum install https://rpm.2ndquadrant.com/site/content/2ndquadrant-repo-10-1-1.el7.noarch.rpm
+curl https://dl.2ndquadrant.com/default/release/get/10/rpm | sudo bash</programlisting>
-			</programlisting>
 		  </para>
 		  <para>
 			Verify that the repository is installed with:
@@ -96,8 +84,8 @@ sudo yum install https://rpm.2ndquadrant.com/site/content/2ndquadrant-repo-10-1-
 sudo yum repolist</programlisting>
 			The output should contain two entries like this:
 			<programlisting>
-2ndquadrant-repo-10/7/x86_64         2ndQuadrant packages for PG10 for rhel 7 - x86_64           1
+2ndquadrant-dl-default-release-pg10/7/x86_64        2ndQuadrant packages (PG10) for 7 - x86_64          4
-2ndquadrant-repo-10-debug/7/x86_64   2ndQuadrant packages for PG10 for rhel 7 - x86_64 - Debug   1</programlisting>
+2ndquadrant-dl-default-release-pg10-debug/7/x86_64  2ndQuadrant packages (PG10) for 7 - x86_64 - Debug  3</programlisting>
 		  </para>
 		</listitem>
 
@@ -167,7 +155,7 @@ $ yum install repmgr10</programlisting>
   </para>
   <para>
     For more information on the package contents, including details of installation
-    paths and relevant <link linkend="configuration-service-commands">service commands</link>,
+    paths and relevant <link linkend="configuration-file-service-commands">service commands</link>,
     see the appendix section <xref linkend="packages-debian-ubuntu">.
   </para>
 
@@ -177,52 +165,43 @@ $ yum install repmgr10</programlisting>
     <para>
       Beginning with <ulink url="https://repmgr.org/docs/4.0/release-4.0.5.html">repmgr 4.0.5</ulink>,
       <ulink url="https://2ndquadrant.com/">2ndQuadrant</ulink> provides a
-	  <ulink url="https://apt.2ndquadrant.com/">public apt repository</ulink> for 2ndQuadrant software,
+	  <ulink url="https://dl.2ndquadrant.com/">public apt repository</ulink> for 2ndQuadrant software,
 	  including &repmgr;.
 	</para>
 	<para>
 	  General instructions for using this repository can be found on its
-	  <ulink url="https://apt.2ndquadrant.com/">homepage</ulink>. Specific instructions
+	  <ulink url="https://dl.2ndquadrant.com/">homepage</ulink>. Specific instructions
 	  for installing &repmgr; follow below.
 	</para>
 
     <para>
+
       <emphasis>Installation</emphasis>
 
       <itemizedlist>
 
 		<listitem>
 		  <para>
-			If not already present, install the  <application>apt-transport-https</application> package:
+            Install the repository definition for your distribution and PostgreSQL version
-			<programlisting>
+			(this enables the 2ndQuadrant repository as a source of &repmgr; packages) by executing:
-sudo apt-get install apt-transport-https</programlisting>
+            <programlisting>
+curl https://dl.2ndquadrant.com/default/release/get/deb | sudo bash</programlisting>
 		  </para>
-		</listitem>
+          <note>
+            <para>
+              This will automatically install the following additional packages, if not already present:
+              <itemizedlist spacing="compact" mark="bullet">
+                <listitem>
+                  <simpara><literal>lsb-release</literal></simpara>
+                </listitem>
+                <listitem>
+                  <simpara><literal>apt-transport-https</literal></simpara>
+                </listitem>
+              </itemizedlist>
+            </para>
+          </note>
+        </listitem>
 
-		<listitem>
-		  <para>
-			Create <filename>/etc/apt/sources.list.d/2ndquadrant.list</filename> as follows:
-			<programlisting>
-sudo sh -c 'echo "deb https://apt.2ndquadrant.com/ $(lsb_release -cs)-2ndquadrant main" > /etc/apt/sources.list.d/2ndquadrant.list'</programlisting>
-		  </para>
-		</listitem>
-
-		<listitem>
-		  <para>
-			Install the 2ndQuadrant <ulink url="https://apt.2ndquadrant.com/site/keys/9904CD4BD6BAF0C3.asc">repository key</ulink>:
-			<programlisting>
-sudo apt-get install curl ca-certificates
-curl https://apt.2ndquadrant.com/site/keys/9904CD4BD6BAF0C3.asc | sudo apt-key add -</programlisting>
-		  </para>
-		</listitem>
-
-		<listitem>
-		  <para>
-			Update the package list
-			<programlisting>
- sudo apt-get update</programlisting>
-		  </para>
-		</listitem>
 
 		<listitem>
 		  <para>

doc/install-source.sgml

@@ -12,8 +12,8 @@
    To install &repmgr; the prerequisites for compiling
    &postgres; must be installed. These are described in &postgres;'s
    documentation
-   on <ulink url="https://www.postgresql.org/docs/current/install-requirements.html">build requirements</ulink>
+   on <ulink url="https://www.postgresql.org/docs/current/static/install-requirements.html">build requirements</ulink>
-   and <ulink url="https://www.postgresql.org/docs/current/docguide-toolsets.html">build requirements for documentation</ulink>.
+   and <ulink url="https://www.postgresql.org/docs/current/static/docguide-toolsets.html">build requirements for documentation</ulink>.
   </para>
 
   <para>

doc/quickstart.sgml

@@ -234,17 +234,34 @@
   <para>
    <filename>repmgr.conf</filename> should not be stored inside the PostgreSQL data directory,
    as it could be overwritten when setting up or reinitialising the PostgreSQL
-   server. See sections on <xref linkend="configuration-file"> and <xref linkend="configuration-file-settings">
+   server. See sections <xref linkend="configuration"> and <xref linkend="configuration-file">
    for further details about <filename>repmgr.conf</filename>.
   </para>
   <tip>
    <simpara>
     For Debian-based distributions we recommend explictly setting
-    <literal>pg_bindir</literal> to the directory where <command>pg_ctl</command> and other binaries
+    <option>pg_bindir</option> to the directory where <command>pg_ctl</command> and other binaries
     not in the standard path are located. For PostgreSQL 9.6 this would be <filename>/usr/lib/postgresql/9.6/bin/</filename>.
    </simpara>
   </tip>
 
+  <note>
+    <para>
+      &repmgr; only uses <option>pg_bindir</option> when it executes
+      PostgreSQL binaries directly.
+    </para>
+    <para>
+      For user-defined scripts such as <option>promote_command</option> and the
+      various <option>service_*_command</option>s, you <emphasis>must</emphasis>
+      always explicitly provide the full path to the binary or script being
+      executed, even if it is &repmgr; itself.
+    </para>
+    <para>
+      This is because these options can contain user-defined scripts in arbitrary
+      locations, so prepending <option>pg_bindir</option> may break them.
+    </para>
+  </note>
+
   <para>
    See the file
    <ulink url="https://raw.githubusercontent.com/2ndQuadrant/repmgr/master/repmgr.conf.sample">repmgr.conf.sample</>

doc/repmgr-cluster-cleanup.sgml

@@ -15,9 +15,14 @@
     <title>Description</title>
     <para>
       Purges monitoring history from the <literal>repmgr.monitoring_history</literal> table to
-      prevent excessive table growth. Use the <literal>-k/--keep-history</literal> to specify the
+      prevent excessive table growth.
-      number of days of monitoring history to retain. This command can be used
+    </para>
-      manually or as a cronjob.
+    <para>
+      By default <emphasis>all</emphasis> data will be removed; Use the <option>-k/--keep-history</option>
+      option to specify the number of days of monitoring history to retain.
+    </para>
+    <para>
+      This command can be executed manually or as a cronjob.
     </para>
   </refsect1>
 
@@ -38,4 +43,21 @@
       <filename>repmgr.conf</filename>.
     </para>
   </refsect1>
+
+  <refsect1 id="repmgr-cluster-cleanup-events">
+    <title>Event notifications</title>
+    <para>
+      A <literal>cluster_cleanup</literal> <link linkend="event-notifications">event notification</link> will be generated.
+    </para>
+  </refsect1>
+
+
+  <refsect1>
+    <title>See also</title>
+    <para>
+      For more details see the sections <xref linkend="repmgrd-monitoring"> and
+      <xref linkend="repmgrd-monitoring-configuration">.
+    </para>
+  </refsect1>
+
 </refentry>

doc/repmgr-cluster-crosscheck.sgml

@@ -56,7 +56,7 @@
       </varlistentry>
 
       <varlistentry>
-        <term><option>ERR_CLUSTER_CHECK (25)</option></term>
+        <term><option>ERR_NODE_STATUS (25)</option></term>
         <listitem>
           <para>
             One or more nodes could not be reached.

doc/repmgr-cluster-event.sgml

@@ -49,6 +49,22 @@
     </para>
   </refsect1>
 
+  <refsect1>
+    <title>Output format</title>
+    <para>
+      <itemizedlist spacing="compact" mark="bullet">
+
+        <listitem>
+          <simpara>
+            <literal>--csv</literal>: generate output in CSV format. Note that the <literal>Details</literal>
+            column will currently not be emitted in CSV format.
+          </simpara>
+        </listitem>
+
+      </itemizedlist>
+    </para>
+  </refsect1>
+
   <refsect1>
     <title>Example</title>
     <para>

doc/repmgr-cluster-matrix.sgml

@@ -116,7 +116,7 @@
       </varlistentry>
 
       <varlistentry>
-        <term><option>ERR_CLUSTER_CHECK (25)</option></term>
+        <term><option>ERR_NODE_STATUS (25)</option></term>
         <listitem>
           <para>
             One or more nodes could not be reached.

doc/repmgr-cluster-show.sgml

@@ -81,35 +81,91 @@
 
   <refsect1>
     <title>Options</title>
-    <para>
+
-      <command>repmgr cluster show</command> accepts an optional parameter <literal>--csv</literal>, which
+    <variablelist>
-      outputs the replication cluster's status in a simple CSV format, suitable for
+
-      parsing by scripts:
+      <varlistentry>
-      <programlisting>
+        <term><option>--csv</option></term>
+        <listitem>
+		  <para>
+			<command>repmgr cluster show</command> accepts an optional parameter <literal>--csv</literal>, which
+			outputs the replication cluster's status in a simple CSV format, suitable for
+			parsing by scripts:
+			<programlisting>
     $ repmgr -f /etc/repmgr.conf cluster show --csv
     1,-1,-1
     2,0,0
     3,0,1</programlisting>
-    </para>
+		  </para>
-    <para>
+		  <para>
-      The columns have following meanings:
+			The columns have following meanings:
-      <itemizedlist spacing="compact" mark="bullet">
+			<itemizedlist spacing="compact" mark="bullet">
-        <listitem>
+			  <listitem>
-          <simpara>
+				<simpara>
-            node ID
+				  node ID
-          </simpara>
+				</simpara>
-        </listitem>
+			  </listitem>
-        <listitem>
+			  <listitem>
-          <simpara>
+				<simpara>
             availability (0 = available, -1 = unavailable)
-          </simpara>
+				</simpara>
-        </listitem>
+			  </listitem>
+			  <listitem>
+				<simpara>
+				  recovery state (0 = not in recovery, 1 = in recovery, -1 = unknown)
+				</simpara>
+			  </listitem>
+			</itemizedlist>
+		  </para>
+		</listitem>
+	  </varlistentry>
+
+      <varlistentry>
+        <term><option>--verbose</option></term>
         <listitem>
-          <simpara>
+          <para>
-            recovery state (0 = not in recovery, 1 = in recovery, -1 = unknown)
+			Display the full text of any database connection error messages
-          </simpara>
+          </para>
         </listitem>
-      </itemizedlist>
+      </varlistentry>
+
+	</variablelist>
+
+  </refsect1>
+
+
+  <refsect1>
+    <title>Exit codes</title>
+    <para>
+      Following exit codes can be emitted by <command>repmgr cluster show</command>:
+    </para>
+    <variablelist>
+
+      <varlistentry>
+        <term><option>SUCCESS (0)</option></term>
+        <listitem>
+          <para>
+            No issues were detected.
+          </para>
+        </listitem>
+      </varlistentry>
+
+      <varlistentry>
+        <term><option>ERR_NODE_STATUS (25)</option></term>
+        <listitem>
+          <para>
+            One or more issues were detected.
+          </para>
+        </listitem>
+      </varlistentry>
+
+   </variablelist>
+  </refsect1>
+
+  <refsect1>
+    <title>See also</title>
+    <para>
+     <xref linkend="repmgr-node-status">, <xref linkend="repmgr-node-check">
     </para>
   </refsect1>
 

doc/repmgr-node-check.sgml

@@ -61,7 +61,9 @@
 
      <listitem>
       <simpara>
-        <literal>--archive-ready</literal>: checks for WAL files which have not yet been archived
+        <literal>--archive-ready</literal>: checks for WAL files which have not yet been archived,
+        and returns <literal>WARNING</literal> or <literal>CRITICAL</literal> if the number
+        exceeds <varname>archive_ready_warning</varname> or <varname>archive_ready_critical</varname> respectively.
       </simpara>
      </listitem>
 
@@ -77,6 +79,12 @@
       </simpara>
      </listitem>
 
+     <listitem>
+      <simpara>
+        <literal>--missing-slots</literal>: checks there are no missing replication slots
+      </simpara>
+     </listitem>
+
     </itemizedlist>
   </para>
   </refsect1>
@@ -101,4 +109,80 @@
       </itemizedlist>
     </para>
   </refsect1>
+
+  <refsect1>
+    <title>Exit codes</title>
+
+    <para>
+      When executing <command>repmgr node check</command> with one of the individual
+      checks listed above, &repmgr; will emit one of the following Nagios-style exit codes
+      (even if <literal>--nagios</literal> is not supplied):
+
+      <itemizedlist spacing="compact" mark="bullet">
+
+        <listitem>
+          <simpara>
+            <literal>0</literal>: OK
+          </simpara>
+        </listitem>
+
+        <listitem>
+          <simpara>
+            <literal>1</literal>: WARNING
+          </simpara>
+        </listitem>
+
+        <listitem>
+          <simpara>
+            <literal>2</literal>: ERROR
+          </simpara>
+        </listitem>
+
+        <listitem>
+          <simpara>
+            <literal>3</literal>: UNKNOWN
+          </simpara>
+        </listitem>
+
+      </itemizedlist>
+    </para>
+
+
+
+    <para>
+      Following exit codes can be emitted by <command>repmgr status check</command>
+      if no individual check was specified.
+    </para>
+    <variablelist>
+
+      <varlistentry>
+        <term><option>SUCCESS (0)</option></term>
+        <listitem>
+          <para>
+            No issues were detected.
+          </para>
+        </listitem>
+      </varlistentry>
+
+      <varlistentry>
+        <term><option>ERR_NODE_STATUS (25)</option></term>
+        <listitem>
+          <para>
+            One or more issues were detected.
+          </para>
+        </listitem>
+      </varlistentry>
+
+   </variablelist>
+  </refsect1>
+
+
+
+  <refsect1>
+    <title>See also</title>
+    <para>
+     <xref linkend="repmgr-node-status">, <xref linkend="repmgr-cluster-show">
+    </para>
+  </refsect1>
+
 </refentry>

doc/repmgr-node-rejoin.sgml

@@ -28,6 +28,10 @@
         If the node is running and needs to be attached to the current primary, use
         <xref linkend="repmgr-standby-follow">.
       </para>
+      <para>
+        Note <xref linkend="repmgr-standby-follow"> can only be used for standbys which have not diverged
+        from the rest of the cluster.
+      </para>
     </tip>
   </refsect1>
 
@@ -63,10 +67,10 @@
         <term><option>--force-rewind[=/path/to/pg_rewind]</option></term>
         <listitem>
           <para>
-            Execute <application>pg_rewind</application> if necessary.
+            Execute <application>pg_rewind</application>.
           </para>
           <para>
-            It is only necessary to provide the <application>pg_rewind</application>
+            It is only necessary to provide the <application>pg_rewind</application> path
             if using PostgreSQL 9.3 or 9.4, and <application>pg_rewind</application>
             is not installed in the PostgreSQL <filename>bin</filename> directory.
           </para>
@@ -115,8 +119,26 @@
 
     </variablelist>
   </refsect1>
-
   <refsect1>
+    <title>Configuration file settings</title>
+
+    <para>
+      <itemizedlist spacing="compact" mark="bullet">
+       <listitem>
+         <simpara>
+           <literal>node_rejoin_timeout</literal>:
+		   the maximum length of time (in seconds) to wait for
+		   the node to reconnect to the replication cluster (defaults to
+		   the value set in <literal>standby_reconnect_timeout</literal>,
+		   60 seconds).
+		 </simpara>
+	   </listitem>
+	  </itemizedlist>
+	</para>
+
+  </refsect1>
+
+  <refsect1 id="repmgr-node-rejoin-events">
     <title>Event notifications</title>
     <para>
       A <literal>node_rejoin</literal> <link linkend="event-notifications">event notification</link> will be generated.
@@ -171,7 +193,7 @@
     </note>
 
     <para>
-      To have <command>repmgr node rejoin</command> use <command>pg_rewind</command> if required,
+      To have <command>repmgr node rejoin</command> use <command>pg_rewind</command>,
       pass the command line option <literal>--force-rewind</literal>, which will tell &repmgr;
       to execute <command>pg_rewind</command> to ensure the node can be rejoined successfully.
     </para>
@@ -204,6 +226,15 @@
     INFO: pg_rewind would now be executed
     DETAIL: pg_rewind command is:
       pg_rewind -D '/var/lib/postgresql/data' --source-server='host=node1 dbname=repmgr user=repmgr'</programlisting>
+
+    <note>
+      <para>
+        If <option>--force-rewind</option> is used with the <option>--dry-run</option> option,
+        this checks the prerequisites for using <application>pg_rewind</application>, but cannot
+        predict the outcome of actually executing <application>pg_rewind</application>.
+      </para>
+    </note>
+
     <programlisting>
     $ repmgr node rejoin -f /etc/repmgr.conf -d 'host=node1 dbname=repmgr user=repmgr' \
          --force-rewind --config-files=postgresql.local.conf,postgresql.conf --verbose

doc/repmgr-node-status.sgml

@@ -52,10 +52,40 @@
     </para>
   </refsect1>
 
+  <refsect1>
+    <title>Exit codes</title>
+    <para>
+      Following exit codes can be emitted by <command>repmgr node status</command>:
+    </para>
+    <variablelist>
+
+      <varlistentry>
+        <term><option>SUCCESS (0)</option></term>
+        <listitem>
+          <para>
+            No issues were detected.
+          </para>
+        </listitem>
+      </varlistentry>
+
+      <varlistentry>
+        <term><option>ERR_NODE_STATUS (25)</option></term>
+        <listitem>
+          <para>
+            One or more issues were detected.
+          </para>
+        </listitem>
+      </varlistentry>
+
+   </variablelist>
+  </refsect1>
+
+
   <refsect1>
     <title>See also</title>
     <para>
-      See <xref linkend="repmgr-node-check"> to diagnose issues.
+      See <xref linkend="repmgr-node-check"> to diagnose issues and <xref linkend="repmgr-cluster-show">
+      for an overview of all nodes in the cluster.
     </para>
   </refsect1>
 </refentry>

doc/repmgr-primary-register.sgml

@@ -17,7 +17,7 @@
     <title>Description</title>
     <para>
       <command>repmgr primary register</command> registers a primary node in a
-      streaming replication cluster, and configures it for use with repmgr, including
+      streaming replication cluster, and configures it for use with &repmgr;, including
       installing the &repmgr; extension. This command needs to be executed before any
       standby nodes are registered.
     </para>
@@ -75,10 +75,18 @@
   </refsect1>
 
 
-  <refsect1>
+  <refsect1 id="repmgr-primary-register-events">
     <title>Event notifications</title>
     <para>
-      A <literal>primary_register</literal> <link linkend="event-notifications">event notification</link> will be generated.
+      Following <link linkend="event-notifications">event notifications</link> will be generated:
+      <itemizedlist spacing="compact" mark="bullet">
+        <listitem>
+          <simpara><literal>cluster_created</literal></simpara>
+        </listitem>
+        <listitem>
+          <simpara><literal>primary_register</literal></simpara>
+        </listitem>
+      </itemizedlist>
     </para>
   </refsect1>
 

doc/repmgr-primary-unregister.sgml

@@ -64,7 +64,7 @@
 
   </refsect1>
 
-  <refsect1>
+  <refsect1 id="repmgr-primary-unregister-events">
     <title>Event notifications</title>
     <para>
       A <literal>primary_unregister</literal> <link linkend="event-notifications">event notification</link> will be generated.

doc/repmgr-standby-clone.sgml

@@ -49,7 +49,7 @@
     not be copied by default. &repmgr; can copy these files, either to the same
     location on the standby server (provided appropriate directory and file permissions
     are available), or into the standby's data directory. This requires passwordless
-    SSH access to the primary server. Add the option <literal>--copy-external-config-files</literal>
+    SSH access to the primary server. Add the option <option>--copy-external-config-files</option>
     to the <command>repmgr standby clone</command> command; by default files will be copied to
     the same path as on the upstream server. Note that the user executing <command>repmgr</command>
     must have write access to those directories.
@@ -59,12 +59,29 @@
     <literal>--copy-external-config-files=pgdata</literal>, but note that
     any include directives in the copied files may need to be updated.
    </para>
+
+   <note>
+	 <para>
+	   When executing <command>repmgr standby clone</command> with the
+	   <option>--copy-external-config-files</option> aand <option>--dry-run</option>
+	   options, &repmgr; will check the SSH connection to the source node, but
+	   will not verify whether the files can actually be copied.
+	 </para>
+	 <para>
+	   During the actual clone operation, a check will be made before the database itself
+	   is cloned to determine whether the files can actually be copied; if any problems are
+	   encountered, the clone operation will be aborted, enabling the user to fix
+	   any issues before retrying the clone operation.
+	 </para>
+   </note>
+
    <tip>
     <simpara>
      For reliable configuration file management we recommend using a
      configuration management tool such as Ansible, Chef, Puppet or Salt.
     </simpara>
    </tip>
+
   </refsect1>
 
   <refsect1 id="repmgr-standby-clone-recovery-conf">
@@ -213,6 +230,15 @@
 
     <variablelist>
 
+      <varlistentry>
+        <term><option>-d, --dbname=CONNINFO</option></term>
+        <listitem>
+          <para>
+            Connection string of the upstream node to use for cloning.
+          </para>
+        </listitem>
+      </varlistentry>
+
       <varlistentry>
         <term><option>--dry-run</option></term>
         <listitem>
@@ -324,7 +350,7 @@
     </variablelist>
   </refsect1>
 
-  <refsect1>
+  <refsect1 id="repmgr-standby-clone-events">
     <title>Event notifications</title>
     <para>
       A <literal>standby_clone</literal> <link linkend="event-notifications">event notification</link> will be generated.

doc/repmgr-standby-follow.sgml

@@ -94,7 +94,7 @@
     </variablelist>
   </refsect1>
 
-  <refsect1>
+  <refsect1 id="repmgr-standby-follow-events">
     <title>Event notifications</title>
     <para>
       A <literal>standby_follow</literal> <link linkend="event-notifications">event notification</link> will be generated.

doc/repmgr-standby-promote.sgml

@@ -50,7 +50,7 @@
   </refsect1>
 
 
-  <refsect1>
+  <refsect1 id="repmgr-standby-promote-events">
     <title>Event notifications</title>
     <para>
       A <literal>standby_promote</literal> <link linkend="event-notifications">event notification</link> will be generated.

doc/repmgr-standby-register.sgml

@@ -159,7 +159,7 @@
     </variablelist>
   </refsect1>
 
-  <refsect1>
+  <refsect1 id="repmgr-standby-register-events">
     <title>Event notifications</title>
     <para>
       A <literal>standby_register</literal> <link linkend="event-notifications">event notification</link>

doc/repmgr-standby-switchover.sgml

@@ -46,6 +46,9 @@
         <application>repmgrd</application> should not be active on any nodes while a switchover is being
         executed. This restriction may be lifted in a later version.
       </para>
+      <para>
+        &repmgr; will not perform the switchover if an exclusive backup is running on the current primary.
+      </para>
     </note>
 
   </refsect1>
@@ -163,8 +166,8 @@
        <listitem>
          <simpara>
            <literal>standby_reconnect_timeout</literal>:
-           Number of seconds to attempt to reconnect to the demoted primary
+           number of seconds to attempt to wait for the demoted primary
-           once it has been restarted.
+           to reconnect to the promoted primary (default: 60 seconds)
          </simpara>
        </listitem>
 
@@ -193,7 +196,7 @@
     </para>
   </refsect1>
 
-  <refsect1>
+  <refsect1 id="repmgr-standby-switchover-events">
     <title>Event notifications</title>
     <para>
       <literal>standby_switchover</literal> and <literal>standby_promote</literal>

doc/repmgr-standby-unregister.sgml

@@ -59,7 +59,7 @@
     </variablelist>
   </refsect1>
 
-  <refsect1>
+  <refsect1 id="repmgr-standby-unregister-events">
     <title>Event notifications</title>
     <para>
       A <literal>standby_unregister</literal> <link linkend="event-notifications">event notification</link> will be generated.

doc/repmgr-witness-register.sgml

@@ -50,7 +50,7 @@
   </refsect1>
 
 
-  <refsect1>
+  <refsect1 id="repmgr-witness-register-events">
     <title>Event notifications</title>
     <para>
       A <literal>witness_register</literal> <link linkend="event-notifications">event notification</link> will be generated.

doc/repmgr-witness-unregister.sgml

@@ -20,7 +20,10 @@
     </para>
     <para>
       The node does not have to be running to be unregistered, however if this is the
-      case then connection information for the primary server must be provided.
+      case then either provide connection information for the primary server, or
+	  execute <command>repmgr witness unregister</command> on a running node and
+	  provide the parameter <option>--node-id</option> with the node ID of the
+	  witness server.
     </para>
     <para>
       Execute with the <literal>--dry-run</literal> option to check what would happen
@@ -36,17 +39,17 @@
     INFO: connecting to witness node "node3" (ID: 3)
     INFO: unregistering witness node 3
     INFO: witness unregistration complete
-    DETAIL: witness node with id 3 (conninfo: host=node3 dbname=repmgr user=repmgr port=5499) successfully unregistered</programlisting>
+    DETAIL: witness node with UD 3 successfully unregistered</programlisting>
     </para>
     <para>
       Unregistering a non-running witness node:
       <programlisting>
         $ repmgr -f /etc/repmgr.conf witness unregister -h node1 -p 5501  -F
-        INFO: connecting to witness node "node3" (ID: 3)
+        INFO: connecting to node "node3" (ID: 3)
-        NOTICE: unable to connect to witness node "node3" (ID: 3), removing node record on cluster primary only
+        NOTICE: unable to connect to node "node3" (ID: 3), removing node record on cluster primary only
         INFO: unregistering witness node 3
         INFO: witness unregistration complete
-        DETAIL: witness node with id 3 (conninfo: host=node3 dbname=repmgr user=repmgr port=5499) successfully unregistered</programlisting>
+        DETAIL: witness node with id ID 3 successfully unregistered</programlisting>
     </para>
   </refsect1>
 
@@ -62,8 +65,34 @@
     </para>
   </refsect1>
 
-
   <refsect1>
+
+    <title>Options</title>
+    <variablelist>
+
+      <varlistentry>
+        <term><option>--dry-run</option></term>
+        <listitem>
+          <para>
+            Check prerequisites but don't actually unregister the witness.
+          </para>
+        </listitem>
+      </varlistentry>
+
+      <varlistentry>
+       <term><option>--node-id</option></term>
+        <listitem>
+          <para>
+            Unregister witness server with the specified node ID.
+          </para>
+        </listitem>
+      </varlistentry>
+
+    </variablelist>
+  </refsect1>
+
+
+  <refsect1 id="repmgr-witness-unregister-events">
     <title>Event notifications</title>
     <para>
       A <literal>witness_unregister</literal> <link linkend="event-notifications">event notification</link> will be generated.

doc/repmgrd-bdr.sgml

@@ -15,7 +15,7 @@
   </para>
   <note>
     <simpara>
-      Due to the nature of BDR, it's only safe to use this solution for
+      Due to the nature of BDR 1.x/2.x, it's only safe to use this solution for
       a two-node scenario. Introducing additional nodes will create an inherent
       risk of node desynchronisation if a node goes down without being cleanly
       removed from the cluster.

doc/repmgrd-configuration.sgml

@@ -24,7 +24,7 @@
 
     <para>
       To use <application>repmgrd</application>, its associated function library <emphasis>must</emphasis> be
-      included in <filename>postgresql.conf</filename> with:
+      included via <filename>postgresql.conf</filename> with:
 
       <programlisting>
         shared_preload_libraries = 'repmgr'</programlisting>
@@ -34,23 +34,6 @@
       the <ulink url="https://www.postgresql.org/docs/current/static/runtime-config-client.html#GUC-SHARED-PRELOAD-LIBRARIES">PostgreSQL documentation</ulink>.
     </para>
 
-    <para>
-      To apply configuration file changes to a running <application>repmgrd</application>
-      daemon, execute the operating system's r<application>repmgrd</application> service reload command
-      (see <xref linkend="appendix-packages"> for examples),
-      or for instances  which were manually started, execute <command>kill -HUP</command>, e.g.
-      <command>kill -HUP `cat /tmp/repmgrd.pid`</command>.
-    </para>
-    <note>
-      <para>
-        Check the <application>repmgrd</application> log to see what changes were
-        applied, or if any issues were encountered when reloading the configuration.
-      </para>
-    </note>
-    <para>
-      Note that only a subset of configuration file parameters can be changed on a
-      running <application>repmgrd</application> daemon.
-    </para>
 
     <sect2 id="repmgrd-automatic-failover-configuration">
       <title>automatic failover configuration</title>
@@ -63,8 +46,17 @@
           follow_command='/usr/bin/repmgr standby follow -f /etc/repmgr.conf --log-to-file --upstream-node-id=%n'</programlisting>
       </para>
       <para>
-        Adjust file paths as appropriate; we recomment specifying the full path to the &repmgr; binary.
+        Adjust file paths as appropriate; alway specify the full path to the &repmgr; binary.
       </para>
+
+      <note>
+        <para>
+          &repmgr; will not apply <option>pg_bindir</option> when executing <option>promote_command</option>
+          or <option>follow_command</option>; these can be user-defined scripts so must always be
+          specified with the full path.
+        </para>
+      </note>
+
       <para>
         Note that the <literal>--log-to-file</literal> option will cause
         output generated by the &repmgr; command, when executed by <application>repmgrd</application>,
@@ -130,11 +122,11 @@
         particularly on <application>systemd</application>-based systems.
       </para>
       <para>
-        For more details, see <xref linkend="configuration-service-commands">.
+        For more details, see <xref linkend="configuration-file-service-commands">.
       </para>
     </sect2>
 
-    <sect2 id="repmgrd-monitoring-configuration">
+    <sect2 id="repmgrd-monitoring-configuration" xreflabel="repmgrd monitoring configuration">
       <indexterm>
         <primary>repmgrd</primary>
         <secondary>monitoring configuration</secondary>
@@ -157,6 +149,203 @@
       </para>
     </sect2>
 
+    <sect2 id="repmgrd-reloading-configuration"xreflabel="reloading repmgrd configuration">
+      <indexterm>
+        <primary>repmgrd</primary>
+        <secondary>applying configuration changes</secondary>
+      </indexterm>
+      <title>Applying configuration changes to repmgrd</title>
+      <para>
+        To apply configuration file changes to a running <application>repmgrd</application>
+        daemon, execute the operating system's <application>repmgrd</application> service reload command
+        (see <xref linkend="appendix-packages"> for examples),
+          or for instances  which were manually started, execute <command>kill -HUP</command>, e.g.
+          <command>kill -HUP `cat /tmp/repmgrd.pid`</command>.
+      </para>
+      <tip>
+        <para>
+          Check the <application>repmgrd</application> log to see what changes were
+          applied, or if any issues were encountered when reloading the configuration.
+        </para>
+      </tip>
+      <para>
+        Note that only the following subset of configuration file parameters can be changed on a
+        running <application>repmgrd</application> daemon:
+      </para>
+      <itemizedlist spacing="compact" mark="bullet">
+
+        <listitem>
+          <simpara>
+            <varname>async_query_timeout</varname>
+          </simpara>
+        </listitem>
+
+        <listitem>
+          <simpara>
+            <varname>bdr_local_monitoring_only</varname>
+          </simpara>
+        </listitem>
+
+        <listitem>
+          <simpara>
+            <varname>bdr_recovery_timeout</varname>
+          </simpara>
+        </listitem>
+
+        <listitem>
+          <simpara>
+            <varname>conninfo</varname>
+          </simpara>
+        </listitem>
+
+        <listitem>
+          <simpara>
+            <varname>degraded_monitoring_timeout</varname>
+          </simpara>
+        </listitem>
+
+        <listitem>
+          <simpara>
+            <varname>event_notification_command</varname>
+          </simpara>
+        </listitem>
+
+        <listitem>
+          <simpara>
+            <varname>event_notifications</varname>
+          </simpara>
+        </listitem>
+
+        <listitem>
+          <simpara>
+            <varname>failover</varname>
+          </simpara>
+        </listitem>
+
+        <listitem>
+          <simpara>
+            <varname>follow_command</varname>
+          </simpara>
+        </listitem>
+
+        <listitem>
+          <simpara>
+            <varname>log_facility</varname>
+          </simpara>
+        </listitem>
+
+        <listitem>
+          <simpara>
+            <varname>log_file</varname>
+          </simpara>
+        </listitem>
+
+        <listitem>
+          <simpara>
+            <varname>log_level</varname>
+          </simpara>
+        </listitem>
+
+        <listitem>
+          <simpara>
+            <varname>log_status_interval</varname>
+          </simpara>
+        </listitem>
+
+        <listitem>
+          <simpara>
+            <varname>monitor_interval_secs</varname>
+          </simpara>
+        </listitem>
+
+        <listitem>
+          <simpara>
+            <varname>monitoring_history</varname>
+          </simpara>
+        </listitem>
+
+        <listitem>
+          <simpara>
+            <varname>primary_notification_timeout</varname>
+          </simpara>
+        </listitem>
+
+        <listitem>
+          <simpara>
+            <varname>promote_command</varname>
+          </simpara>
+        </listitem>
+
+        <listitem>
+          <simpara>
+            <varname>reconnect_attempts</varname>
+          </simpara>
+        </listitem>
+
+        <listitem>
+          <simpara>
+            <varname>reconnect_interval</varname>
+          </simpara>
+        </listitem>
+
+        <listitem>
+          <simpara>
+            <varname>repmgrd_standby_startup_timeout</varname>
+          </simpara>
+        </listitem>
+
+      </itemizedlist>
+
+      <para>
+        The following set of configuration file parameters must be updated via
+        <command><link linkend="repmgr-standby-register">repmgr standby register --force</link></command>,
+        as they require changes to the <literal>repmgr.nodes</literal> table so they are visible to
+        all nodes in the replication cluster:
+      </para>
+      <itemizedlist spacing="compact" mark="bullet">
+
+        <listitem>
+          <simpara>
+            <varname>node_id</varname>
+          </simpara>
+        </listitem>
+
+        <listitem>
+          <simpara>
+            <varname>node_name</varname>
+          </simpara>
+        </listitem>
+
+        <listitem>
+          <simpara>
+            <varname>data_directory</varname>
+          </simpara>
+        </listitem>
+
+        <listitem>
+          <simpara>
+            <varname>location</varname>
+          </simpara>
+        </listitem>
+
+
+        <listitem>
+          <simpara>
+            <varname>priority</varname>
+          </simpara>
+        </listitem>
+
+      </itemizedlist>
+
+      <note>
+        <para>
+          After executing <command><link linkend="repmgr-standby-register">repmgr standby register --force</link></command>,
+          <application>repmgrd</application> <emphasis>must</emphasis> be restarted for the changes to take effect.
+        </para>
+      </note>
+
+    </sect2>
+
   </sect1>
 
   <sect1 id="repmgrd-daemon">
@@ -177,10 +366,63 @@
     <para>
       <application>repmgrd</application> can be started manually like this:
       <programlisting>
-        repmgrd -f /etc/repmgr.conf --pid-file /tmp/repmgrd.pid --daemonize</programlisting>
+        repmgrd -f /etc/repmgr.conf --pid-file /tmp/repmgrd.pid</programlisting>
       and stopped with <command>kill `cat /tmp/repmgrd.pid`</command>. Adjust paths as appropriate.
     </para>
 
+    <sect2 id="repmgrd-pid-file" xreflabel="repmgrd's PID file">
+      <indexterm>
+        <primary>repmgrd</primary>
+        <secondary>PID file</secondary>
+      </indexterm>
+      <indexterm>
+        <primary>PID file</primary>
+        <secondary>repmgrd</secondary>
+      </indexterm>
+      <title>repmgrd's PID file</title>
+      <para>
+        <application>repmgrd</application> will generate a PID file by default.
+      </para>
+      <note>
+        <simpara>
+          This is a behaviour change from previous versions (earlier than 4.1), where
+          the PID file had to be explicitly specified with the command line
+          parameter <option> --pid-file</option>.
+        </simpara>
+      </note>
+      <para>
+        The PID file can be specified in <filename>repmgr.conf</filename> with the configuration
+        parameter <varname>repmgrd_pid_file</varname>.
+      </para>
+      <para>
+        It can also be specified on the command line (as in previous versions) with
+        the command line parameter <option>--pid-file</option>. Note this will override
+        any value set in <filename>repmgr.conf</filename> with <varname>repmgrd_pid_file</varname>.
+        <option>--pid-file</option> may be deprecated in future releases.
+      </para>
+      <para>
+        If a PID file location was specified by the package maintainer, <application>repmgrd</application>
+        will use that. This only applies if &repmgr; was installed from a package and the package
+        maintainer has specified the PID file location.
+      </para>
+      <para>
+        If none of the above apply, <application>repmgrd</application> will create a PID file
+        in the operating system's temporary directory (das etermined by the environment variable
+        <varname>TMPDIR</varname>, or if that is not set, will use <filename>/tmp</filename>).
+      </para>
+      <para>
+        To prevent a PID file being generated at all, provide the command line option
+        <option>--no-pid-file</option>.
+      </para>
+      <para>
+        To see which PID file <application>repmgrd</application> would use, execute <application>repmgrd</application>
+        with the option <option>--show-pid-file</option>. <application>repmgrd</application>
+        will not start if this option is provided. Note that the value shown is the
+        file  <application>repmgrd</application> would use next time it starts, and is
+        not necessarily the PID file currently in use.
+      </para>
+    </sect2>
+
     <sect2 id="repmgrd-configuration-debian-ubuntu">
       <indexterm>
         <primary>repmgrd</primary>
@@ -269,25 +511,34 @@ REPMGRD_ENABLED=no
      <secondary>repmgrd</secondary>
    </indexterm>
 
+   <indexterm>
+     <primary>repmgrd</primary>
+     <secondary>log rotation</secondary>
+   </indexterm>
+
   <title>repmgrd log rotation</title>
   <para>
    To ensure the current <application>repmgrd</application> logfile
    (specified in <filename>repmgr.conf</filename> with the parameter
-   <option>log_file</option> does not grow indefinitely, configure your
+   <option>log_file</option>) does not grow indefinitely, configure your
    system's <command>logrotate</command> to regularly rotate it.
   </para>
   <para>
    Sample configuration to rotate logfiles weekly with retention for
    up to 52 weeks and rotation forced if a file grows beyond 100Mb:
    <programlisting>
-    /var/log/postgresql/repmgr-9.6.log {
+    /var/log/repmgr/repmgrd.log {
         missingok
         compress
         rotate 52
         maxsize 100M
         weekly
         create 0600 postgres postgres
+        postrotate
+            /usr/bin/killall -HUP repmgrd
+        endscript
     }</programlisting>
   </para>
+
  </sect1>
 </chapter>

doc/repmgrd-degraded-monitoring.sgml

@@ -1,4 +1,4 @@
-<chapter id="repmgrd-degraded-monitoring">
+<chapter id="repmgrd-degraded-monitoring" xreflabel="repmgrd degraded monitoring">
  <indexterm>
    <primary>repmgrd</primary>
    <secondary>degraded monitoring</secondary>
@@ -7,8 +7,8 @@
  <title>"degraded monitoring" mode</title>
  <para>
   In certain circumstances, <application>repmgrd</application> is not able to fulfill its primary mission
-  of monitoring the nodes' upstream server. In these cases it enters "degraded
+  of monitoring the node's upstream server. In these cases it enters &quot;degraded monitoring&quot;
-  monitoring" mode, where <application>repmgrd</application> remains active but is waiting for the situation
+  mode, where <application>repmgrd</application> remains active but is waiting for the situation
   to be resolved.
  </para>
  <para>

doc/repmgrd-monitoring.sgml

@@ -1,4 +1,4 @@
-<chapter id="repmgrd-monitoring">
+<chapter id="repmgrd-monitoring" xreflabel="Monitoring with repmgrd">
  <indexterm>
    <primary>repmgrd</primary>
    <secondary>monitoring</secondary>

doc/repmgrd-network-split.sgml

@@ -40,8 +40,8 @@
   In a failover situation, <application>repmgrd</application> will check if any servers in the
   same location as the current primary node are visible.  If not, <application>repmgrd</application>
   will assume a network interruption and not promote any node in any
-  other location (it will however enter <xref linkend="repmgrd-degraded-monitoring"> mode until
+  other location (it will however enter <link linkend="repmgrd-degraded-monitoring">degraded monitoring</link>
-  a primary becomes visible).
+  mode until a primary becomes visible).
  </para>
 
 </chapter>

doc/switchover.sgml

@@ -57,7 +57,14 @@
 
    <para>
     As mentioned in the previous section, success of the switchover operation depends on
-    &repmgr;  being able to shut down the current primary server quickly and cleanly.
+    &repmgr; being able to shut down the current primary server quickly and cleanly.
+   </para>
+
+   <para>
+     Ensure that the promotion candidate has sufficient free walsenders available
+     (PostgreSQL configuration item <varname>max_wal_senders</varname>), and if replication
+     slots are in use, at least one free slot is available for the demotion candidate (
+     PostgreSQL configuration item <varname>max_replication_slots</varname>).
    </para>
 
    <para>
@@ -104,7 +111,7 @@
        server.
      </para>
      <para>
-       For more details, see <xref linkend="configuration-service-commands">.
+       For more details, see <xref linkend="configuration-file-service-commands">.
      </para>
    </important>
 
@@ -121,15 +128,21 @@
     </simpara>
    </note>
 
-
    <para>
-    Check that access from applications is minimalized or preferably blocked
+     Check that access from applications is minimalized or preferably blocked
-    completely, so applications are not unexpectedly interrupted.
+     completely, so applications are not unexpectedly interrupted.
    </para>
 
+   <note>
+     <para>
+       If an exclusive backup is running on the current primary, &repmgr; will not perform the
+       switchover.
+     </para>
+   </note>
+
    <para>
-    Check there is no significant replication lag on standbys attached to the
+     Check there is no significant replication lag on standbys attached to the
-    current primary.
+     current primary.
    </para>
 
    <para>
@@ -147,6 +160,7 @@
      </para>
    </note>
 
+
    <para>
     Finally, consider executing <command>repmgr standby switchover</command> with the
     <literal>--dry-run</literal> option; this will perform any necessary checks and inform you about

doc/upgrading-repmgr.sgml

@@ -29,8 +29,18 @@
       </listitem>
       <listitem>
         <simpara>
-          In the database where the &repmgr; extension is installed, execute
+          <application>repmgrd</application> (if running) must be restarted.
-          <command>ALTER EXTENSION repmgr UPDATE</command>.
+        </simpara>
+      </listitem>
+      <listitem>
+        <simpara>
+          For major releases, e.g. from <literal>4.0.x</literal> to <literal>4.1</literal>,
+          execute <command>ALTER EXTENSION repmgr UPDATE</command>
+          on the primary node in the database where the &repmgr; extension is installed.
+        </simpara>
+        <simpara>
+          This will update the extension metadata and, if necessary, apply
+          changes to the &repmgr; extension objects.
         </simpara>
       </listitem>
     </orderedlist>
@@ -41,10 +51,6 @@
     release as they may contain upgrade instructions particular to individual versions.
   </para>
 
-  <para>
-    If the <application>repmgrd</application> daemon is in use, we recommend stopping it
-    before upgrading &repmgr;.
-  </para>
   <para>
     Note that it may be necessary to restart the PostgreSQL server if the upgrade contains
     changes to the shared object file used by <application>repmgrd</application>; check the

doc/version.sgml

@@ -1 +1 @@
-<!ENTITY repmgrversion "4.0.6">
+<!ENTITY repmgrversion "4.1.1">

errcode.h

@@ -46,6 +46,6 @@
 #define ERR_SWITCHOVER_INCOMPLETE 22
 #define ERR_FOLLOW_FAIL 23
 #define ERR_REJOIN_FAIL 24
-#define ERR_CLUSTER_CHECK 25
+#define ERR_NODE_STATUS 25
 
 #endif							/* _ERRCODE_H_ */

log.c

@@ -42,7 +42,7 @@ _stderr_log_with_level(const char *level_name, int level, const char *fmt, va_li
 __attribute__((format(PG_PRINTF_ATTRIBUTE, 3, 0)));
 
 int			log_type = REPMGR_STDERR;
-int			log_level = LOG_NOTICE;
+int			log_level = LOG_INFO;
 int			last_log_level = LOG_INFO;
 int			verbose_logging = false;
 int			terse_logging = false;
@@ -70,7 +70,7 @@ _stderr_log_with_level(const char *level_name, int level, const char *fmt, va_li
 
 	/*
 	 * Store the requested level so that if there's a subsequent log_hint() or
-	 * log_detail(), we can suppress that if appropriate.
+	 * log_detail(), we can suppress that if --terse was specified,
 	 */
 	last_log_level = level;
 
@@ -329,6 +329,13 @@ logger_set_terse(void)
 }
 
 
+void
+logger_set_level(int new_log_level)
+{
+	log_level = new_log_level;
+}
+
+
 void
 logger_set_min_level(int min_log_level)
 {
@@ -336,6 +343,7 @@ logger_set_min_level(int min_log_level)
 		log_level = min_log_level;
 }
 
+
 int
 detect_log_level(const char *level)
 {

log.h

@@ -129,6 +129,7 @@ bool		logger_shutdown(void);
 void		logger_set_verbose(void);
 void		logger_set_terse(void);
 void		logger_set_min_level(int min_log_level);
+void		logger_set_level(int new_log_level);
 
 void
 log_detail(const char *fmt,...)

repmgr--4.0--4.1.sql

@@ -0,0 +1,2 @@
+-- complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "CREATE EXTENSION repmgr" to load this file. \quit

repmgr--4.1.sql

@@ -0,0 +1,167 @@
+-- complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "CREATE EXTENSION repmgr" to load this file. \quit
+
+CREATE TABLE repmgr.nodes (
+  node_id          INTEGER     PRIMARY KEY,
+  upstream_node_id INTEGER     NULL REFERENCES nodes (node_id) DEFERRABLE,
+  active           BOOLEAN     NOT NULL DEFAULT TRUE,
+  node_name        TEXT        NOT NULL,
+  type             TEXT        NOT NULL CHECK (type IN('primary','standby','witness','bdr')),
+  location         TEXT        NOT NULL DEFAULT 'default',
+  priority         INT         NOT NULL DEFAULT 100,
+  conninfo         TEXT        NOT NULL,
+  repluser         VARCHAR(63) NOT NULL,
+  slot_name        TEXT        NULL,
+  config_file      TEXT        NOT NULL
+);
+
+CREATE TABLE repmgr.events (
+  node_id          INTEGER NOT NULL,
+  event            TEXT NOT NULL,
+  successful       BOOLEAN NOT NULL DEFAULT TRUE,
+  event_timestamp  TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  details          TEXT NULL
+);
+
+DO $repmgr$
+DECLARE
+  DECLARE server_version_num INT;
+BEGIN
+  SELECT setting
+    FROM pg_catalog.pg_settings
+   WHERE name = 'server_version_num'
+    INTO server_version_num;
+  IF server_version_num >= 90400 THEN
+    EXECUTE $repmgr_func$
+CREATE TABLE repmgr.monitoring_history (
+  primary_node_id                INTEGER NOT NULL,
+  standby_node_id                INTEGER NOT NULL,
+  last_monitor_time              TIMESTAMP WITH TIME ZONE NOT NULL,
+  last_apply_time                TIMESTAMP WITH TIME ZONE,
+  last_wal_primary_location      PG_LSN NOT NULL,
+  last_wal_standby_location      PG_LSN,
+  replication_lag                BIGINT NOT NULL,
+  apply_lag                      BIGINT NOT NULL
+)
+    $repmgr_func$;
+  ELSE
+    EXECUTE $repmgr_func$
+CREATE TABLE repmgr.monitoring_history (
+  primary_node_id                INTEGER NOT NULL,
+  standby_node_id                INTEGER NOT NULL,
+  last_monitor_time              TIMESTAMP WITH TIME ZONE NOT NULL,
+  last_apply_time                TIMESTAMP WITH TIME ZONE,
+  last_wal_primary_location      TEXT NOT NULL,
+  last_wal_standby_location      TEXT,
+  replication_lag                BIGINT NOT NULL,
+  apply_lag                      BIGINT NOT NULL
+)
+    $repmgr_func$;
+  END IF;
+END$repmgr$;
+
+
+
+CREATE INDEX idx_monitoring_history_time
+          ON repmgr.monitoring_history (last_monitor_time, standby_node_id);
+
+CREATE VIEW repmgr.show_nodes AS
+   SELECT n.node_id,
+          n.node_name,
+          n.active,
+          n.upstream_node_id,
+          un.node_name AS upstream_node_name,
+          n.type,
+          n.priority,
+          n.conninfo
+     FROM repmgr.nodes n
+LEFT JOIN repmgr.nodes un
+       ON un.node_id = n.upstream_node_id;
+
+
+/* XXX update upgrade scripts! */
+CREATE TABLE repmgr.voting_term (
+  term INT NOT NULL
+);
+
+CREATE UNIQUE INDEX voting_term_restrict
+ON repmgr.voting_term ((TRUE));
+
+CREATE RULE voting_term_delete AS
+   ON DELETE TO repmgr.voting_term
+   DO INSTEAD NOTHING;
+
+
+/* ================= */
+/* repmgrd functions */
+/* ================= */
+
+/* monitoring functions */
+
+CREATE FUNCTION set_local_node_id(INT)
+  RETURNS VOID
+  AS 'MODULE_PATHNAME', 'set_local_node_id'
+  LANGUAGE C STRICT;
+
+CREATE FUNCTION get_local_node_id()
+  RETURNS INT
+  AS 'MODULE_PATHNAME', 'get_local_node_id'
+  LANGUAGE C STRICT;
+
+CREATE FUNCTION standby_set_last_updated()
+  RETURNS TIMESTAMP WITH TIME ZONE
+  AS 'MODULE_PATHNAME', 'standby_set_last_updated'
+  LANGUAGE C STRICT;
+
+CREATE FUNCTION standby_get_last_updated()
+  RETURNS TIMESTAMP WITH TIME ZONE
+  AS 'MODULE_PATHNAME', 'standby_get_last_updated'
+  LANGUAGE C STRICT;
+
+/* failover functions */
+
+CREATE FUNCTION notify_follow_primary(INT)
+  RETURNS VOID
+  AS 'MODULE_PATHNAME', 'notify_follow_primary'
+  LANGUAGE C STRICT;
+
+CREATE FUNCTION get_new_primary()
+  RETURNS INT
+  AS 'MODULE_PATHNAME', 'get_new_primary'
+  LANGUAGE C STRICT;
+
+CREATE FUNCTION reset_voting_status()
+  RETURNS VOID
+  AS 'MODULE_PATHNAME', 'reset_voting_status'
+  LANGUAGE C STRICT;
+
+CREATE FUNCTION am_bdr_failover_handler(INT)
+  RETURNS BOOL
+  AS 'MODULE_PATHNAME', 'am_bdr_failover_handler'
+  LANGUAGE C STRICT;
+
+CREATE FUNCTION unset_bdr_failover_handler()
+  RETURNS VOID
+  AS 'MODULE_PATHNAME', 'unset_bdr_failover_handler'
+  LANGUAGE C STRICT;
+
+
+CREATE VIEW repmgr.replication_status AS
+  SELECT m.primary_node_id, m.standby_node_id, n.node_name AS standby_name,
+ 	     n.type AS node_type, n.active, last_monitor_time,
+         CASE WHEN n.type='standby' THEN m.last_wal_primary_location ELSE NULL END AS last_wal_primary_location,
+         m.last_wal_standby_location,
+         CASE WHEN n.type='standby' THEN pg_catalog.pg_size_pretty(m.replication_lag) ELSE NULL END AS replication_lag,
+         CASE WHEN n.type='standby' THEN
+           CASE WHEN replication_lag > 0 THEN age(now(), m.last_apply_time) ELSE '0'::INTERVAL END
+           ELSE NULL
+         END AS replication_time_lag,
+         CASE WHEN n.type='standby' THEN pg_catalog.pg_size_pretty(m.apply_lag) ELSE NULL END AS apply_lag,
+         AGE(NOW(), CASE WHEN pg_catalog.pg_is_in_recovery() THEN repmgr.standby_get_last_updated() ELSE m.last_monitor_time END) AS communication_time_lag
+    FROM repmgr.monitoring_history m
+    JOIN repmgr.nodes n ON m.standby_node_id = n.node_id
+   WHERE (m.standby_node_id, m.last_monitor_time) IN (
+	          SELECT m1.standby_node_id, MAX(m1.last_monitor_time)
+			    FROM repmgr.monitoring_history m1 GROUP BY 1
+         );
+

repmgr-action-bdr.c

@@ -83,9 +83,10 @@ do_bdr_register(void)
 		exit(ERR_BAD_CONFIG);
 	}
 
-	if (bdr_nodes.node_count > 2)
+	/* BDR 2 implementation is for 2 nodes only */
+	if (get_bdr_version_num() < 3 && bdr_nodes.node_count > 2)
 	{
-		log_error(_("repmgr can only support BDR clusters with 2 nodes"));
+		log_error(_("repmgr can only support BDR 2.x clusters with 2 nodes"));
 		log_detail(_("this BDR cluster has %i nodes"), bdr_nodes.node_count);
 		PQfinish(conn);
 		pfree(dbname);
@@ -176,6 +177,7 @@ do_bdr_register(void)
 
 	if (bdr_node_has_repmgr_set(conn, config_file_options.node_name) == false)
 	{
+		log_debug("bdr_node_has_repmgr_set() = false");
 		bdr_node_set_repmgr_set(conn, config_file_options.node_name);
 	}
 
@@ -201,6 +203,7 @@ do_bdr_register(void)
 			if (bdr_nodes.node_count == 0)
 			{
 				log_error(_("unable to retrieve any BDR node records"));
+				log_detail("%s", PQerrorMessage(conn));
 				PQfinish(conn);
 				exit(ERR_BAD_CONFIG);
 			}
@@ -252,7 +255,35 @@ do_bdr_register(void)
 	}
 
 	/* Add the repmgr extension tables to a replication set */
-	add_extension_tables_to_bdr_replication_set(conn);
+
+	if (get_bdr_version_num() < 3)
+	{
+		add_extension_tables_to_bdr_replication_set(conn);
+	}
+	else
+	{
+		/* this is the only table we need to replicate */
+		char *replication_set = get_default_bdr_replication_set(conn);
+
+		/*
+		 * this probably won't happen, but we need to be sure we're using
+		 * the replication set metadata correctly...
+		 */
+		if (conn == NULL)
+		{
+			log_error(_("unable to retrieve default BDR replication set"));
+			log_hint(_("see preceding messages"));
+			log_debug("check query in get_default_bdr_replication_set()");
+			exit(ERR_BAD_CONFIG);
+		}
+
+		if (is_table_in_bdr_replication_set(conn, "nodes", replication_set) == false)
+		{
+			add_table_to_bdr_replication_set(conn, "nodes", replication_set);
+		}
+
+		pfree(replication_set);
+	}
 
 	initPQExpBuffer(&event_details);
 

repmgr-action-cluster.c

@@ -83,6 +83,8 @@ do_cluster_show(void)
 	int			i = 0;
 	ItemList	warnings = {NULL, NULL};
 	bool		success = false;
+	bool		error_found = false;
+	bool		connection_error_found = false;
 
 	/* Connect to local database to obtain cluster connection data */
 	log_verbose(LOG_INFO, _("connecting to database"));
@@ -140,14 +142,26 @@ do_cluster_show(void)
 		}
 		else
 		{
-			char		error[MAXLEN];
-
-			strncpy(error, PQerrorMessage(cell->node_info->conn), MAXLEN);
 			cell->node_info->node_status = NODE_STATUS_DOWN;
 			cell->node_info->recovery_type = RECTYPE_UNKNOWN;
-			item_list_append_format(&warnings,
+
-									"when attempting to connect to node \"%s\" (ID: %i), following error encountered :\n\"%s\"",
+			connection_error_found = true;
-									cell->node_info->node_name, cell->node_info->node_id, trim(error));
+
+			if (runtime_options.verbose)
+			{
+				char		error[MAXLEN];
+
+				strncpy(error, PQerrorMessage(cell->node_info->conn), MAXLEN);
+				item_list_append_format(&warnings,
+										"when attempting to connect to node \"%s\" (ID: %i), following error encountered :\n\"%s\"",
+										cell->node_info->node_name, cell->node_info->node_id, trim(error));
+			}
+			else
+			{
+				item_list_append_format(&warnings,
+										"unable to  connect to node \"%s\" (ID: %i)",
+										cell->node_info->node_name, cell->node_info->node_id);
+			}
 		}
 
 		initPQExpBuffer(&details);
@@ -218,6 +232,7 @@ do_cluster_show(void)
 						else
 						{
 							appendPQExpBuffer(&details, "- failed");
+							error_found = true;
 						}
 					}
 				}
@@ -281,6 +296,7 @@ do_cluster_show(void)
 						else
 						{
 							appendPQExpBuffer(&details, "- failed");
+							error_found = true;
 						}
 					}
 				}
@@ -292,17 +308,27 @@ do_cluster_show(void)
 					if (cell->node_info->node_status == NODE_STATUS_UP)
 					{
 						if (cell->node_info->active == true)
+						{
 							appendPQExpBuffer(&details, "* running");
+						}
 						else
+						{
 							appendPQExpBuffer(&details, "! running");
+							error_found = true;
+						}
 					}
 					/* node is unreachable */
 					else
 					{
 						if (cell->node_info->active == true)
+						{
 							appendPQExpBuffer(&details, "? unreachable");
+						}
 						else
+						{
 							appendPQExpBuffer(&details, "- failed");
+							error_found = true;
+						}
 					}
 				}
 				break;
@@ -310,6 +336,7 @@ do_cluster_show(void)
 				{
 					/* this should never happen */
 					appendPQExpBuffer(&details, "? unknown node type");
+						error_found = true;
 				}
 				break;
 		}
@@ -414,7 +441,6 @@ do_cluster_show(void)
 	PQfinish(conn);
 
 	/* emit any warnings */
-
 	if (warnings.head != NULL && runtime_options.terse == false && runtime_options.output_mode != OM_CSV)
 	{
 		ItemListCell *cell = NULL;
@@ -424,6 +450,25 @@ do_cluster_show(void)
 		{
 			printf(_("  - %s\n"), cell->string);
 		}
+
+		if (runtime_options.verbose == false && connection_error_found == true)
+		{
+			log_hint(_("execute with --verbose option to see connection error messages"));
+		}
+	}
+
+	/*
+	 * If warnings were noted, even if they're not displayed (e.g. in --csv node),
+	 * that means something's not right so we need to emit a non-zero exit code.
+	 */
+	if (warnings.head != NULL)
+	{
+		error_found = true;
+	}
+
+	if (error_found == true)
+	{
+		exit(ERR_NODE_STATUS);
 	}
 }
 
@@ -436,6 +481,7 @@ do_cluster_show(void)
  *   --all
  *   --node-[id|name]
  *   --event
+ *   --csv
  */
 
 void
@@ -480,8 +526,12 @@ do_cluster_event(void)
 	strncpy(headers_event[EV_TIMESTAMP].title, _("Timestamp"), MAXLEN);
 	strncpy(headers_event[EV_DETAILS].title, _("Details"), MAXLEN);
 
-	/* if --terse provided, simply omit the "Details" column */
+	/*
-	if (runtime_options.terse == true)
+	 * If --terse or --csv provided, simply omit the "Details" column.
+	 * In --csv mode we'd need to quote/escape the contents "Details" column,
+	 * which is doable but which will remain a TODO for now.
+	 */
+	if (runtime_options.terse == true || runtime_options.output_mode == OM_CSV)
 		column_count --;
 
 	for (i = 0; i < column_count; i++)
@@ -504,47 +554,64 @@ do_cluster_event(void)
 
 	}
 
-	for (i = 0; i < column_count; i++)
+	if (runtime_options.output_mode == OM_TEXT)
 	{
-		if (i == 0)
+		for (i = 0; i < column_count; i++)
-			printf(" ");
+		{
-		else
+			if (i == 0)
-			printf(" | ");
+				printf(" ");
+			else
+				printf(" | ");
 
-		printf("%-*s",
+			printf("%-*s",
-			   headers_event[i].max_length,
+				   headers_event[i].max_length,
-			   headers_event[i].title);
+				   headers_event[i].title);
+		}
+		printf("\n");
+		printf("-");
+		for (i = 0; i < column_count; i++)
+		{
+			int			j;
+
+			for (j = 0; j < headers_event[i].max_length; j++)
+				printf("-");
+
+			if (i < (column_count - 1))
+				printf("-+-");
+			else
+				printf("-");
+		}
+
+		printf("\n");
 	}
-	printf("\n");
-	printf("-");
-	for (i = 0; i < column_count; i++)
-	{
-		int			j;
-
-		for (j = 0; j < headers_event[i].max_length; j++)
-			printf("-");
-
-		if (i < (column_count - 1))
-			printf("-+-");
-		else
-			printf("-");
-	}
-
-	printf("\n");
 
 	for (i = 0; i < PQntuples(res); i++)
 	{
 		int			j;
 
-		printf(" ");
+		if (runtime_options.output_mode == OM_CSV)
-		for (j = 0; j < column_count; j++)
 		{
-			printf("%-*s",
+			for (j = 0; j < column_count; j++)
-				   headers_event[j].max_length,
+			{
-				   PQgetvalue(res, i, j));
+				printf("%s", PQgetvalue(res, i, j));
+				if ((j + 1) < column_count)
+				{
+					printf(",");
+				}
+			}
+		}
+		else
+		{
+			printf(" ");
+			for (j = 0; j < column_count; j++)
+			{
+				printf("%-*s",
+					   headers_event[j].max_length,
+					   PQgetvalue(res, i, j));
 
-			if (j < (column_count - 1))
+				if (j < (column_count - 1))
-				printf(" | ");
+					printf(" | ");
+			}
 		}
 
 		printf("\n");
@@ -554,7 +621,8 @@ do_cluster_event(void)
 
 	PQfinish(conn);
 
-	puts("");
+	if (runtime_options.output_mode == OM_TEXT)
+		puts("");
 }
 
 
@@ -696,7 +764,7 @@ do_cluster_crosscheck(void)
 
 	if (error_found == true)
 	{
-		exit(ERR_CLUSTER_CHECK);
+		exit(ERR_NODE_STATUS);
 	}
 }
 
@@ -786,7 +854,7 @@ do_cluster_matrix()
 
 	if (error_found == true)
 	{
-		exit(ERR_CLUSTER_CHECK);
+		exit(ERR_NODE_STATUS);
 	}
 }
 
@@ -1282,6 +1350,7 @@ do_cluster_cleanup(void)
 	PGconn	   *conn = NULL;
 	PGconn	   *primary_conn = NULL;
 	int			entries_to_delete = 0;
+	PQExpBufferData event_details;
 
 	conn = establish_db_connection(config_file_options.conninfo, true);
 
@@ -1295,7 +1364,13 @@ do_cluster_cleanup(void)
 
 	entries_to_delete = get_number_of_monitoring_records_to_delete(primary_conn, runtime_options.keep_history);
 
-	if (entries_to_delete == 0)
+	if (entries_to_delete < 0)
+	{
+		log_error(_("unable to query number of monitoring records to clean up"));
+		PQfinish(primary_conn);
+		exit(ERR_DB_QUERY);
+	}
+	else if (entries_to_delete == 0)
 	{
 		log_info(_("no monitoring records to delete"));
 		PQfinish(primary_conn);
@@ -1305,10 +1380,23 @@ do_cluster_cleanup(void)
 	log_debug("at least %i monitoring records for deletion",
 			  entries_to_delete);
 
+	initPQExpBuffer(&event_details);
+
 	if (delete_monitoring_records(primary_conn, runtime_options.keep_history) == false)
 	{
-		log_error(_("unable to delete monitoring records"));
+		appendPQExpBuffer(&event_details,
+						  _("unable to delete monitoring records"));
+
+		log_error("%s", event_details.data);
 		log_detail("%s", PQerrorMessage(primary_conn));
+
+		create_event_notification(primary_conn,
+								  &config_file_options,
+								  config_file_options.node_id,
+								  "cluster_cleanup",
+								  false,
+								  event_details.data);
+
 		PQfinish(primary_conn);
 		exit(ERR_DB_QUERY);
 	}
@@ -1320,7 +1408,22 @@ do_cluster_cleanup(void)
 		log_detail("%s", PQerrorMessage(primary_conn));
 	}
 
+	appendPQExpBuffer(&event_details,
+					  _("monitoring records deleted"));
 
+	if (runtime_options.keep_history > 0)
+		appendPQExpBuffer(&event_details,
+						  _("; records newer than %i day(s) retained"),
+						  runtime_options.keep_history);
+
+	create_event_notification(primary_conn,
+							  &config_file_options,
+							  config_file_options.node_id,
+							  "cluster_cleanup",
+							  true,
+							  event_details.data);
+
+	termPQExpBuffer(&event_details);
 	PQfinish(primary_conn);
 
 	if (runtime_options.keep_history > 0)
@@ -1347,6 +1450,7 @@ do_cluster_help(void)
 	printf(_("    %s [OPTIONS] cluster matrix\n"), progname());
 	printf(_("    %s [OPTIONS] cluster crosscheck\n"), progname());
 	printf(_("    %s [OPTIONS] cluster event\n"), progname());
+	printf(_("    %s [OPTIONS] cluster cleanup\n"), progname());
 	puts("");
 
 	printf(_("CLUSTER SHOW\n"));
@@ -1386,6 +1490,7 @@ do_cluster_help(void)
 	printf(_("    --event                   filter specific event\n"));
 	printf(_("    --node-id                 restrict entries to node with this ID\n"));
 	printf(_("    --node-name               restrict entries to node with this name\n"));
+	printf(_("    --csv                     emit output as CSV\n"));
 	puts("");
 
 	printf(_("CLUSTER CLEANUP\n"));

repmgr-action-node.c

@@ -47,6 +47,7 @@ static CheckStatus do_node_check_downstream(PGconn *conn, OutputMode mode, Check
 static CheckStatus do_node_check_replication_lag(PGconn *conn, OutputMode mode, t_node_info *node_info, CheckStatusList *list_output);
 static CheckStatus do_node_check_role(PGconn *conn, OutputMode mode, t_node_info *node_info, CheckStatusList *list_output);
 static CheckStatus do_node_check_slots(PGconn *conn, OutputMode mode, t_node_info *node_info, CheckStatusList *list_output);
+static CheckStatus do_node_check_missing_slots(PGconn *conn, OutputMode mode, t_node_info *node_info, CheckStatusList *list_output);
 
 /*
  * NODE STATUS
@@ -169,11 +170,17 @@ do_node_status(void)
 	}
 	else
 	{
+		/* "archive_mode" is not "off", i.e. one of "on", "always" */
 		bool		enabled = true;
 		PQExpBufferData archiving_status;
 		char		archive_command[MAXLEN] = "";
 
 		initPQExpBuffer(&archiving_status);
+
+		/*
+		 * if the node is a standby, and "archive_mode" is "on", archiving will
+		 * actually be disabled.
+		 */
 		if (recovery_type == RECTYPE_STANDBY)
 		{
 			if (guc_set(conn, "archive_mode", "=", "on"))
@@ -251,6 +258,55 @@ do_node_status(void)
 								  "disabled");
 	}
 
+	/* check for attached nodes */
+	{
+		NodeInfoList downstream_nodes = T_NODE_INFO_LIST_INITIALIZER;
+		NodeInfoListCell *node_cell = NULL;
+		ItemList	missing_nodes = {NULL, NULL};
+		int			missing_nodes_count = 0;
+		int			expected_nodes_count = 0;
+
+		get_downstream_node_records(conn, config_file_options.node_id, &downstream_nodes);
+
+		/* if a witness node is present, we'll need to remove this from the total */
+		expected_nodes_count = downstream_nodes.node_count;
+
+		for (node_cell = downstream_nodes.head; node_cell; node_cell = node_cell->next)
+		{
+			/* skip witness server */
+			if (node_cell->node_info->type == WITNESS)
+			{
+				expected_nodes_count --;
+				continue;
+			}
+
+			if (is_downstream_node_attached(conn, node_cell->node_info->node_name) == false)
+			{
+				missing_nodes_count++;
+				item_list_append_format(&missing_nodes,
+										"%s (ID: %i)",
+										node_cell->node_info->node_name,
+										node_cell->node_info->node_id);
+			}
+		}
+
+		if (missing_nodes_count)
+		{
+			ItemListCell *missing_cell = NULL;
+
+			item_list_append_format(&warnings,
+									_("- %i of %i downstream nodes not attached:"),
+									missing_nodes_count,
+									expected_nodes_count);
+
+			for (missing_cell = missing_nodes.head; missing_cell; missing_cell = missing_cell->next)
+			{
+				item_list_append_format(&warnings,
+										"  - %s\n", missing_cell->string);
+			}
+		}
+	}
+
 	if (server_version_num < 90400)
 	{
 		key_value_list_set(&node_status,
@@ -486,18 +542,31 @@ do_node_status(void)
 
 	termPQExpBuffer(&output);
 
-	if (runtime_options.output_mode == OM_TEXT && warnings.head != NULL && runtime_options.terse == false)
+	if (warnings.head != NULL && runtime_options.terse == false && runtime_options.output_mode == OM_TEXT)
 	{
 		log_warning(_("following issue(s) were detected:"));
 		print_item_list(&warnings);
 		log_hint(_("execute \"repmgr node check\" for more details"));
 	}
 
+	clear_node_info_list(&missing_slots);
 	key_value_list_free(&node_status);
 	item_list_free(&warnings);
 	PQfinish(conn);
+
+	/*
+	 * If warnings were noted, even if they're not displayed (e.g. in --csv node),
+	 * that means something's not right so we need to emit a non-zero exit code.
+	 */
+	if (warnings.head != NULL)
+	{
+		exit(ERR_NODE_STATUS);
+	}
+
+	return;
 }
 
+
 /*
  * Returns information about the running state of the node.
  * For internal use during "standby switchover".
@@ -628,6 +697,7 @@ do_node_check(void)
 	CheckStatusList status_list = {NULL, NULL};
 	CheckStatusListCell *cell = NULL;
 
+	bool			issue_detected = false;
 
 	/* for internal use */
 	if (runtime_options.has_passfile == true)
@@ -712,6 +782,17 @@ do_node_check(void)
 		exit(return_code);
 	}
 
+	if (runtime_options.missing_slots == true)
+	{
+		return_code = do_node_check_missing_slots(conn,
+												  runtime_options.output_mode,
+												  &node_info,
+												  NULL);
+		PQfinish(conn);
+		exit(return_code);
+	}
+
+
 	if (runtime_options.output_mode == OM_NAGIOS)
 	{
 		log_error(_("--nagios can only be used with a specific check"));
@@ -725,11 +806,23 @@ do_node_check(void)
 	initPQExpBuffer(&output);
 
 	/* order functions are called is also output order */
-	(void) do_node_check_role(conn, runtime_options.output_mode, &node_info, &status_list);
+	if (do_node_check_role(conn, runtime_options.output_mode, &node_info, &status_list) != CHECK_STATUS_OK)
-	(void) do_node_check_replication_lag(conn, runtime_options.output_mode, &node_info, &status_list);
+		issue_detected = true;
-	(void) do_node_check_archive_ready(conn, runtime_options.output_mode, &status_list);
+
-	(void) do_node_check_downstream(conn, runtime_options.output_mode, &status_list);
+	if (do_node_check_replication_lag(conn, runtime_options.output_mode, &node_info, &status_list) != CHECK_STATUS_OK)
-	(void) do_node_check_slots(conn, runtime_options.output_mode, &node_info, &status_list);
+		issue_detected = true;
+
+	if (do_node_check_archive_ready(conn, runtime_options.output_mode, &status_list) != CHECK_STATUS_OK)
+		issue_detected = true;
+
+	if (do_node_check_downstream(conn, runtime_options.output_mode, &status_list) != CHECK_STATUS_OK)
+		issue_detected = true;
+
+	if (do_node_check_slots(conn, runtime_options.output_mode, &node_info, &status_list) != CHECK_STATUS_OK)
+		issue_detected = true;
+
+	if (do_node_check_missing_slots(conn, runtime_options.output_mode, &node_info, &status_list) != CHECK_STATUS_OK)
+		issue_detected = true;
 
 	if (runtime_options.output_mode == OM_CSV)
 	{
@@ -786,6 +879,11 @@ do_node_check(void)
 	check_status_list_free(&status_list);
 
 	PQfinish(conn);
+
+	if (issue_detected == true)
+	{
+		exit(ERR_NODE_STATUS);
+	}
 }
 
 
@@ -1047,6 +1145,7 @@ do_node_check_downstream(PGconn *conn, OutputMode mode, CheckStatusList *list_ou
 
 	for (cell = downstream_nodes.head; cell; cell = cell->next)
 	{
+		/* skip witness server */
 		if (cell->node_info->type == WITNESS)
 		{
 			expected_nodes_count --;
@@ -1583,6 +1682,130 @@ do_node_check_slots(PGconn *conn, OutputMode mode, t_node_info *node_info, Check
 }
 
 
+static CheckStatus
+do_node_check_missing_slots(PGconn *conn, OutputMode mode, t_node_info *node_info, CheckStatusList *list_output)
+{
+	CheckStatus status = CHECK_STATUS_OK;
+	PQExpBufferData details;
+	NodeInfoList missing_slots = T_NODE_INFO_LIST_INITIALIZER;
+
+	if (mode == OM_CSV && list_output == NULL)
+	{
+		log_error(_("--csv output not provided with --missing-slots option"));
+		PQfinish(conn);
+		exit(ERR_BAD_CONFIG);
+	}
+
+	initPQExpBuffer(&details);
+
+	if (server_version_num < 90400)
+	{
+		appendPQExpBuffer(&details,
+						  _("replication slots not available for this PostgreSQL version"));
+	}
+	else
+	{
+		get_downstream_nodes_with_missing_slot(conn,
+											   config_file_options.node_id,
+											   &missing_slots);
+
+		if (missing_slots.node_count == 0)
+		{
+			appendPQExpBuffer(&details,
+						  _("node has no missing replication slots"));
+
+		}
+		else
+		{
+			NodeInfoListCell *missing_slot_cell = NULL;
+			bool first_element = true;
+
+			status = CHECK_STATUS_CRITICAL;
+
+			appendPQExpBuffer(&details,
+							  _("%i replication slots are missing"),
+							  missing_slots.node_count);
+
+			if (missing_slots.node_count)
+			{
+				appendPQExpBuffer(&details, ": ");
+
+				for (missing_slot_cell = missing_slots.head; missing_slot_cell; missing_slot_cell = missing_slot_cell->next)
+				{
+					if (first_element == true)
+					{
+						first_element = false;
+					}
+					else
+					{
+						appendPQExpBuffer(&details, ", ");
+					}
+
+					appendPQExpBuffer(&details, "%s", missing_slot_cell->node_info->slot_name);
+				}
+			}
+		}
+	}
+
+	switch (mode)
+	{
+		case OM_NAGIOS:
+		{
+			printf("REPMGR_MISSING_SLOTS %s: %s | missing_slots=%i",
+				   output_check_status(status),
+				   details.data,
+				   missing_slots.node_count);
+
+			if (missing_slots.node_count)
+			{
+				NodeInfoListCell *missing_slot_cell = NULL;
+				bool first_element = true;
+
+				printf(";");
+
+				for (missing_slot_cell = missing_slots.head; missing_slot_cell; missing_slot_cell = missing_slot_cell->next)
+				{
+					if (first_element == true)
+					{
+						first_element = false;
+					}
+					else
+					{
+						printf(",");
+					}
+					printf("%s", missing_slot_cell->node_info->slot_name);
+				}
+			}
+			printf("\n");
+			break;
+		}
+		case OM_CSV:
+		case OM_TEXT:
+			if (list_output != NULL)
+			{
+				check_status_list_set(list_output,
+									  "Replication slots",
+									  status,
+									  details.data);
+			}
+			else
+			{
+				printf("%s (%s)\n",
+					   output_check_status(status),
+					   details.data);
+			}
+		default:
+			break;
+	}
+
+	clear_node_info_list(&missing_slots);
+
+	termPQExpBuffer(&details);
+	return status;
+}
+
+
+
 void
 do_node_service(void)
 {
@@ -2136,19 +2359,19 @@ do_node_rejoin(void)
 			{
 				log_verbose(LOG_INFO, _("waiting for node %i to respond to pings; %i of max %i attempts"),
 							config_file_options.node_id,
-							i + 1, config_file_options.standby_reconnect_timeout);
+							i + 1, config_file_options.node_rejoin_timeout);
 			}
 			else
 			{
 				log_debug("sleeping 1 second waiting for node %i to respond to pings; %i of max %i attempts",
 						  config_file_options.node_id,
-						  i + 1, config_file_options.standby_reconnect_timeout);
+						  i + 1, config_file_options.node_rejoin_timeout);
 			}
 
 			sleep(1);
 		}
 
-		for (;  i < config_file_options.standby_reconnect_timeout; i++)
+		for (;  i < config_file_options.node_rejoin_timeout; i++)
 		{
 			success = is_downstream_node_attached(upstream_conn, config_file_options.node_name);
 
@@ -2163,13 +2386,13 @@ do_node_rejoin(void)
 			{
 				log_info(_("waiting for node %i to connect to new primary; %i of max %i attempts"),
 						 config_file_options.node_id,
-						 i + 1, config_file_options.standby_reconnect_timeout);
+						 i + 1, config_file_options.node_rejoin_timeout);
 			}
 			else
 			{
 				log_debug("sleeping 1 second waiting for node %i to connect to new primary; %i of max %i attempts",
 						  config_file_options.node_id,
-						  i + 1, config_file_options.standby_reconnect_timeout);
+						  i + 1, config_file_options.node_rejoin_timeout);
 			}
 
 			sleep(1);
@@ -2194,6 +2417,54 @@ do_node_rejoin(void)
 		success = is_downstream_node_attached(upstream_conn, config_file_options.node_name);
 	}
 
+	/*
+	 * Handle replication slots:
+	 *  - if a slot for the new upstream exists, delete that
+	 *  - warn about any other inactive replication slots
+	 */
+	if (runtime_options.force_rewind_used == false && config_file_options.use_replication_slots)
+	{
+		PGconn	   *local_conn = NULL;
+		local_conn = establish_db_connection(config_file_options.conninfo, false);
+
+		if (PQstatus(local_conn) != CONNECTION_OK)
+		{
+			log_warning(_("unable to connect to local node to check replication slot status"));
+			log_hint(_("execute \"repmgr node check\" to check inactive slots and drop manually if necessary"));
+		}
+		else
+		{
+			KeyValueList inactive_replication_slots = {NULL, NULL};
+			KeyValueListCell *cell = NULL;
+			int inactive_count = 0;
+			PQExpBufferData slotinfo;
+
+			drop_replication_slot_if_exists(local_conn,
+											config_file_options.node_id,
+											primary_node_record.slot_name);
+
+			(void) get_inactive_replication_slots(local_conn, &inactive_replication_slots);
+
+			initPQExpBuffer(&slotinfo);
+			for (cell = inactive_replication_slots.head; cell; cell = cell->next)
+			{
+				appendPQExpBuffer(&slotinfo,
+								  "  - %s (%s)", cell->key, cell->value);
+				inactive_count++;
+			}
+
+			if (inactive_count > 0)
+			{
+				log_warning(_("%i inactive replication slots detected"), inactive_count);
+				log_detail(_("inactive replication slots:\n%s"), slotinfo.data);
+				log_hint(_("these replication slots may need to be removed manually"));
+			}
+
+			termPQExpBuffer(&slotinfo);
+
+			PQfinish(local_conn);
+		}
+	}
 
 	if (success == true)
 	{
@@ -2203,7 +2474,8 @@ do_node_rejoin(void)
 	else
 	{
 		/*
-		 * if we reach here, no record found in upstream node's pg_stat_replication */
+		 * if we reach here, no record found in upstream node's pg_stat_replication
+		 */
 		log_notice(_("NODE REJOIN has completed but node is not yet reattached to upstream"));
 		log_hint(_("you will need to manually check the node's replication status"));
 	}
@@ -2664,6 +2936,7 @@ do_node_help(void)
 	printf(_("    --replication-lag       replication lag in seconds (standbys only)\n"));
 	printf(_("    --role                  check node has expected role\n"));
 	printf(_("    --slots                 check for inactive replication slots\n"));
+	printf(_("    --missing-slots         check for missing replication slots\n"));
 
 	puts("");
 

repmgr-action-primary.c

@@ -64,12 +64,10 @@ do_primary_register(void)
 			PQfinish(conn);
 			exit(ERR_BAD_CONFIG);
 		}
-		else
+
-		{
+		log_error(_("unable to determine server's recovery type"));
-			log_error(_("connection to node lost"));
+		PQfinish(conn);
-			PQfinish(conn);
+		exit(ERR_DB_CONN);
-			exit(ERR_DB_CONN);
-		}
 	}
 
 	log_verbose(LOG_INFO, _("server is not in recovery"));

repmgr-action-standby.c

@@ -89,8 +89,6 @@ static int	run_file_backup(t_node_info *node_record);
 
 static void copy_configuration_files(bool delete_after_copy);
 
-static void drop_replication_slot_if_exists(PGconn *conn, int node_id, char *slot_name);
-
 static void tablespace_data_append(TablespaceDataList *list, const char *name, const char *oid, const char *location);
 
 static void get_barman_property(char *dst, char *name, char *local_repmgr_directory);
@@ -471,6 +469,7 @@ do_standby_clone(void)
 			termPQExpBuffer(&msg);
 
 			r = test_ssh_connection(runtime_options.host, runtime_options.remote_user);
+
 			if (r != 0)
 			{
 				log_error(_("remote host \"%s\" is not reachable via SSH - unable to copy external configuration files"),
@@ -498,32 +497,41 @@ do_standby_clone(void)
 
 			termPQExpBuffer(&msg);
 
+
 			/*
 			 * Here we'll attempt an initial test copy of the detected external
 			 * files, to detect any issues before we run the base backup.
 			 *
 			 * Note this will exit with an error, unless -F/--force supplied.
 			 *
+			 * We don't do this during a --dry-run as it may introduce unexpected changes
+			 * on the local node; during an actual clone operation, any problems with
+			 * copying files will be detected early and the operation aborted before
+			 * the actual database cloning commences.
+			 *
 			 * TODO: put the files in a temporary directory and move to their final
 			 * destination once the database has been cloned.
 			 */
 
-			if (runtime_options.copy_external_config_files_destination == CONFIG_FILE_SAMEPATH)
+			if (runtime_options.dry_run == false)
 			{
-				/*
+				if (runtime_options.copy_external_config_files_destination == CONFIG_FILE_SAMEPATH)
-				 * Files will be placed in the same path as on the source server;
+				{
-				 * don't delete after copying.
+					/*
-				 */
+					 * Files will be placed in the same path as on the source server;
-				copy_configuration_files(false);
+					 * don't delete after copying.
+					 */
+					copy_configuration_files(false);
 
-			}
+				}
-			else
+				else
-			{
+				{
-				/*
+					/*
-				 * Files will be placed in the data directory - delete after copying.
+					 * Files will be placed in the data directory - delete after copying.
-				 * They'll be copied again later; see TODO above.
+					 * They'll be copied again later; see TODO above.
-				 */
+					 */
-				copy_configuration_files(true);
+					copy_configuration_files(true);
+				}
 			}
 		}
 
@@ -1054,6 +1062,7 @@ _do_create_recovery_conf(void)
 								  local_node_record.slot_name,
 								  upstream_node_record.node_name,
 								  upstream_node_id);
+
 				if (runtime_options.force == false && runtime_options.dry_run == false)
 				{
 					log_error("%s", msg.data);
@@ -1085,7 +1094,7 @@ _do_create_recovery_conf(void)
 				initPQExpBuffer(&msg);
 
 				appendPQExpBuffer(&msg,
-								  _("insufficient free replicaiton slots on upstream node \"%s\" (ID: %i)"),
+								  _("insufficient free replication slots on upstream node \"%s\" (ID: %i)"),
 								  upstream_node_record.node_name,
 								  upstream_node_id);
 
@@ -1141,14 +1150,14 @@ _do_create_recovery_conf(void)
 	if (runtime_options.dry_run == true)
 	{
 		char		recovery_conf_contents[MAXLEN] = "";
-		create_recovery_file(&upstream_node_record, &recovery_conninfo, recovery_conf_contents, false);
+		create_recovery_file(&local_node_record, &recovery_conninfo, recovery_conf_contents, false);
 
 		log_info(_("would create \"recovery.conf\" file in \"%s\""), local_data_directory);
 		log_detail(_("\n%s"), recovery_conf_contents);
 	}
 	else
 	{
-		if (!create_recovery_file(&upstream_node_record, &recovery_conninfo, local_data_directory, true))
+		if (!create_recovery_file(&local_node_record, &recovery_conninfo, local_data_directory, true))
 		{
 			log_error(_("unable to create \"recovery.conf\""));
 		}
@@ -1557,8 +1566,8 @@ do_standby_register(void)
 						exit(ERR_BAD_CONFIG);
 					}
 					log_warning(_("this node does not appear to be attached to upstream node \"%s\" (ID: %i)"),
-								config_file_options.node_name,
+								upstream_node_record.node_name,
-								config_file_options.node_id);
+								upstream_node_record.node_id);
 				}
 				PQfinish(upstream_conn);
 			}
@@ -1708,11 +1717,16 @@ do_standby_register(void)
 
 	termPQExpBuffer(&details);
 
-	/* if --wait-sync option set, wait for the records to synchronise */
+	/*
+	 * If --wait-sync option set, wait for the records to synchronise
+	 * (unless 0 seconds provided, which disables it, which is the same as
+	 *  not providing the option). The default value is -1, which means
+	 * no timeout.
+	 */
 
 	if (PQstatus(conn) == CONNECTION_OK &&
 		runtime_options.wait_register_sync == true &&
-		runtime_options.wait_register_sync_seconds > 0)
+		runtime_options.wait_register_sync_seconds != 0)
 	{
 		bool		sync_ok = false;
 		int			timer = 0;
@@ -1736,7 +1750,11 @@ do_standby_register(void)
 		{
 			bool		records_match = true;
 
-			if (runtime_options.wait_register_sync_seconds && runtime_options.wait_register_sync_seconds == timer)
+			/*
+			 * If timeout set to a positive value, check if we've reached it and
+			 * exit the loop
+			 */
+			if (runtime_options.wait_register_sync_seconds > 0 && runtime_options.wait_register_sync_seconds == timer)
 				break;
 
 			node_record_status = get_node_record(conn,
@@ -2040,6 +2058,8 @@ _do_standby_promote_internal(PGconn *conn)
 			   local_node_record.node_name,
 			   local_node_record.node_id,
 			   script);
+	log_detail(_("waiting up to %i seconds (parameter \"promote_check_timeout\") for promotion to complete"),
+			   config_file_options.promote_check_timeout);
 
 	r = system(script);
 	if (r != 0)
@@ -2065,6 +2085,8 @@ _do_standby_promote_internal(PGconn *conn)
 		if (recovery_type == RECTYPE_STANDBY)
 		{
 			log_error(_("STANDBY PROMOTE failed, node is still a standby"));
+			log_detail(_("node still in recovery after %i seconds"), config_file_options.promote_check_timeout);
+			log_hint(_("the node may need more time to promote itself, check the PostgreSQL log for details"));
 			PQfinish(conn);
 			exit(ERR_PROMOTION_FAIL);
 		}
@@ -2710,6 +2732,10 @@ do_standby_follow_internal(PGconn *primary_conn, t_node_info *primary_node_recor
 	 * If replication slots are in use, and an inactive one for this node
 	 * exists on the former upstream, drop it.
 	 *
+	 * Note that if this function is called by do_standby_switchover(), the
+	 * "repmgr node rejoin" command executed on the demotion candidate may already
+	 * have removed the slot, so there may be nothing to do.
+	 *
 	 * XXX check if former upstream is current primary?
 	 */
 
@@ -2817,6 +2843,12 @@ do_standby_switchover(void)
 	int			reachable_sibling_nodes_with_slot_count = 0;
 	int			unreachable_sibling_node_count = 0;
 
+	/* number of free walsenders required on promotion candidate */
+	int			min_required_wal_senders = 1;
+
+	/* this will be calculated as max_wal_senders - COUNT(*) FROM pg_stat_replication */
+	int			available_wal_senders = 0;
+
 	/* number of free replication slots required on promotion candidate */
 	int			min_required_free_slots = 0;
 
@@ -2901,6 +2933,25 @@ do_standby_switchover(void)
 		exit(ERR_DB_QUERY);
 	}
 
+	/*
+	 * Check that there's no exclusive backups running on the primary.
+	 * We don't want to end up damaging the backup and also leaving the server in an
+	 * state where there's control data saying it's in backup mode but there's no
+	 * backup_label in PGDATA.
+	 * If the DBA wants to do the switchover anyway, he should first stop the
+	 * backup that's running.
+	 */
+	if (server_in_exclusive_backup_mode(remote_conn) != BACKUP_STATE_NO_BACKUP)
+	{
+		log_error(_("unable to perform a switchover while primary server is in exclusive backup mode"));
+		log_hint(_("stop backup before attempting the switchover"));
+
+		PQfinish(local_conn);
+		PQfinish(remote_conn);
+
+		exit(ERR_SWITCHOVER_FAIL);
+	}
+
 	/*
 	 * Check this standby is attached to the demotion candidate
 	 * TODO:
@@ -3067,6 +3118,176 @@ do_standby_switchover(void)
 	}
 	termPQExpBuffer(&command_output);
 
+	/*
+	 * populate local node record with current state of various replication-related
+	 * values, so we can check for sufficient walsenders and replication slots
+	 */
+	get_node_replication_stats(local_conn, server_version_num, &local_node_record);
+
+	available_wal_senders = local_node_record.max_wal_senders -
+		local_node_record.attached_wal_receivers;
+
+	/*
+	 * If --siblings-follow specified, get list and check they're reachable
+	 * (if not just issue a warning)
+	 */
+	get_active_sibling_node_records(local_conn,
+									local_node_record.node_id,
+									local_node_record.upstream_node_id,
+									&sibling_nodes);
+
+	if (runtime_options.siblings_follow == false)
+	{
+		if (sibling_nodes.node_count > 0)
+		{
+			log_warning(_("%i sibling nodes found, but option \"--siblings-follow\" not specified"),
+						sibling_nodes.node_count);
+			log_detail(_("these nodes will remain attached to the current primary"));
+		}
+	}
+	else
+	{
+		char		host[MAXLEN] = "";
+		NodeInfoListCell *cell;
+
+		log_verbose(LOG_INFO, _("%i active sibling nodes found"),
+					sibling_nodes.node_count);
+
+		if (sibling_nodes.node_count == 0)
+		{
+			log_warning(_("option \"--sibling-nodes\" specified, but no sibling nodes exist"));
+		}
+		else
+		{
+			/* include walsender for promotion candidate in total */
+
+
+			for (cell = sibling_nodes.head; cell; cell = cell->next)
+			{
+				/* get host from node record */
+				get_conninfo_value(cell->node_info->conninfo, "host", host);
+				r = test_ssh_connection(host, runtime_options.remote_user);
+
+				if (r != 0)
+				{
+					cell->node_info->reachable = false;
+					unreachable_sibling_node_count++;
+				}
+				else
+				{
+					cell->node_info->reachable = true;
+					reachable_sibling_node_count++;
+					min_required_wal_senders++;
+
+					if (cell->node_info->slot_name[0] != '\0')
+					{
+						reachable_sibling_nodes_with_slot_count++;
+						min_required_free_slots++;
+					}
+				}
+			}
+
+			if (unreachable_sibling_node_count > 0)
+			{
+				if (runtime_options.force == false)
+				{
+					log_error(_("%i of %i sibling nodes unreachable via SSH:"),
+							  unreachable_sibling_node_count,
+							  sibling_nodes.node_count);
+				}
+				else
+				{
+					log_warning(_("%i of %i sibling nodes unreachable via SSH:"),
+								unreachable_sibling_node_count,
+								sibling_nodes.node_count);
+				}
+
+				/* display list of unreachable sibling nodes */
+				for (cell = sibling_nodes.head; cell; cell = cell->next)
+				{
+					if (cell->node_info->reachable == true)
+						continue;
+					log_detail("  %s (ID: %i)",
+							   cell->node_info->node_name,
+							   cell->node_info->node_id);
+				}
+
+				if (runtime_options.force == false)
+				{
+					log_hint(_("use -F/--force to proceed in any case"));
+					PQfinish(local_conn);
+					exit(ERR_BAD_CONFIG);
+				}
+
+				if (runtime_options.dry_run == true)
+				{
+					log_detail(_("F/--force specified, would proceed anyway"));
+				}
+				else
+				{
+					log_detail(_("F/--force specified, proceeding anyway"));
+				}
+			}
+			else
+			{
+				char	   *msg = _("all sibling nodes are reachable via SSH");
+
+				if (runtime_options.dry_run == true)
+				{
+					log_info("%s", msg);
+				}
+				else
+				{
+					log_verbose(LOG_INFO, "%s", msg);
+				}
+			}
+		}
+	}
+
+
+	/*
+	 * check there are sufficient free walsenders - obviously there's potential
+	 * for a later race condition if some walsenders come into use before the
+	 * switchover operation gets around to attaching the sibling nodes, but
+	 * this should catch any actual existing configuration issue (and if anyone's
+	 * performing a switchover in such an unstable environment, they only have
+	 * themselves to blame).
+	 */
+	if (available_wal_senders < min_required_wal_senders)
+	{
+		if (runtime_options.force == false || runtime_options.dry_run == true)
+		{
+			log_error(_("insufficient free walsenders on promotion candidate"));
+			log_detail(_("at least %i walsenders required but only %i free walsenders on promotion candidate"),
+					   min_required_wal_senders,
+					   available_wal_senders);
+			log_hint(_("increase parameter \"max_wal_senders\" or use -F/--force to proceed in any case"));
+
+			if (runtime_options.dry_run == false)
+			{
+				PQfinish(local_conn);
+				exit(ERR_BAD_CONFIG);
+			}
+		}
+		else
+		{
+			log_warning(_("insufficient free walsenders on promotion candidate"));
+			log_detail(_("at least %i walsenders required but only %i free walsender(s) on promotion candidate"),
+					   min_required_wal_senders,
+					   available_wal_senders);
+		}
+	}
+	else
+	{
+		if (runtime_options.dry_run == true)
+		{
+			log_info(_("%i walsenders required, %i available"),
+					 min_required_wal_senders,
+					 available_wal_senders);
+		}
+	}
+
+
 	/* check demotion candidate can make replication connection to promotion candidate */
 	{
 		initPQExpBuffer(&remote_command_str);
@@ -3310,171 +3531,6 @@ do_standby_switchover(void)
 
 	PQfinish(remote_conn);
 
-	/*
-	 * populate local node record with current state of various replication-related
-	 * values, so we can check for sufficient walsenders and replication slots
-	 */
-	get_node_replication_stats(local_conn, server_version_num, &local_node_record);
-
-	/*
-	 * If --siblings-follow specified, get list and check they're reachable
-	 * (if not just issue a warning)
-	 */
-	get_active_sibling_node_records(local_conn,
-									local_node_record.node_id,
-									local_node_record.upstream_node_id,
-									&sibling_nodes);
-
-	if (runtime_options.siblings_follow == false)
-	{
-		if (sibling_nodes.node_count > 0)
-		{
-			log_warning(_("%i sibling nodes found, but option \"--siblings-follow\" not specified"),
-						sibling_nodes.node_count);
-			log_detail(_("these nodes will remain attached to the current primary"));
-		}
-	}
-	else
-	{
-		char		host[MAXLEN] = "";
-		NodeInfoListCell *cell;
-
-		log_verbose(LOG_INFO, _("%i active sibling nodes found"),
-					sibling_nodes.node_count);
-
-		if (sibling_nodes.node_count == 0)
-		{
-			log_warning(_("option \"--sibling-nodes\" specified, but no sibling nodes exist"));
-		}
-		else
-		{
-			/* include walsender for promotion candidate in total */
-			int			min_required_wal_senders = 1;
-			int			available_wal_senders = local_node_record.max_wal_senders -
-				local_node_record.attached_wal_receivers;
-
-			for (cell = sibling_nodes.head; cell; cell = cell->next)
-			{
-				/* get host from node record */
-				get_conninfo_value(cell->node_info->conninfo, "host", host);
-				r = test_ssh_connection(host, runtime_options.remote_user);
-
-				if (r != 0)
-				{
-					cell->node_info->reachable = false;
-					unreachable_sibling_node_count++;
-				}
-				else
-				{
-					cell->node_info->reachable = true;
-					reachable_sibling_node_count++;
-					min_required_wal_senders++;
-
-					if (cell->node_info->slot_name[0] != '\0')
-					{
-						reachable_sibling_nodes_with_slot_count++;
-						min_required_free_slots++;
-					}
-				}
-			}
-
-			if (unreachable_sibling_node_count > 0)
-			{
-				if (runtime_options.force == false)
-				{
-					log_error(_("%i of %i sibling nodes unreachable via SSH:"),
-							  unreachable_sibling_node_count,
-							  sibling_nodes.node_count);
-				}
-				else
-				{
-					log_warning(_("%i of %i sibling nodes unreachable via SSH:"),
-								unreachable_sibling_node_count,
-								sibling_nodes.node_count);
-				}
-
-				/* display list of unreachable sibling nodes */
-				for (cell = sibling_nodes.head; cell; cell = cell->next)
-				{
-					if (cell->node_info->reachable == true)
-						continue;
-					log_detail("  %s (ID: %i)",
-							   cell->node_info->node_name,
-							   cell->node_info->node_id);
-				}
-
-				if (runtime_options.force == false)
-				{
-					log_hint(_("use -F/--force to proceed in any case"));
-					PQfinish(local_conn);
-					exit(ERR_BAD_CONFIG);
-				}
-
-				if (runtime_options.dry_run == true)
-				{
-					log_detail(_("F/--force specified, would proceed anyway"));
-				}
-				else
-				{
-					log_detail(_("F/--force specified, proceeding anyway"));
-				}
-			}
-			else
-			{
-				char	   *msg = _("all sibling nodes are reachable via SSH");
-
-				if (runtime_options.dry_run == true)
-				{
-					log_info("%s", msg);
-				}
-				else
-				{
-					log_verbose(LOG_INFO, "%s", msg);
-				}
-			}
-
-			/*
-			 * check there are sufficient free walsenders - obviously there's potential
-			 * for a later race condition if some walsenders come into use before the
-			 * switchover operation gets around to attaching the sibling nodes, but
-			 * this should catch any actual existing configuration issue.
-			 */
-			if (available_wal_senders < min_required_wal_senders)
-			{
-				if (runtime_options.force == false || runtime_options.dry_run == true)
-				{
-					log_error(_("insufficient free walsenders to attach all sibling nodes"));
-					log_detail(_("at least %i walsenders required but only %i free walsenders on promotion candidate"),
-							   min_required_wal_senders,
-							   available_wal_senders);
-					log_hint(_("increase parameter \"max_wal_senders\" or use -F/--force to proceed in any case"));
-
-					if (runtime_options.dry_run == false)
-					{
-						PQfinish(local_conn);
-						exit(ERR_BAD_CONFIG);
-					}
-				}
-				else
-				{
-					log_warning(_("insufficient free walsenders to attach all sibling nodes"));
-					log_detail(_("at least %i walsenders required but only %i free walsender(s) on promotion candidate"),
-							   min_required_wal_senders,
-							   available_wal_senders);
-				}
-			}
-			else
-			{
-				if (runtime_options.dry_run == true)
-				{
-					log_info(_("%i walsenders required, %i available"),
-							 min_required_wal_senders,
-							 available_wal_senders);
-				}
-			}
-		}
-	}
-
 
 	/*
 	 * if replication slots are required by demotion candidate and/or siblings,
@@ -5082,65 +5138,81 @@ run_basebackup(t_node_info *node_record)
 		{
 			PGconn	   *upstream_conn = NULL;
 
-			upstream_conn = establish_db_connection(upstream_node_record.conninfo, true);
+			upstream_conn = establish_db_connection(upstream_node_record.conninfo, false);
 
-			record_status = get_slot_record(upstream_conn, node_record->slot_name, &slot_info);
+			/*
-
+			 * It's possible the upstream node is not yet running, in which case we'll
-			if (record_status == RECORD_FOUND)
+			 * have to rely on the user taking action to create the slot
+			 */
+			if (PQstatus(upstream_conn) != CONNECTION_OK)
 			{
-				log_verbose(LOG_INFO,
+				log_warning(_("unable to connect to upstream node to create replication slot"));
-							_("replication slot \"%s\" aleady exists on upstream node %i"),
+				/*
-							node_record->slot_name,
+				 * TODO: if slot creation also handled by "standby register", update warning
-							upstream_node_id);
+				 */
-				slot_exists_on_upstream = true;
+				log_hint(_("you may need to create the replication slot manually"));
 			}
 			else
 			{
-				PQExpBufferData event_details;
+				record_status = get_slot_record(upstream_conn, node_record->slot_name, &slot_info);
 
-				log_notice(_("creating replication slot \"%s\" on upstream node %i"),
+				if (record_status == RECORD_FOUND)
-						   node_record->slot_name,
-						   upstream_node_id);
-
-				get_superuser_connection(&upstream_conn, &superuser_conn, &privileged_conn);
-
-				initPQExpBuffer(&event_details);
-				if (create_replication_slot(privileged_conn, node_record->slot_name, source_server_version_num, &event_details) == false)
 				{
-					log_error("%s", event_details.data);
+					log_verbose(LOG_INFO,
+								_("replication slot \"%s\" aleady exists on upstream node %i"),
+								node_record->slot_name,
+								upstream_node_id);
+					slot_exists_on_upstream = true;
+				}
+				else
+				{
+					PQExpBufferData event_details;
 
-					create_event_notification(
+					log_notice(_("creating replication slot \"%s\" on upstream node %i"),
-											  primary_conn,
+							   node_record->slot_name,
-											  &config_file_options,
+							   upstream_node_id);
-											  config_file_options.node_id,
-											  "standby_clone",
-											  false,
-											  event_details.data);
 
-					PQfinish(source_conn);
+					get_superuser_connection(&upstream_conn, &superuser_conn, &privileged_conn);
+
+					initPQExpBuffer(&event_details);
+					if (create_replication_slot(privileged_conn, node_record->slot_name, source_server_version_num, &event_details) == false)
+					{
+						log_error("%s", event_details.data);
+
+						create_event_notification(primary_conn,
+												  &config_file_options,
+												  config_file_options.node_id,
+												  "standby_clone",
+												  false,
+												  event_details.data);
+
+						PQfinish(source_conn);
+
+						if (superuser_conn != NULL)
+							PQfinish(superuser_conn);
+
+						exit(ERR_DB_QUERY);
+					}
 
 					if (superuser_conn != NULL)
 						PQfinish(superuser_conn);
 
-					exit(ERR_DB_QUERY);
+					termPQExpBuffer(&event_details);
 				}
 
-				if (superuser_conn != NULL)
+				PQfinish(upstream_conn);
-					PQfinish(superuser_conn);
-
-				termPQExpBuffer(&event_details);
 			}
-
-			PQfinish(upstream_conn);
 		}
 
+		/* delete slot on source server */
+
 		get_superuser_connection(&source_conn, &superuser_conn, &privileged_conn);
 
 		if (slot_info.active == false)
 		{
 			if (slot_exists_on_upstream == false)
 			{
-				if (drop_replication_slot(source_conn, node_record->slot_name) == true)
+				if (drop_replication_slot(privileged_conn, node_record->slot_name) == true)
 				{
 					log_notice(_("replication slot \"%s\" deleted on source node"), node_record->slot_name);
 				}
@@ -5798,7 +5870,7 @@ get_barman_property(char *dst, char *name, char *local_repmgr_directory)
 	initPQExpBuffer(&command_output);
 
 	maxlen_snprintf(command,
-					"grep \"^\t%s:\" %s/show-server.txt",
+					"grep \"^[[:space:]]%s:\" %s/show-server.txt",
 					name, local_repmgr_tmp_directory);
 	(void) local_command(command, &command_output);
 
@@ -5995,45 +6067,6 @@ check_recovery_type(PGconn *conn)
 }
 
 
-static void
-drop_replication_slot_if_exists(PGconn *conn, int node_id, char *slot_name)
-{
-	t_replication_slot slot_info = T_REPLICATION_SLOT_INITIALIZER;
-	RecordStatus record_status = get_slot_record(conn, slot_name, &slot_info);
-
-	log_verbose(LOG_DEBUG, "attempting to delete slot \"%s\" on node %i",
-				slot_name, node_id);
-
-	if (record_status != RECORD_FOUND)
-	{
-		log_info(_("no slot record found for slot \"%s\" on node %i"),
-				 slot_name, node_id);
-	}
-	else
-	{
-		if (slot_info.active == false)
-		{
-			if (drop_replication_slot(conn, slot_name) == true)
-			{
-				log_notice(_("replication slot \"%s\" deleted on node %i"), slot_name, node_id);
-			}
-			else
-			{
-				log_error(_("unable to delete replication slot \"%s\" on node %i"), slot_name, node_id);
-			}
-		}
-
-		/*
-		 * if active replication slot exists, call Houston as we have a
-		 * problem
-		 */
-		else
-		{
-			log_warning(_("replication slot \"%s\" is still active on node %i"), slot_name, node_id);
-		}
-	}
-}
-
 
 /*
  * Creates a recovery.conf file for a standby
@@ -6504,6 +6537,7 @@ do_standby_help(void)
 	puts("");
 	printf(_("  \"standby clone\" clones a standby from the primary or an upstream node.\n"));
 	puts("");
+	printf(_("  -d, --dbname=conninfo               conninfo of the upstream node to use for cloning.\n"));
 	printf(_("  -c, --fast-checkpoint               force fast checkpoint\n"));
 	printf(_("  --copy-external-config-files[={samepath|pgdata}]\n" \
 			 "                                      copy configuration files located outside the \n" \

repmgr-action-witness.c

@@ -310,55 +310,59 @@ do_witness_register(void)
 void
 do_witness_unregister(void)
 {
-	PGconn	   *witness_conn = NULL;
+	PGconn	   *local_conn = NULL;
 	PGconn	   *primary_conn = NULL;
 	t_node_info node_record = T_NODE_INFO_INITIALIZER;
 	RecordStatus record_status = RECORD_NOT_FOUND;
 	bool		node_record_deleted = false;
-	bool		witness_available = true;
+	bool		local_node_available = true;
+	int			witness_node_id = UNKNOWN_NODE_ID;
 
-	log_info(_("connecting to witness node \"%s\" (ID: %i)"),
+	if (runtime_options.node_id != UNKNOWN_NODE_ID)
+	{
+		/* user has specified the witness node id */
+		witness_node_id = runtime_options.node_id;
+	}
+	else
+	{
+		/* assume witness node is local node */
+		witness_node_id = config_file_options.node_id;
+	}
+
+	log_info(_("connecting to node \"%s\" (ID: %i)"),
 			 config_file_options.node_name,
 			 config_file_options.node_id);
 
-	witness_conn = establish_db_connection_quiet(config_file_options.conninfo);
+	local_conn = establish_db_connection_quiet(config_file_options.conninfo);
 
-	if (PQstatus(witness_conn) != CONNECTION_OK)
+	if (PQstatus(local_conn) != CONNECTION_OK)
 	{
 		if (!runtime_options.force)
 		{
-			log_error(_("unable to connect to witness node \"%s\" (ID: %i)"),
+			log_error(_("unable to connect to node \"%s\" (ID: %i)"),
 					  config_file_options.node_name,
 					  config_file_options.node_id);
-			log_detail("%s", PQerrorMessage(witness_conn));
+			log_detail("%s", PQerrorMessage(local_conn));
-			log_hint(_("provide -F/--force to remove the witness record if the server is not running"));
 			exit(ERR_BAD_CONFIG);
 		}
 
 		log_notice(_("unable to connect to witness node \"%s\" (ID: %i), removing node record on cluster primary only"),
 				   config_file_options.node_name,
 				   config_file_options.node_id);
-		witness_available = false;
+		local_node_available = false;
 	}
 
-	if (witness_available == true)
+	if (local_node_available == true)
 	{
-		primary_conn = get_primary_connection_quiet(witness_conn, NULL, NULL);
+		primary_conn = get_primary_connection_quiet(local_conn, NULL, NULL);
 	}
 	else
 	{
 		/*
-		 * Extract the repmgr user and database names from the conninfo string
+		 * Assume user has provided connection details for the primary server
-		 * provided in repmgr.conf
 		 */
-		get_conninfo_value(config_file_options.conninfo, "user", repmgr_user);
-		get_conninfo_value(config_file_options.conninfo, "dbname", repmgr_db);
-
-		param_set_ine(&source_conninfo, "user", repmgr_user);
-		param_set_ine(&source_conninfo, "dbname", repmgr_db);
 
 		primary_conn = establish_db_connection_by_params(&source_conninfo, false);
-
 	}
 
 	if (PQstatus(primary_conn) != CONNECTION_OK)
@@ -366,26 +370,26 @@ do_witness_unregister(void)
 		log_error(_("unable to connect to primary"));
 		log_detail("%s", PQerrorMessage(primary_conn));
 
-		if (witness_available == true)
+		if (local_node_available == true)
 		{
-			PQfinish(witness_conn);
+			PQfinish(local_conn);
 		}
-		else
+		else if (runtime_options.connection_param_provided == false)
 		{
-			log_hint(_("provide connection details to primary server"));
+			log_hint(_("provide connection details for the primary server"));
 		}
 		exit(ERR_BAD_CONFIG);
 	}
 
 	/* Check node exists and is really a witness */
-	record_status = get_node_record(primary_conn, config_file_options.node_id, &node_record);
+	record_status = get_node_record(primary_conn, witness_node_id, &node_record);
 
 	if (record_status != RECORD_FOUND)
 	{
-		log_error(_("no record found for node %i"), config_file_options.node_id);
+		log_error(_("no record found for node %i"), witness_node_id);
 
-		if (witness_available == true)
+		if (local_node_available == true)
-			PQfinish(witness_conn);
+			PQfinish(local_conn);
 		PQfinish(primary_conn);
 
 		exit(ERR_BAD_CONFIG);
@@ -393,11 +397,17 @@ do_witness_unregister(void)
 
 	if (node_record.type != WITNESS)
 	{
+		/*
+		 * The node (either explicitly provided with --node-id, or the local node)
+		 * is not a witness.
+		 *
+		 * TODO: scan node list and print hint about identity of known witness servers.
+		 */
 		log_error(_("node %i is not a witness node"), config_file_options.node_id);
 		log_detail(_("node %i is a %s node"), config_file_options.node_id, get_node_type_string(node_record.type));
 
-		if (witness_available == true)
+		if (local_node_available == true)
-			PQfinish(witness_conn);
+			PQfinish(local_conn);
 		PQfinish(primary_conn);
 
 		exit(ERR_BAD_CONFIG);
@@ -406,49 +416,43 @@ do_witness_unregister(void)
 	if (runtime_options.dry_run == true)
 	{
 		log_info(_("prerequisites for unregistering the witness node are met"));
-		if (witness_available == true)
+		if (local_node_available == true)
-			PQfinish(witness_conn);
+			PQfinish(local_conn);
 		PQfinish(primary_conn);
 
 		exit(SUCCESS);
 	}
 
-	log_info(_("unregistering witness node %i"), config_file_options.node_id);
+	log_info(_("unregistering witness node %i"), witness_node_id);
 	node_record_deleted = delete_node_record(primary_conn,
-										     config_file_options.node_id);
+										     witness_node_id);
 
 	if (node_record_deleted == false)
 	{
 		PQfinish(primary_conn);
-		PQfinish(witness_conn);
-		exit(ERR_BAD_CONFIG);
-	}
 
-	/* sync records from primary */
+		if (local_node_available == true)
-	if (witness_available == true && witness_copy_node_records(primary_conn, witness_conn) == false)
+			PQfinish(local_conn);
-	{
+		PQfinish(local_conn);
-		log_error(_("unable to copy repmgr node records from primary"));
-		PQfinish(primary_conn);
-		PQfinish(witness_conn);
 		exit(ERR_BAD_CONFIG);
 	}
 
 	/* Log the event */
 	create_event_record(primary_conn,
 						&config_file_options,
-						config_file_options.node_id,
+						witness_node_id,
 						"witness_unregister",
 						true,
 						NULL);
 
 	PQfinish(primary_conn);
 
-	if (witness_available == true)
+	if (local_node_available == true)
-		PQfinish(witness_conn);
+		PQfinish(local_conn);
 
 	log_info(_("witness unregistration complete"));
-	log_detail(_("witness node with id %i (conninfo: %s) successfully unregistered"),
+	log_detail(_("witness node with ID %i successfully unregistered"),
-			    config_file_options.node_id, config_file_options.conninfo);
+			    witness_node_id);
 
 	return;
 }
@@ -468,16 +472,19 @@ void do_witness_help(void)
 	puts("");
 	printf(_("  Requires provision of connection information for the primary\n"));
 	puts("");
-	printf(_("  --dry-run                           check prerequisites but don't make any changes\n"));
+	printf(_("  --dry-run                check prerequisites but don't make any changes\n"));
-	printf(_("  -F, --force                         overwrite an existing node record\n"));
+	printf(_("  -F, --force              overwrite an existing node record\n"));
 	puts("");
 
 	printf(_("WITNESS UNREGISTER\n"));
 	puts("");
 	printf(_("  \"witness register\" unregisters a witness node.\n"));
 	puts("");
-	printf(_("  --dry-run                           check prerequisites but don't make any changes\n"));
+	printf(_("  --dry-run                check prerequisites but don't make any changes\n"));
-	printf(_("  -F, --force                         unregister when witness node not running\n"));
+	printf(_("  -F, --force              unregister when witness node not running\n"));
+	printf(_("  --node-id                node ID of the witness node (provide if executing on\n"));
+	printf(_("                             another node)\n"));
+
 	puts("");
 
 	return;

repmgr-client-global.h

@@ -47,6 +47,7 @@ typedef struct
 	/* logging options */
 	char		log_level[MAXLEN];	/* overrides setting in repmgr.conf */
 	bool		log_to_file;
+	bool		quiet;
 	bool		terse;
 	bool		verbose;
 
@@ -106,6 +107,7 @@ typedef struct
 	bool		replication_lag;
 	bool		role;
 	bool		slots;
+	bool		missing_slots;
 	bool		has_passfile;
 	bool		replication_connection;
 
@@ -137,7 +139,7 @@ typedef struct
 		/* general configuration options */	\
 		"", false, false, "", false, false,	\
 		/* logging options */ \
-		"", false, false, false, \
+		"", false, false, false, false,	\
 		/* output options */ \
 		false, false, false,  \
 		/* database connection options */ \
@@ -152,13 +154,13 @@ typedef struct
 		/* "standby clone"/"standby follow" options */ \
 		NO_UPSTREAM_NODE, \
 		/* "standby register" options */ \
-		false, 0, DEFAULT_WAIT_START,   \
+		false, -1, DEFAULT_WAIT_START,   \
 		/* "standby switchover" options */ \
 		false, false, "", false,		   \
 		/* "node status" options */ \
 		false, \
 		/* "node check" options */ \
-		false, false, false, false, false, false, false, \
+		false, false, false, false, false, false, false, false,	\
 		/* "node join" options */ \
 		"", \
 		/* "node service" options */ \
@@ -235,5 +237,6 @@ extern void get_node_config_directory(char *config_dir_buf);
 extern void get_node_data_directory(char *data_dir_buf);
 extern void init_node_record(t_node_info *node_record);
 extern bool can_use_pg_rewind(PGconn *conn, const char *data_directory, PQExpBufferData *reason);
+extern void drop_replication_slot_if_exists(PGconn *conn, int node_id, char *slot_name);
 
 #endif							/* _REPMGR_CLIENT_GLOBAL_H_ */

repmgr-client.c

@@ -98,7 +98,7 @@ main(int argc, char **argv)
 {
 	t_conninfo_param_list default_conninfo = T_CONNINFO_PARAM_LIST_INITIALIZER;
 
-	int			optindex;
+	int			optindex = 0;
 	int			c;
 
 	char	   *repmgr_command = NULL;
@@ -108,6 +108,7 @@ main(int argc, char **argv)
 	char	   *dummy_action = "";
 
 	bool		help_option = false;
+	bool		option_error_found = false;
 
 	set_progname(argv[0]);
 
@@ -178,7 +179,10 @@ main(int argc, char **argv)
 		strncpy(runtime_options.username, pw->pw_name, MAXLEN);
 	}
 
-	while ((c = getopt_long(argc, argv, "?Vb:f:FwWd:h:p:U:R:S:D:ck:L:tvC:", long_options,
+	/* Make getopt emitting errors */
+	opterr = 1;
+
+	while ((c = getopt_long(argc, argv, "?Vb:f:FwWd:h:p:U:R:S:D:ck:L:qtvC:", long_options,
 							&optindex)) != -1)
 	{
 		/*
@@ -196,13 +200,7 @@ main(int argc, char **argv)
 			case OPT_HELP:		/* --help */
 				help_option = true;
 				break;
-			case '?':
+
-				/* Actual help option given */
-				if (strcmp(argv[optind - 1], "-?") == 0)
-				{
-					help_option = true;
-				}
-				break;
 			case 'V':
 
 				/*
@@ -473,6 +471,10 @@ main(int argc, char **argv)
 				runtime_options.slots = true;
 				break;
 
+			case OPT_MISSING_SLOTS:
+				runtime_options.missing_slots = true;
+				break;
+
 			case OPT_HAS_PASSFILE:
 				runtime_options.has_passfile = true;
 				break;
@@ -572,6 +574,12 @@ main(int argc, char **argv)
 				logger_output_mode = OM_DAEMON;
 				break;
 
+
+				/* --quiet */
+			case 'q':
+				runtime_options.quiet = true;
+				break;
+
 				/* --terse */
 			case 't':
 				runtime_options.terse = true;
@@ -627,9 +635,24 @@ main(int argc, char **argv)
 								 _("--recovery-min-apply-delay is now a configuration file parameter, \"recovery_min_apply_delay\""));
 				break;
 
+			case ':':   /* missing option argument */
+				option_error_found = true;
+				break;
+			case '?':
+				/* Actual help option given? */
+				if (strcmp(argv[optind - 1], "-?") == 0)
+				{
+					help_option = true;
+					break;
+				}
+				/* otherwise fall through to default */
+			default:    /* invalid option */
+				option_error_found = true;
+				break;
 		}
 	}
 
+
 	/*
 	 * If -d/--dbname appears to be a conninfo string, validate by attempting
 	 * to parse it (and if successful, store the parsed parameters)
@@ -730,9 +753,10 @@ main(int argc, char **argv)
 	if (cli_errors.head != NULL)
 	{
 		free_conninfo_params(&source_conninfo);
-		exit_with_cli_errors(&cli_errors);
+		exit_with_cli_errors(&cli_errors, NULL);
 	}
 
+
 	/*----------
 	 * Determine the node type and action; following are valid:
 	 *
@@ -979,9 +1003,30 @@ main(int argc, char **argv)
 	if (cli_errors.head != NULL)
 	{
 		free_conninfo_params(&source_conninfo);
-		exit_with_cli_errors(&cli_errors);
+
+		exit_with_cli_errors(&cli_errors, valid_repmgr_command_found == true ? repmgr_command : NULL);
 	}
 
+	/* no errors detected by repmgr, but getopt might have */
+	if (option_error_found == true)
+	{
+		if (valid_repmgr_command_found == true)
+		{
+			printf(_("Try \"%s --help\" or \"%s %s --help\" for more information.\n"),
+				   progname(),
+				   progname(),
+				   repmgr_command);
+		}
+		else
+		{
+			printf(_("Try \"repmgr --help\" for more information.\n"));
+		}
+
+		free_conninfo_params(&source_conninfo);
+		exit(ERR_BAD_CONFIG);
+	}
+
+
 	/*
 	 * Print any warnings about inappropriate command line options, unless
 	 * -t/--terse set
@@ -1077,6 +1122,17 @@ main(int argc, char **argv)
 		logger_set_min_level(LOG_INFO);
 	}
 
+	/*
+	 * If -q/--quiet supplied, suppress any non-ERROR log output.
+	 * This overrides everything else; we'll leave it up to the user to deal with the
+	 * consequences of e.g. running --dry-run together with -q/--quiet.
+	 */
+	if (runtime_options.quiet == true)
+	{
+		logger_set_level(LOG_ERROR);
+	}
+
+
 
 	/*
 	 * Node configuration information is not needed for all actions, with
@@ -1463,6 +1519,7 @@ check_cli_parameters(const int action)
 		{
 			case PRIMARY_UNREGISTER:
 			case STANDBY_UNREGISTER:
+			case WITNESS_UNREGISTER:
 			case CLUSTER_EVENT:
 			case CLUSTER_MATRIX:
 			case CLUSTER_CROSSCHECK:
@@ -1503,6 +1560,7 @@ check_cli_parameters(const int action)
 			case STANDBY_CLONE:
 			case STANDBY_REGISTER:
 			case STANDBY_FOLLOW:
+			case BDR_REGISTER:
 				break;
 			default:
 				item_list_append_format(&cli_warnings,
@@ -1845,7 +1903,7 @@ do_help(void)
 	printf(_("    %s [OPTIONS] standby {register|unregister|clone|promote|follow|switchover}\n"), progname());
 	printf(_("    %s [OPTIONS] bdr     {register|unregister}\n"), progname());
 	printf(_("    %s [OPTIONS] node    {status|check|rejoin|service}\n"), progname());
-	printf(_("    %s [OPTIONS] cluster {show|event|matrix|crosscheck}\n"), progname());
+	printf(_("    %s [OPTIONS] cluster {show|event|matrix|crosscheck|cleanup}\n"), progname());
 	printf(_("    %s [OPTIONS] witness {register|unregister}\n"), progname());
 
 	puts("");
@@ -1894,6 +1952,7 @@ do_help(void)
 	printf(_("  --dry-run                           show what would happen for action, but don't execute it\n"));
 	printf(_("  -L, --log-level                     set log level (overrides configuration file; default: NOTICE)\n"));
 	printf(_("  --log-to-file                       log to file (or logging facility) defined in repmgr.conf\n"));
+	printf(_("  -q, --quiet                         suppress all log output apart from errors\n"));
 	printf(_("  -t, --terse                         don't display detail, hints and other non-critical output\n"));
 	printf(_("  -v, --verbose                       display additional log output (useful for debugging)\n"));
 
@@ -2919,3 +2978,46 @@ can_use_pg_rewind(PGconn *conn, const char *data_directory, PQExpBufferData *rea
 
 	return can_use;
 }
+
+
+void
+drop_replication_slot_if_exists(PGconn *conn, int node_id, char *slot_name)
+{
+	t_replication_slot slot_info = T_REPLICATION_SLOT_INITIALIZER;
+	RecordStatus record_status = get_slot_record(conn, slot_name, &slot_info);
+
+	log_verbose(LOG_DEBUG, "attempting to delete slot \"%s\" on node %i",
+				slot_name, node_id);
+
+	if (record_status != RECORD_FOUND)
+	{
+		/* this is a good thing */
+		log_verbose(LOG_INFO,
+					_("slot \"%s\" does not exist on node %i, nothing to remove"),
+					slot_name, node_id);
+	}
+	else
+	{
+		if (slot_info.active == false)
+		{
+			if (drop_replication_slot(conn, slot_name) == true)
+			{
+				log_notice(_("replication slot \"%s\" deleted on node %i"), slot_name, node_id);
+			}
+			else
+			{
+				log_error(_("unable to delete replication slot \"%s\" on node %i"), slot_name, node_id);
+			}
+		}
+
+		/*
+		 * if active replication slot exists, call Houston as we have a
+		 * problem
+		 */
+		else
+		{
+			log_warning(_("replication slot \"%s\" is still active on node %i"), slot_name, node_id);
+		}
+	}
+}
+

repmgr-client.h

@@ -87,6 +87,7 @@
 #define OPT_REMOTE_NODE_ID                 1038
 #define OPT_RECOVERY_CONF_ONLY             1039
 #define OPT_NO_WAIT                        1040
+#define OPT_MISSING_SLOTS                  1041
 
 /* deprecated since 3.3 */
 #define OPT_DATA_DIR						999
@@ -125,6 +126,7 @@ static struct option long_options[] =
 /* logging options */
 	{"log-level", required_argument, NULL, 'L'},
 	{"log-to-file", no_argument, NULL, OPT_LOG_TO_FILE},
+	{"quiet",  no_argument, NULL, 'q'},
 	{"terse", no_argument, NULL, 't'},
 	{"verbose", no_argument, NULL, 'v'},
 
@@ -164,6 +166,7 @@ static struct option long_options[] =
 	{"replication-lag", no_argument, NULL, OPT_REPLICATION_LAG},
 	{"role", no_argument, NULL, OPT_ROLE},
 	{"slots", no_argument, NULL, OPT_SLOTS},
+	{"missing-slots", no_argument, NULL, OPT_MISSING_SLOTS},
 	{"has-passfile", no_argument, NULL, OPT_HAS_PASSFILE},
 	{"replication-connection", no_argument, NULL, OPT_REPL_CONN},
 

repmgr.c

@@ -416,9 +416,9 @@ unset_bdr_failover_handler(PG_FUNCTION_ARGS)
 		LWLockAcquire(shared_state->lock, LW_EXCLUSIVE);
 
 		shared_state->bdr_failover_handler = UNKNOWN_NODE_ID;
-
-		LWLockRelease(shared_state->lock);
 	}
 
+	LWLockRelease(shared_state->lock);
+
 	PG_RETURN_VOID();
 }

repmgr.conf.sample

@@ -98,7 +98,7 @@
 #log_facility=STDERR		 # Logging facility: possible values are STDERR, or for
 				 # syslog integration, one of LOCAL0, LOCAL1, ..., LOCAL7, USER
 
-#log_file=''			 # stderr can be redirected to an arbitrary file
+#log_file=''			 # STDERR can be redirected to an arbitrary file
 #log_status_interval=300	 # interval (in seconds) for repmgrd to log a status message
 
 
@@ -143,6 +143,11 @@
 					# Debian/Ubuntu users: you will probably need to
 					# set this to the directory where `pg_ctl` is located,
 					# e.g. /usr/lib/postgresql/9.6/bin/
+					#
+					# *NOTE* "pg_bindir" is only used when repmgr directly
+					# executes PostgreSQL binaries; any user-defined scripts
+					# *must* be specified with the full path
+					#
 #use_primary_conninfo_password=false	# explicitly set "password" in recovery.conf's
 					# "primary_conninfo" parameter using the value contained
 					# in the environment variable PGPASSWORD
@@ -156,7 +161,7 @@
 # Examples:
 #
 #   pg_ctl_options='-s'
-#   pg_basebackup_options='--label=repmgr_backup
+#   pg_basebackup_options='--label=repmgr_backup'
 #   rsync_options=--archive --checksum --compress --progress --rsh="ssh -o \"StrictHostKeyChecking no\""
 #   ssh_options=-o "StrictHostKeyChecking no"
 
@@ -183,11 +188,11 @@ ssh_options='-q -o ConnectTimeout=10'	# Options to append to "ssh"
 					# parameter can be provided multiple times.
 
 #restore_command=''			# This will be placed in the recovery.conf file generated
-                                        # by repmgr.
+					# by repmgr.
 
 #archive_cleanup_command=''		# This will be placed in the recovery.conf file generated
-                                        # by repmgr. Note we recommend using Barman for managing
+					# by repmgr. Note we recommend using Barman for managing
-                                        # WAL archives (see: https://www.pgbarman.org )
+					# WAL archives (see: https://www.pgbarman.org )
 
 #recovery_min_apply_delay=		# If provided, "recovery_min_apply_delay" in recovery.conf
 					# will be set to this value (PostgreSQL 9.4 and later).
@@ -207,7 +212,7 @@ ssh_options='-q -o ConnectTimeout=10'	# Options to append to "ssh"
 
 
 #------------------------------------------------------------------------------
-# Standby follow settings
+# "standby follow" settings
 #------------------------------------------------------------------------------
 
 # These settings apply when instructing a standby to follow the new primary
@@ -219,6 +224,28 @@ ssh_options='-q -o ConnectTimeout=10'	# Options to append to "ssh"
 					# for the standby to connect to the primary
 
 
+#------------------------------------------------------------------------------
+# "standby switchover" settings
+#------------------------------------------------------------------------------
+
+# These settings apply when switching roles between a primary and a standby
+# ("repmgr standby switchover").
+
+#standby_reconnect_timeout=60		# The max length of time (in seconds) to wait
+					# for the demoted standby to reconnect to the promoted
+					# primary (note: this value should be equal to or greater
+					# than that set for "node_rejoin_timeout")
+
+#------------------------------------------------------------------------------
+# "node rejoin" settings
+#------------------------------------------------------------------------------
+
+# These settings apply when reintegrating a node into a replication cluster
+# with "repmgrd_node_rejoin"
+
+#node_rejoin_timeout=60		# The maximum length of time (in seconds) to wait for
+					# the node to reconnect to the replication cluster
+
 #------------------------------------------------------------------------------
 # Barman options
 #------------------------------------------------------------------------------
@@ -236,6 +263,11 @@ ssh_options='-q -o ConnectTimeout=10'	# Options to append to "ssh"
 # These settings are only applied when repmgrd is running. Values shown
 # are defaults.
 
+#repmgrd_pid_file=			# Path of PID file to use for repmgrd; if not set, a PID file will
+					# be generated in a temporary directory specified by the environment
+					# variable $TMPDIR, or if not set, in "/tmp". This value can be overridden
+					# by the command line option "-p/--pid-file"; the command line option
+					# "--no-pid-file" will force PID file creation to be skipped.
 #failover=manual			# one of 'automatic', 'manual'.
 					# determines what action to take in the event of upstream failure
 					#
@@ -245,13 +277,13 @@ ssh_options='-q -o ConnectTimeout=10'	# Options to append to "ssh"
 					#    manual attention to reattach it to replication
 					# (does not apply to BDR mode)
 
-#priority=100				# indicate a preferred priorty for promoting nodes;
+#priority=100				# indicate a preferred priority for promoting nodes;
 					# a value of zero prevents the node being promoted to primary
 					# (default: 100)
 
-#reconnect_attempts=6			# Number attempts which will be made to reconnect to an unreachable
+#reconnect_attempts=6			# Number of attempts which will be made to reconnect to an unreachable
 					# primary (or other upstream node)
-#reconnect_interval=10			# Interval between attempts  to reconnect to an unreachable
+#reconnect_interval=10			# Interval between attempts to reconnect to an unreachable
 					# primary (or other upstream node)
 #promote_command=			# command repmgrd executes when promoting a new primary; use something like:
 					#
@@ -265,8 +297,9 @@ ssh_options='-q -o ConnectTimeout=10'	# Options to append to "ssh"
 #primary_notification_timeout=60	# Interval (in seconds) which repmgrd on a standby
 					# will wait for a notification from the new primary,
 					# before falling back to degraded monitoring
-#standby_reconnect_timeout=60	# Interval (in seconds) which repmgrd on a standby will wait
+#repmgrd_standby_startup_timeout=60	# Interval (in seconds) which repmgrd on a standby will wait
-					# to reconnect to the local node after executing "follow_command"
+					# for the the local node to restart and become ready to accept connections after
+					# executing "follow_command" (defaults to the value set in "standby_reconnect_timeout")
 
 #monitoring_history=no                  # Whether to write monitoring data to the "montoring_history" table
 #monitor_interval_secs=2                # Interval (in seconds) at which to write monitoring data
@@ -304,7 +337,7 @@ ssh_options='-q -o ConnectTimeout=10'	# Options to append to "ssh"
 #
 # Debian/Ubuntu users: use "sudo pg_ctlcluster" to execute service control commands.
 #
-# For more details, see: https://repmgr.org/docs/4.0/configuration-service-commands.html
+# For more details, see: https://repmgr.org/docs/4.1/configuration-service-commands.html
 
 #service_start_command = ''
 #service_stop_command = ''
@@ -348,7 +381,7 @@ ssh_options='-q -o ConnectTimeout=10'	# Options to append to "ssh"
 #------------------------------------------------------------------------------
 
 #bdr_local_monitoring_only=false         # Only monitor the local node; no checks will be
-                                         # performed on the other node
+					 # performed on the other node
 #bdr_recovery_timeout                    # If a BDR node was offline and has become available
-                                         # maximum length of time in seconds to wait for the
+					 # maximum length of time in seconds to wait for the
-                                         # node to reconnect to the cluster
+					 # node to reconnect to the cluster

repmgr.control

@@ -1,6 +1,6 @@
 # repmgr extension
 comment = 'Replication manager for PostgreSQL'
-default_version = '4.0'
+default_version = '4.1'
 module_pathname = '$libdir/repmgr'
 relocatable = false
 schema = repmgr

repmgr.h

@@ -49,6 +49,8 @@
 #define REPLICATION_TYPE_BDR	  2
 
 #define UNKNOWN_SERVER_VERSION_NUM -1
+#define UNKNOWN_BDR_VERSION_NUM -1
+
 #define UNKNOWN_TIMELINE_ID -1
 #define UNKNOWN_SYSTEM_IDENTIFIER 0
 
@@ -58,6 +60,8 @@
 
 #define VOTING_TERM_NOT_SET -1
 
+#define BDR2_REPLICATION_SET_NAME "repmgr"
+
 /*
  * various default values - ensure repmgr.conf.sample is update
  * if any of these are changed
@@ -81,6 +85,7 @@
 #define DEFAULT_PROMOTE_CHECK_TIMEOUT        60  /* seconds */
 #define DEFAULT_PROMOTE_CHECK_INTERVAL       1   /* seconds */
 #define DEFAULT_STANDBY_RECONNECT_TIMEOUT    60  /* seconds */
+#define DEFAULT_NODE_REJOIN_TIMEOUT          60  /* seconds */
 
 #ifndef RECOVERY_COMMAND_FILE
 #define RECOVERY_COMMAND_FILE "recovery.conf"

repmgr_version.h.in

@@ -1,3 +1,2 @@
 #define REPMGR_VERSION_DATE ""
-#define REPMGR_VERSION "4.0.6"
+#define REPMGR_VERSION "4.1.2"
-

repmgrd-bdr.c

@@ -214,7 +214,8 @@ monitor_bdr(void)
 
 								log_warning(_("unable to connect to node %s (ID %i)"),
 											cell->node_info->node_name, cell->node_info->node_id);
-								cell->node_info->conn = try_reconnect(cell->node_info);
+								//cell->node_info->conn = try_reconnect(cell->node_info);
+								try_reconnect(&cell->node_info->conn, cell->node_info);
 
 								/* node has recovered - log and continue */
 								if (cell->node_info->node_status == NODE_STATUS_UP)
@@ -293,7 +294,7 @@ loop:
 			/*
 			 * if we can reload, then could need to change local_conn
 			 */
-			if (reload_config(&config_file_options))
+			if (reload_config(&config_file_options, BDR))
 			{
 				PQfinish(local_conn);
 				local_conn = establish_db_connection(config_file_options.conninfo, true);
@@ -303,11 +304,12 @@ loop:
 			got_SIGHUP = false;
 		}
 
+		/* XXX this looks like it will never be called */
 		if (got_SIGHUP)
 		{
 			log_debug("SIGHUP received");
 
-			if (reload_config(&config_file_options))
+			if (reload_config(&config_file_options, BDR))
 			{
 				PQfinish(local_conn);
 				local_conn = establish_db_connection(config_file_options.conninfo, true);

repmgrd-physical.c

@@ -60,6 +60,8 @@ static int	primary_node_id = UNKNOWN_NODE_ID;
 static t_node_info upstream_node_info = T_NODE_INFO_INITIALIZER;
 static NodeInfoList sibling_nodes = T_NODE_INFO_LIST_INITIALIZER;
 
+static instr_time last_monitoring_update;
+
 
 static ElectionResult do_election(void);
 static const char *_print_election_result(ElectionResult result);
@@ -81,6 +83,8 @@ static bool do_witness_failover(void);
 
 static void update_monitoring_history(void);
 
+static void handle_sighup(PGconn **conn, t_server_type server_type);
+
 static const char * format_failover_state(FailoverState failover_state);
 
 
@@ -162,8 +166,8 @@ do_physical_node_check(void)
 	if (config_file_options.failover == FAILOVER_AUTOMATIC)
 	{
 		/*
-		 * check that promote/follow commands are defined, otherwise repmgrd
+		 * Check that "promote_command" and "follow_command" are defined, otherwise repmgrd
-		 * won't be able to perform any useful action
+		 * won't be able to perform any useful action in a failover situation.
 		 */
 
 		bool		required_param_missing = false;
@@ -175,14 +179,24 @@ do_physical_node_check(void)
 			if (config_file_options.service_promote_command[0] != '\0')
 			{
 				/*
-				 * if repmgrd executes "service_promote_command" directly,
+				 * "service_promote_command" is *not* a substitute for "promote_command";
-				 * repmgr metadata won't get updated
+				 * it is intended for use in those systems (e.g. Debian) where there's a service
+				 * level promote command (e.g. pg_ctlcluster).
+				 *
+				 * "promote_command" should either execute "repmgr standby promote" directly, or
+				 * a script which executes "repmgr standby promote". This is essential, as the
+				 * repmgr metadata is updated by "repmgr standby promote".
+				 *
+				 * "service_promote_command", if set, will be executed by "repmgr standby promote",
+				 * but never by repmgrd.
+				 *
 				 */
 				log_hint(_("\"service_promote_command\" is set, but can only be executed by \"repmgr standby promote\""));
 			}
 
 			required_param_missing = true;
 		}
+
 		if (config_file_options.follow_command[0] == '\0')
 		{
 			log_error(_("\"follow_command\" must be defined in the configuration file"));
@@ -254,7 +268,12 @@ monitor_streaming_primary(void)
 		 * TODO: cache node list here, refresh at `node_list_refresh_interval`
 		 * also return reason for inavailability so we can log it
 		 */
-		if (is_server_available(local_node_info.conninfo) == false)
+
+		(void) connection_ping(local_conn);
+
+		check_connection(&local_node_info, &local_conn);
+
+		if (PQstatus(local_conn) != CONNECTION_OK)
 		{
 
 			/* local node is down, we were expecting it to be up */
@@ -274,8 +293,6 @@ monitor_streaming_primary(void)
 
 				local_node_info.node_status = NODE_STATUS_UNKNOWN;
 
-				close_connection(&local_conn);
-
 				/*
 				 * as we're monitoring the primary, no point in trying to
 				 * write the event to the database
@@ -291,11 +308,12 @@ monitor_streaming_primary(void)
 
 				termPQExpBuffer(&event_details);
 
-				local_conn = try_reconnect(&local_node_info);
+				try_reconnect(&local_conn, &local_node_info);
 
 				if (local_node_info.node_status == NODE_STATUS_UP)
 				{
 					int			local_node_unreachable_elapsed = calculate_elapsed(local_node_unreachable_start);
+					int 		stored_local_node_id = UNKNOWN_NODE_ID;
 
 					initPQExpBuffer(&event_details);
 
@@ -312,6 +330,17 @@ monitor_streaming_primary(void)
 											  event_details.data);
 					termPQExpBuffer(&event_details);
 
+					/*
+					 * If the local node was restarted, we'll need to reinitialise values
+					 * stored in shared memory.
+					 */
+
+					stored_local_node_id = repmgrd_get_local_node_id(local_conn);
+					if (stored_local_node_id == UNKNOWN_NODE_ID)
+					{
+						repmgrd_set_local_node_id(local_conn, config_file_options.node_id);
+					}
+
 					goto loop;
 				}
 
@@ -535,26 +564,7 @@ loop:
 
 		if (got_SIGHUP)
 		{
-			log_debug("SIGHUP received");
+			handle_sighup(&local_conn, PRIMARY);
-
-			if (reload_config(&config_file_options))
-			{
-				close_connection(&local_conn);
-				local_conn = establish_db_connection(config_file_options.conninfo, true);
-
-				if (*config_file_options.log_file)
-				{
-					FILE	   *fd;
-
-					fd = freopen(config_file_options.log_file, "a", stderr);
-					if (fd == NULL)
-					{
-						fprintf(stderr, "error reopening stderr to \"%s\": %s",
-								config_file_options.log_file, strerror(errno));
-					}
-				}
-			}
-			got_SIGHUP = false;
 		}
 
 		log_verbose(LOG_DEBUG, "sleeping %i seconds (parameter \"monitor_interval_secs\")",
@@ -572,9 +582,11 @@ monitor_streaming_standby(void)
 	instr_time	log_status_interval_start;
 	PQExpBufferData event_details;
 
+	log_debug("monitor_streaming_standby()");
+
 	reset_node_voting_status();
 
-	log_debug("monitor_streaming_standby()");
+	INSTR_TIME_SET_ZERO(last_monitoring_update);
 
 	/*
 	 * If no upstream node id is specified in the metadata, we'll try and
@@ -723,10 +735,9 @@ monitor_streaming_standby(void)
 								  _("unable to connect to upstream node \"%s\" (node ID: %i)"),
 								  upstream_node_info.node_name, upstream_node_info.node_id);
 
-				/* */
+				/* XXX possible pre-action event */
 				if (upstream_node_info.type == STANDBY)
 				{
-					/* XXX possible pre-action event */
 					create_event_record(primary_conn,
 										&config_file_options,
 										config_file_options.node_id,
@@ -748,8 +759,6 @@ monitor_streaming_standby(void)
 				log_warning("%s", event_details.data);
 				termPQExpBuffer(&event_details);
 
-				close_connection(&upstream_conn);
-
 				/*
 				 * if local node is unreachable, make a last-minute attempt to reconnect
 				 * before continuing with the failover process
@@ -760,13 +769,18 @@ monitor_streaming_standby(void)
 					check_connection(&local_node_info, &local_conn);
 				}
 
-				upstream_conn = try_reconnect(&upstream_node_info);
+				try_reconnect(&upstream_conn, &upstream_node_info);
 
 				/* Node has recovered - log and continue */
 				if (upstream_node_info.node_status == NODE_STATUS_UP)
 				{
 					int			upstream_node_unreachable_elapsed = calculate_elapsed(upstream_node_unreachable_start);
 
+					if (upstream_node_info.type == PRIMARY)
+					{
+						primary_conn = upstream_conn;
+					}
+
 					initPQExpBuffer(&event_details);
 
 					appendPQExpBuffer(&event_details,
@@ -774,7 +788,7 @@ monitor_streaming_standby(void)
 									  upstream_node_unreachable_elapsed);
 					log_notice("%s", event_details.data);
 
-					create_event_notification(upstream_conn,
+					create_event_notification(primary_conn,
 											  &config_file_options,
 											  config_file_options.node_id,
 											  "repmgrd_upstream_reconnect",
@@ -994,6 +1008,13 @@ monitor_streaming_standby(void)
 								continue;
 							}
 
+							/* skip witness node - we can't possibly "follow" that */
+
+							if (cell->node_info->type == WITNESS)
+							{
+								continue;
+							}
+
 							cell->node_info->conn = establish_db_connection(cell->node_info->conninfo, false);
 
 							if (PQstatus(cell->node_info->conn) != CONNECTION_OK)
@@ -1016,6 +1037,7 @@ monitor_streaming_standby(void)
 							follow_new_primary(follow_node_id);
 						}
 					}
+
 					clear_node_info_list(&sibling_nodes);
 				}
 			}
@@ -1044,8 +1066,7 @@ loop:
 
 				if (config_file_options.failover == FAILOVER_MANUAL)
 				{
-					appendPQExpBuffer(
+					appendPQExpBuffer(&monitoring_summary,
-									  &monitoring_summary,
 									  _(" (automatic failover disabled)"));
 				}
 
@@ -1055,6 +1076,18 @@ loop:
 				{
 					log_detail(_("waiting for upstream or another primary to reappear"));
 				}
+				else if (config_file_options.monitoring_history == true)
+				{
+					if (INSTR_TIME_IS_ZERO(last_monitoring_update))
+					{
+						log_detail(_("no monitoring statistics have been written yet"));
+					}
+					else
+					{
+						log_detail(_("last monitoring statistics update was %i seconds ago"),
+								   calculate_elapsed(last_monitoring_update));
+					}
+				}
 
 				INSTR_TIME_SET_CURRENT(log_status_interval_start);
 			}
@@ -1066,7 +1099,16 @@ loop:
 		}
 		else
 		{
-			connection_ping(local_conn);
+			if (config_file_options.monitoring_history == true)
+			{
+				log_verbose(LOG_WARNING, _("monitoring_history requested but primary connection not available"));
+			}
+
+			/*
+			 * if monitoring not in use, we'll need to ensure the local connection
+			 * handle isn't stale
+			 */
+			(void) connection_ping(local_conn);
 		}
 
 		/*
@@ -1119,8 +1161,11 @@ loop:
 		}
 		else
 		{
+			/* we've reconnected to the local node after an outage */
 			if (local_node_info.active == false)
 			{
+				int stored_local_node_id = UNKNOWN_NODE_ID;
+
 				if (PQstatus(primary_conn) == CONNECTION_OK)
 				{
 					if (update_node_record_set_active(primary_conn, local_node_info.node_id, true) == true)
@@ -1136,45 +1181,36 @@ loop:
 										  local_node_info.node_name,
 										  local_node_info.node_id);
 
-						log_warning("%s", event_details.data)
+						log_notice("%s", event_details.data);
 
-
+						create_event_notification(primary_conn,
-							create_event_notification(primary_conn,
+												  &config_file_options,
-													  &config_file_options,
+												  local_node_info.node_id,
-													  local_node_info.node_id,
+												  "standby_recovery",
-													  "standby_recovery",
+												  true,
-													  true,
+												  event_details.data);
-													  event_details.data);
 
 						termPQExpBuffer(&event_details);
 					}
 				}
+
+				/*
+				 * If the local node was restarted, we'll need to reinitialise values
+				 * stored in shared memory.
+				 */
+
+				stored_local_node_id = repmgrd_get_local_node_id(local_conn);
+				if (stored_local_node_id == UNKNOWN_NODE_ID)
+				{
+					repmgrd_set_local_node_id(local_conn, config_file_options.node_id);
+				}
 			}
 		}
 
 
 		if (got_SIGHUP)
 		{
-			log_debug("SIGHUP received");
+			handle_sighup(&local_conn, STANDBY);
-
-			if (reload_config(&config_file_options))
-			{
-				close_connection(&local_conn);
-				local_conn = establish_db_connection(config_file_options.conninfo, true);
-
-				if (*config_file_options.log_file)
-				{
-					FILE	   *fd;
-
-					fd = freopen(config_file_options.log_file, "a", stderr);
-					if (fd == NULL)
-					{
-						fprintf(stderr, "error reopening stderr to \"%s\": %s",
-								config_file_options.log_file, strerror(errno));
-					}
-				}
-			}
-			got_SIGHUP = false;
 		}
 
 		log_verbose(LOG_DEBUG, "sleeping %i seconds (parameter \"monitor_interval_secs\")",
@@ -1194,36 +1230,18 @@ monitor_streaming_witness(void)
 	PQExpBufferData event_details;
 	RecordStatus record_status;
 
+	int primary_node_id = UNKNOWN_NODE_ID;
+
 	reset_node_voting_status();
 
 	log_debug("monitor_streaming_witness()");
 
-	if (get_primary_node_record(local_conn, &upstream_node_info) == false)
+	/*
-	{
+	 * At this point we can't trust the local copy of "repmgr.nodes", as
-		PQExpBufferData event_details;
+	 * it may not have been updated. We'll scan the cluster for the current
-
+[''	 * primary and refresh the copy from that before proceeding further.
-		initPQExpBuffer(&event_details);
+	 */
-
+	primary_conn = get_primary_connection_quiet(local_conn, &primary_node_id, NULL);
-		appendPQExpBuffer(&event_details,
-						  _("unable to retrieve record for primary node"));
-
-		log_error("%s", event_details.data);
-		log_hint(_("execute \"repmgr witness register --force\" to update the witness node "));
-		close_connection(&local_conn);
-
-		create_event_notification(NULL,
-								  &config_file_options,
-								  config_file_options.node_id,
-								  "repmgrd_shutdown",
-								  false,
-								  event_details.data);
-
-		termPQExpBuffer(&event_details);
-
-		terminate(ERR_BAD_CONFIG);
-	}
-
-	primary_conn = establish_db_connection(upstream_node_info.conninfo, false);
 
 	/*
 	 * Primary node must be running at repmgrd startup.
@@ -1248,7 +1266,7 @@ monitor_streaming_witness(void)
 	 * refresh upstream node record from primary, so it's as up-to-date
 	 * as possible
 	 */
-	record_status = get_node_record(primary_conn, upstream_node_info.node_id, &upstream_node_info);
+	record_status = get_node_record(primary_conn, primary_node_id, &upstream_node_info);
 
 	/*
 	 * This is unlikely to happen; if it does emit a warning for diagnostic
@@ -1320,8 +1338,7 @@ monitor_streaming_witness(void)
 									true,
 									event_details.data);
 
-				close_connection(&primary_conn);
+				try_reconnect(&primary_conn, &upstream_node_info);
-				primary_conn = try_reconnect(&upstream_node_info);
 
 				/* Node has recovered - log and continue */
 				if (upstream_node_info.node_status == NODE_STATUS_UP)
@@ -1335,7 +1352,7 @@ monitor_streaming_witness(void)
 									  upstream_node_unreachable_elapsed);
 					log_notice("%s", event_details.data);
 
-					create_event_notification(upstream_conn,
+					create_event_notification(primary_conn,
 											  &config_file_options,
 											  config_file_options.node_id,
 											  "repmgrd_upstream_reconnect",
@@ -1458,6 +1475,105 @@ monitor_streaming_witness(void)
 		}
 loop:
 
+		/*
+		 * handle local node failure
+		 *
+		 * currently we'll just check the connection, and try to reconnect
+		 *
+		 * TODO: add timeout, after which we run in degraded state
+		 */
+
+		(void) connection_ping(local_conn);
+
+		check_connection(&local_node_info, &local_conn);
+
+		if (PQstatus(local_conn) != CONNECTION_OK)
+		{
+			if (local_node_info.active == true)
+			{
+				bool success = true;
+				PQExpBufferData event_details;
+
+				initPQExpBuffer(&event_details);
+
+				local_node_info.active = false;
+
+				appendPQExpBuffer(&event_details,
+								  _("unable to connect to local node \"%s\" (ID: %i), marking inactive"),
+								  local_node_info.node_name,
+								  local_node_info.node_id);
+				log_notice("%s", event_details.data);
+
+				if (PQstatus(primary_conn) == CONNECTION_OK)
+				{
+					if (update_node_record_set_active(primary_conn, local_node_info.node_id, false) == false)
+					{
+						success = false;
+						log_warning(_("unable to mark node \"%s\" (ID: %i) as inactive"),
+									  local_node_info.node_name,
+									  local_node_info.node_id);
+					}
+				}
+
+				create_event_notification(primary_conn,
+										  &config_file_options,
+										  local_node_info.node_id,
+										  "standby_failure",
+										  success,
+										  event_details.data);
+
+				termPQExpBuffer(&event_details);
+			}
+		}
+		else
+		{
+			/* we've reconnected to the local node after an outage */
+			if (local_node_info.active == false)
+			{
+				int stored_local_node_id = UNKNOWN_NODE_ID;
+
+				if (PQstatus(primary_conn) == CONNECTION_OK)
+				{
+					if (update_node_record_set_active(primary_conn, local_node_info.node_id, true) == true)
+					{
+						PQExpBufferData event_details;
+
+						initPQExpBuffer(&event_details);
+
+						local_node_info.active = true;
+
+						appendPQExpBuffer(&event_details,
+										  _("reconnected to local node \"%s\" (ID: %i), marking active"),
+										  local_node_info.node_name,
+										  local_node_info.node_id);
+
+						log_notice("%s", event_details.data);
+
+						create_event_notification(primary_conn,
+												  &config_file_options,
+												  local_node_info.node_id,
+												  "standby_recovery",
+												  true,
+												  event_details.data);
+
+						termPQExpBuffer(&event_details);
+					}
+				}
+
+				/*
+				 * If the local node was restarted, we'll need to reinitialise values
+				 * stored in shared memory.
+				 */
+
+				stored_local_node_id = repmgrd_get_local_node_id(local_conn);
+				if (stored_local_node_id == UNKNOWN_NODE_ID)
+				{
+					repmgrd_set_local_node_id(local_conn, config_file_options.node_id);
+				}
+			}
+		}
+
+
 		/* refresh repmgr.nodes after "witness_sync_interval" seconds */
 
 		{
@@ -1501,28 +1617,10 @@ loop:
 		}
 
 
+
 		if (got_SIGHUP)
 		{
-			log_debug("SIGHUP received");
+			handle_sighup(&local_conn, WITNESS);
-
-			if (reload_config(&config_file_options))
-			{
-				close_connection(&local_conn);
-				local_conn = establish_db_connection(config_file_options.conninfo, true);
-
-				if (*config_file_options.log_file)
-				{
-					FILE	   *fd;
-
-					fd = freopen(config_file_options.log_file, "a", stderr);
-					if (fd == NULL)
-					{
-						fprintf(stderr, "error reopening stderr to \"%s\": %s",
-								config_file_options.log_file, strerror(errno));
-					}
-				}
-			}
-			got_SIGHUP = false;
 		}
 
 		log_verbose(LOG_DEBUG, "sleeping %i seconds (parameter \"monitor_interval_secs\")",
@@ -1539,8 +1637,15 @@ loop:
 static bool
 do_primary_failover(void)
 {
+	ElectionResult election_result;
+
+	/*
+	 * Double-check status of the local connection
+	 */
+	check_connection(&local_node_info, &local_conn);
+
 	/* attempt to initiate voting process */
-	ElectionResult election_result = do_election();
+	election_result = do_election();
 
 	/* TODO add pre-event notification here */
 	failover_state = FAILOVER_STATE_UNKNOWN;
@@ -1761,12 +1866,21 @@ update_monitoring_history(void)
 	long long unsigned int replication_lag_bytes = 0;
 
 	/* both local and primary connections must be available */
-	if (PQstatus(primary_conn) != CONNECTION_OK || PQstatus(local_conn) != CONNECTION_OK)
+	if (PQstatus(primary_conn) != CONNECTION_OK)
+	{
+		log_warning(_("primary connection is not available, unable to update monitoring history"));
 		return;
+	}
+
+	if (PQstatus(local_conn) != CONNECTION_OK)
+	{
+		log_warning(_("local connection is not available, unable to update monitoring history"));
+		return;
+	}
 
 	if (get_replication_info(local_conn, &replication_info) == false)
 	{
-		log_warning(_("unable to retrieve replication status information"));
+		log_warning(_("unable to retrieve replication status information, unable to update monitoring history"));
 		return;
 	}
 
@@ -1818,8 +1932,7 @@ update_monitoring_history(void)
 		replication_lag_bytes = 0;
 	}
 
-	add_monitoring_record(
+	add_monitoring_record(primary_conn,
-						  primary_conn,
 						  local_conn,
 						  primary_node_id,
 						  local_node_info.node_id,
@@ -1829,6 +1942,8 @@ update_monitoring_history(void)
 						  replication_info.last_xact_replay_timestamp,
 						  replication_lag_bytes,
 						  apply_lag_bytes);
+
+	INSTR_TIME_SET_CURRENT(last_monitoring_update);
 }
 
 
@@ -1853,7 +1968,7 @@ do_upstream_standby_failover(void)
 	t_node_info primary_node_info = T_NODE_INFO_INITIALIZER;
 	RecordStatus record_status = RECORD_NOT_FOUND;
 	RecoveryType primary_type = RECTYPE_UNKNOWN;
-	int			i, r;
+	int			i, standby_follow_result;
 	char		parsed_follow_command[MAXPGPATH] = "";
 
 	close_connection(&upstream_conn);
@@ -1887,9 +2002,18 @@ do_upstream_standby_failover(void)
 
 	if (primary_type != RECTYPE_PRIMARY)
 	{
-		log_error(_("last known primary\"%s\" (ID: %i) is in recovery, not following"),
+		if (primary_type == RECTYPE_STANDBY)
-				  primary_node_info.node_name,
+		{
-				  primary_node_info.node_id);
+			log_error(_("last known primary \"%s\" (ID: %i) is in recovery, not following"),
+					  primary_node_info.node_name,
+					  primary_node_info.node_id);
+		}
+		else
+		{
+			log_error(_("unable to determine status of last known primary \"%s\" (ID: %i), not following"),
+					  primary_node_info.node_name,
+					  primary_node_info.node_id);
+		}
 
 		close_connection(&primary_conn);
 		monitoring_state = MS_DEGRADED;
@@ -1900,8 +2024,6 @@ do_upstream_standby_failover(void)
 	/* Close the connection to this server */
 	close_connection(&local_conn);
 
-	initPQExpBuffer(&event_details);
-
 	log_debug(_("standby follow command is:\n  \"%s\""),
 			  config_file_options.follow_command);
 
@@ -1911,10 +2033,12 @@ do_upstream_standby_failover(void)
 	 */
 	parse_follow_command(parsed_follow_command, config_file_options.follow_command, primary_node_info.node_id);
 
-	r = system(parsed_follow_command);
+	standby_follow_result = system(parsed_follow_command);
 
-	if (r != 0)
+	if (standby_follow_result != 0)
 	{
+		initPQExpBuffer(&event_details);
+
 		appendPQExpBuffer(&event_details,
 						  _("unable to execute follow command:\n %s"),
 						  config_file_options.follow_command);
@@ -1925,8 +2049,7 @@ do_upstream_standby_failover(void)
 		 * It may not possible to write to the event notification table but we
 		 * should be able to generate an external notification if required.
 		 */
-		create_event_notification(
+		create_event_notification(primary_conn,
-								  primary_conn,
 								  &config_file_options,
 								  local_node_info.node_id,
 								  "repmgrd_failover_follow",
@@ -1939,18 +2062,22 @@ do_upstream_standby_failover(void)
 	/*
 	 * It's possible that the standby is still starting up after the "follow_command"
 	 * completes, so poll for a while until we get a connection.
+	 *
+	 * NOTE: we've previously closed the local connection, so even if the follow command
+	 * failed for whatever reason and the local node remained up, we can re-open
+	 * the local connection.
 	 */
 
-	for (i = 0; i < config_file_options.standby_reconnect_timeout; i++)
+	for (i = 0; i < config_file_options.repmgrd_standby_startup_timeout; i++)
 	{
 		local_conn = establish_db_connection(local_node_info.conninfo, false);
 
 		if (PQstatus(local_conn) == CONNECTION_OK)
 			break;
 
-		log_debug("sleeping 1 second; %i of %i attempts to reconnect to local node",
+		log_debug("sleeping 1 second; %i of %i (\"repmgrd_standby_startup_timeout\") attempts to reconnect to local node",
 				  i + 1,
-				  config_file_options.standby_reconnect_timeout);
+				  config_file_options.repmgrd_standby_startup_timeout);
 		sleep(1);
 	}
 
@@ -1964,28 +2091,47 @@ do_upstream_standby_failover(void)
 	/* refresh shared memory settings which will have been zapped by the restart */
 	repmgrd_set_local_node_id(local_conn, config_file_options.node_id);
 
-	if (update_node_record_set_upstream(primary_conn,
+	/*
-										local_node_info.node_id,
+	 *
-										primary_node_info.node_id) == false)
+	 */
+
+	if (standby_follow_result != 0)
 	{
-		appendPQExpBuffer(&event_details,
+		monitoring_state = MS_DEGRADED;
-						  _("unable to set node %i's new upstream ID to %i"),
+		INSTR_TIME_SET_CURRENT(degraded_monitoring_start);
-						  local_node_info.node_id,
-						  primary_node_info.node_id);
 
-		log_error("%s", event_details.data);
+		return FAILOVER_STATE_FOLLOW_FAIL;
+	}
 
-		create_event_notification(
+	/*
-								  NULL,
+	 * update upstream_node_id to primary node (but only if follow command
-								  &config_file_options,
+	 * was successful)
-								  local_node_info.node_id,
+	 */
-								  "repmgrd_failover_follow",
-								  false,
-								  event_details.data);
 
-		termPQExpBuffer(&event_details);
+	{
+		if (update_node_record_set_upstream(primary_conn,
+											local_node_info.node_id,
+											primary_node_info.node_id) == false)
+		{
+			initPQExpBuffer(&event_details);
+			appendPQExpBuffer(&event_details,
+							  _("unable to set node %i's new upstream ID to %i"),
+							  local_node_info.node_id,
+							  primary_node_info.node_id);
 
-		terminate(ERR_BAD_CONFIG);
+			log_error("%s", event_details.data);
+
+			create_event_notification(NULL,
+									  &config_file_options,
+									  local_node_info.node_id,
+									  "repmgrd_failover_follow",
+									  false,
+									  event_details.data);
+
+			termPQExpBuffer(&event_details);
+
+			terminate(ERR_BAD_CONFIG);
+		}
 	}
 
 	/* refresh own internal node record */
@@ -2001,6 +2147,8 @@ do_upstream_standby_failover(void)
 		local_node_info.upstream_node_id = primary_node_info.node_id;
 	}
 
+	initPQExpBuffer(&event_details);
+
 	appendPQExpBuffer(&event_details,
 					  _("node %i is now following primary node %i"),
 					  local_node_info.node_id,
@@ -2008,8 +2156,7 @@ do_upstream_standby_failover(void)
 
 	log_notice("%s", event_details.data);
 
-	create_event_notification(
+	create_event_notification(primary_conn,
-							  primary_conn,
 							  &config_file_options,
 							  local_node_info.node_id,
 							  "repmgrd_failover_follow",
@@ -2056,10 +2203,10 @@ promote_self(void)
 		return FAILOVER_STATE_PROMOTION_FAILED;
 	}
 
-	/* the presence of either of this command has been established already */
+	/* the presence of this command has been established already */
 	promote_command = config_file_options.promote_command;
 
-	log_debug("promote command is:\n  \"%s\"",
+	log_info(_("promote_command is:\n  \"%s\""),
 			  promote_command);
 
 	if (log_type == REPMGR_STDERR && *config_file_options.log_file)
@@ -2247,6 +2394,8 @@ follow_new_primary(int new_primary_id)
 	RecordStatus record_status = RECORD_NOT_FOUND;
 	bool		new_primary_ok = false;
 
+	log_verbose(LOG_DEBUG, "follow_new_primary(): new primary id is %i", new_primary_id);
+
 	record_status = get_node_record(local_conn, new_primary_id, &new_primary);
 
 	if (record_status != RECORD_FOUND)
@@ -2391,7 +2540,7 @@ follow_new_primary(int new_primary_id)
 	 * completes, so poll for a while until we get a connection.
 	 */
 
-	for (i = 0; i < config_file_options.standby_reconnect_timeout; i++)
+	for (i = 0; i < config_file_options.repmgrd_standby_startup_timeout; i++)
 	{
 		local_conn = establish_db_connection(local_node_info.conninfo, false);
 
@@ -2400,7 +2549,7 @@ follow_new_primary(int new_primary_id)
 
 		log_debug("sleeping 1 second; %i of %i attempts to reconnect to local node",
 				  i + 1,
-				  config_file_options.standby_reconnect_timeout);
+				  config_file_options.repmgrd_standby_startup_timeout);
 		sleep(1);
 	}
 
@@ -2466,20 +2615,26 @@ witness_follow_new_primary(int new_primary_id)
 	{
 		RecoveryType primary_recovery_type = get_recovery_type(upstream_conn);
 
-		if (primary_recovery_type == RECTYPE_PRIMARY)
+		switch (primary_recovery_type)
 		{
-			new_primary_ok = true;
+			case RECTYPE_PRIMARY:
-		}
+				new_primary_ok = true;
-		else
+				break;
-		{
+			case RECTYPE_STANDBY:
-			new_primary_ok = false;
+				new_primary_ok = false;
-			log_warning(_("new primary is not in recovery"));
+				log_warning(_("new primary is not in recovery"));
-			close_connection(&upstream_conn);
+				break;
+			case RECTYPE_UNKNOWN:
+				new_primary_ok = false;
+				log_warning(_("unable to determine status of new primary"));
+				break;
 		}
 	}
 
 	if (new_primary_ok == false)
 	{
+		close_connection(&upstream_conn);
+
 		return FAILOVER_STATE_FOLLOW_FAIL;
 	}
 
@@ -2919,9 +3074,18 @@ check_connection(t_node_info *node_info, PGconn **conn)
 		}
 		else
 		{
+			int 		stored_local_node_id = UNKNOWN_NODE_ID;
+
 			log_info(_("reconnected to node \"%s\" (ID: %i)"),
 					 node_info->node_name,
 					 node_info->node_id);
+
+			stored_local_node_id = repmgrd_get_local_node_id(*conn);
+			if (stored_local_node_id == UNKNOWN_NODE_ID)
+			{
+				repmgrd_set_local_node_id(*conn, config_file_options.node_id);
+			}
+
 		}
 	}
 }
@@ -2965,3 +3129,30 @@ format_failover_state(FailoverState failover_state)
 }
 
 
+static void
+handle_sighup(PGconn **conn, t_server_type server_type)
+{
+	log_debug("SIGHUP received");
+
+	if (reload_config(&config_file_options, server_type))
+	{
+		PQfinish(*conn);
+		*conn = establish_db_connection(config_file_options.conninfo, true);
+	}
+
+	if (*config_file_options.log_file)
+	{
+		FILE	   *fd;
+
+		log_debug("reopening %s", config_file_options.log_file);
+
+		fd = freopen(config_file_options.log_file, "a", stderr);
+		if (fd == NULL)
+		{
+			fprintf(stderr, "error reopening stderr to \"%s\": %s",
+					config_file_options.log_file, strerror(errno));
+		}
+	}
+
+	got_SIGHUP = false;
+}

repmgrd.c

@@ -35,8 +35,10 @@
 
 static char *config_file = NULL;
 static bool verbose = false;
-static char *pid_file = NULL;
+static char pid_file[MAXPGPATH];
-static bool daemonize = false;
+static bool daemonize = true;
+static bool show_pid_file = false;
+static bool no_pid_file = false;
 
 t_configuration_options config_file_options = T_CONFIGURATION_OPTIONS_INITIALIZER;
 
@@ -99,8 +101,10 @@ main(int argc, char **argv)
 		{"config-file", required_argument, NULL, 'f'},
 
 /* daemon options */
-		{"daemonize", no_argument, NULL, 'd'},
+		{"daemonize", optional_argument, NULL, 'd'},
 		{"pid-file", required_argument, NULL, 'p'},
+		{"show-pid-file", no_argument, NULL, 's'},
+		{"no-pid-file", no_argument, NULL, OPT_NO_PID_FILE},
 
 /* logging options */
 		{"log-level", required_argument, NULL, 'L'},
@@ -113,8 +117,6 @@ main(int argc, char **argv)
 
 	set_progname(argv[0]);
 
-	srand(time(NULL));
-
 	/* Disallow running as root */
 	if (geteuid() == 0)
 	{
@@ -128,6 +130,10 @@ main(int argc, char **argv)
 		exit(1);
 	}
 
+	srand(time(NULL));
+
+	memset(pid_file, 0, MAXPGPATH);
+
 	while ((c = getopt_long(argc, argv, "?Vf:L:vdp:m", long_options, &optindex)) != -1)
 	{
 		switch (c)
@@ -169,11 +175,22 @@ main(int argc, char **argv)
 				/* daemon options */
 
 			case 'd':
-				daemonize = true;
+				if (optarg != NULL)
+				{
+					daemonize = parse_bool(optarg, "-d/--daemonize", &cli_errors);
+				}
 				break;
 
 			case 'p':
-				pid_file = optarg;
+				strncpy(pid_file, optarg, MAXPGPATH);
+				break;
+
+			case 's':
+				show_pid_file = true;
+				break;
+
+			case OPT_NO_PID_FILE:
+				no_pid_file = true;
 				break;
 
 				/* logging options */
@@ -220,7 +237,7 @@ main(int argc, char **argv)
 	/* Exit here already if errors in command line options found */
 	if (cli_errors.head != NULL)
 	{
-		exit_with_cli_errors(&cli_errors);
+		exit_with_cli_errors(&cli_errors, NULL);
 	}
 
 	startup_event_logged = false;
@@ -239,6 +256,58 @@ main(int argc, char **argv)
 	 */
 	load_config(config_file, verbose, false, &config_file_options, argv[0]);
 
+	/* Determine pid file location, unless --no-pid-file supplied */
+
+	if (no_pid_file == false)
+	{
+		if (config_file_options.repmgrd_pid_file[0] != '\0')
+		{
+			if (pid_file[0] != '\0')
+			{
+				log_warning(_("\"repmgrd_pid_file\" will be overridden by --pid-file"));
+			}
+			else
+			{
+				strncpy(pid_file, config_file_options.repmgrd_pid_file, MAXPGPATH);
+			}
+		}
+
+		/* no pid file provided - determine location */
+		if (pid_file[0] == '\0')
+		{
+			/* packagers: if feasible, patch PID file path into "package_pid_file" */
+			char		package_pid_file[MAXPGPATH] = "";
+
+			if (package_pid_file[0] != '\0')
+			{
+				maxpath_snprintf(pid_file, "%s", package_pid_file);
+			}
+			else
+			{
+				const char *tmpdir = getenv("TMPDIR");
+
+				if (!tmpdir)
+					tmpdir = "/tmp";
+
+				maxpath_snprintf(pid_file, "%s/repmgrd.pid", tmpdir);
+			}
+		}
+	}
+	else
+	{
+		/* --no-pid-file supplied - overwrite any value provided with --pid-file ... */
+		memset(pid_file, 0, MAXPGPATH);
+	}
+
+
+	/* If --show-pid-file supplied, output the location (if set) and exit */
+
+	if (show_pid_file == true)
+	{
+		printf("%s\n", pid_file);
+		exit(SUCCESS);
+	}
+
 
 	/* Some configuration file items can be overriden by command line options */
 
@@ -251,8 +320,6 @@ main(int argc, char **argv)
 		strncpy(config_file_options.log_level, cli_log_level, MAXLEN);
 	}
 
-	log_notice(_("repmgrd (repmgr %s) starting up"), REPMGR_VERSION);
-
 	/*
 	 * -m/--monitoring-history, if provided, will override repmgr.conf's
 	 * monitoring_history; this is for backwards compatibility as it's
@@ -280,6 +347,8 @@ main(int argc, char **argv)
 
 	logger_init(&config_file_options, progname());
 
+	log_notice(_("repmgrd (%s %s) starting up"), progname(), REPMGR_VERSION);
+
 	if (verbose)
 		logger_set_verbose();
 
@@ -414,7 +483,7 @@ main(int argc, char **argv)
 		daemonize_process();
 	}
 
-	if (pid_file != NULL)
+	if (pid_file[0] != '\0')
 	{
 		check_and_create_pid_file(pid_file);
 	}
@@ -669,6 +738,8 @@ show_help(void)
 {
 	printf(_("%s: replication management daemon for PostgreSQL\n"), progname());
 	puts("");
+	printf(_("%s monitors a cluster of servers and optionally performs failover.\n"), progname());
+	puts("");
 
 	printf(_("Usage:\n"));
 	printf(_("  %s [OPTIONS]\n"), progname());
@@ -688,19 +759,21 @@ show_help(void)
 
 	puts("");
 
-	printf(_("General configuration options:\n"));
+	printf(_("Daemon configuration options:\n"));
-	printf(_("  -d, --daemonize           detach process from foreground\n"));
+	printf(_("  -d, --daemonize[=true/false]\n"));
-	printf(_("  -p, --pid-file=PATH       write a PID file\n"));
+	printf(_("                            detach process from foreground (default: true)\n"));
+	printf(_("  -p, --pid-file=PATH       use the specified PID file\n"));
+	printf(_("  -s, --show-pid-file       show PID file which would be used by the current configuration\n"));
+	printf(_("  --no-pid-file             don't write a PID file\n"));
 	puts("");
 
-	printf(_("%s monitors a cluster of servers and optionally performs failover.\n"), progname());
 }
 
 
-PGconn *
+void
-try_reconnect(t_node_info *node_info)
+try_reconnect(PGconn **conn, t_node_info *node_info)
 {
-	PGconn	   *conn;
+	PGconn	   *our_conn;
 	t_conninfo_param_list conninfo_params = T_CONNINFO_PARAM_LIST_INITIALIZER;
 
 	int			i;
@@ -709,7 +782,6 @@ try_reconnect(t_node_info *node_info)
 
 	initialize_conninfo_params(&conninfo_params, false);
 
-
 	/* we assume by now the conninfo string is parseable */
 	(void) parse_conninfo_string(node_info->conninfo, &conninfo_params, NULL, false);
 
@@ -732,18 +804,47 @@ try_reconnect(t_node_info *node_info)
 			 * degraded monitoring? - make that configurable
 			 */
 
-			conn = establish_db_connection_by_params(&conninfo_params, false);
+			our_conn = establish_db_connection_by_params(&conninfo_params, false);
 
-			if (PQstatus(conn) == CONNECTION_OK)
+			if (PQstatus(our_conn) == CONNECTION_OK)
 			{
 				free_conninfo_params(&conninfo_params);
 
+				log_info(_("connection to node %i succeeded"), node_info->node_id);
+
+				if (PQstatus(*conn) == CONNECTION_BAD)
+				{
+					log_verbose(LOG_INFO, "original connection handle returned CONNECTION_BAD, using new connection");
+					close_connection(conn);
+					*conn = our_conn;
+				}
+				else
+				{
+					ExecStatusType ping_result;
+
+					ping_result = connection_ping(*conn);
+
+					if (ping_result != PGRES_TUPLES_OK)
+					{
+						log_info("original connnection no longer available, using new connection");
+						close_connection(conn);
+						*conn = our_conn;
+					}
+					else
+					{
+						log_info(_("original connection is still available"));
+
+						PQfinish(our_conn);
+					}
+				}
+
 				node_info->node_status = NODE_STATUS_UP;
-				return conn;
+
+				return;
 			}
 
-			close_connection(&conn);
+			close_connection(&our_conn);
-			log_notice(_("unable to reconnect to node"));
+			log_notice(_("unable to reconnect to node %i"), node_info->node_id);
 		}
 
 		if (i + 1 < max_attempts)
@@ -762,7 +863,7 @@ try_reconnect(t_node_info *node_info)
 
 	free_conninfo_params(&conninfo_params);
 
-	return NULL;
+	return;
 }
 
 
@@ -802,7 +903,7 @@ terminate(int retval)
 {
 	logger_shutdown();
 
-	if (pid_file)
+	if (pid_file[0] != '\0')
 	{
 		unlink(pid_file);
 	}

repmgrd.h

@@ -10,6 +10,8 @@
 #include <time.h>
 #include "portability/instr_time.h"
 
+#define OPT_NO_PID_FILE                  1000
+
 extern volatile sig_atomic_t got_SIGHUP;
 extern MonitoringState monitoring_state;
 extern instr_time degraded_monitoring_start;
@@ -19,11 +21,13 @@ extern t_node_info local_node_info;
 extern PGconn *local_conn;
 extern bool startup_event_logged;
 
-PGconn	   *try_reconnect(t_node_info *node_info);
+void		try_reconnect(PGconn **conn, t_node_info *node_info);
 
 int			calculate_elapsed(instr_time start_time);
 const char *print_monitoring_state(MonitoringState monitoring_state);
 
 void		update_registration(PGconn *conn);
 void		terminate(int retval);
+
+
 #endif							/* _REPMGRD_H_ */