Bump version in configure.in

4.0.2

CONTRIBUTING.md

@@ -2,7 +2,7 @@ License and Contributions
 =========================
 
 `repmgr` is licensed under the GPL v3.  All of its code and documentation is
-Copyright 2010-2017, 2ndQuadrant Limited.  See the files COPYRIGHT and LICENSE for
+Copyright 2010-2018, 2ndQuadrant Limited.  See the files COPYRIGHT and LICENSE for
 details.
 
 The development of repmgr has primarily been sponsored by 2ndQuadrant customers.

COPYRIGHT

@@ -1,4 +1,4 @@
-Copyright (c) 2010-2017, 2ndQuadrant Limited
+Copyright (c) 2010-2018, 2ndQuadrant Limited
 All rights reserved.
 
 This program is free software: you can redistribute it and/or modify

HISTORY

@@ -1,4 +1,18 @@
-4.0.1   2017-12-04
+4.0.2   2018-01-18
+        repmgr: add missing -W option to getopt_long() invocation; GitHub #350 (Ian)
+        repmgr: automatically create slot name if missing; GitHub #343 (Ian)
+        repmgr: fixes to parsing output of remote repmgr invocations; GitHub #349 (Ian)
+        repmgr: BDR support - create missing connection replication set
+          if required; GitHub #347 (Ian)
+        repmgr: handle missing node record in "repmgr node rejoin"; GitHub #358 (Ian)
+        repmgr: enable documentation to be build as single HTML file; GitHub #353 (fanf2)
+        repmgr: recognize "--terse" option for "repmgr cluster event"; GitHub #360 (Ian)
+        repmgr: add "--wait-start" option for "repmgr standby register"; GitHub #356 (Ian)
+        repmgr: add "%p" event notification parameter for "repmgr standby switchover"
+          containing the node ID of the demoted primary (Ian)
+        docs: various fixes and updates (Ian, Daymel, Martín, ams)
+
+4.0.1   2017-12-13
         repmgr: ensure "repmgr node check --action=" returns appropriate return
           code; GitHub #340 (Ian)
         repmgr: add missing schema qualification in get_all_node_records_with_upstream()
@@ -234,7 +248,7 @@
         Add a ssh_options parameter (Jay Taylor)
 
 2.0beta1 2012-07-27
-        Make CLONE command try to make an exact copy including $PGDATA location (Cedric) 
+        Make CLONE command try to make an exact copy including $PGDATA location (Cedric)
         Add detection of master failure (Jaime)
         Add the notion of a witness server (Jaime)
         Add autofailover capabilities (Jaime)

compat.c

@@ -6,7 +6,7 @@
  *    supported PostgreSQL versions. They're unlikely to change but
  *    it would be worth keeping an eye on them for any fixes/improvements.
  *
- * Copyright (c) 2ndQuadrant, 2010-2017
+ * Copyright (c) 2ndQuadrant, 2010-2018
  *
  * Portions Copyright (c) 1996-2013, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California

compat.h

@@ -1,6 +1,6 @@
 /*
  * compat.h
- * Copyright (c) 2ndQuadrant, 2010-2017
+ * Copyright (c) 2ndQuadrant, 2010-2018
  *
  * Portions Copyright (c) 1996-2013, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California

configfile.c

@@ -1,7 +1,7 @@
 /*
  * config.c - parse repmgr.conf and other configuration-related functionality
  *
- * Copyright (c) 2ndQuadrant, 2010-2017
+ * Copyright (c) 2ndQuadrant, 2010-2018
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by

configfile.h

@@ -1,7 +1,7 @@
 /*
  * configfile.h
  *
- * Copyright (c) 2ndQuadrant, 2010-2017
+ * Copyright (c) 2ndQuadrant, 2010-2018
  *
  *
  * This program is free software: you can redistribute it and/or modify

configure

@@ -1,6 +1,6 @@
 #! /bin/sh
 # Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.69 for repmgr 4.0.1.
+# Generated by GNU Autoconf 2.69 for repmgr 4.0.2.
 #
 # Report bugs to <pgsql-bugs@postgresql.org>.
 #
@@ -11,7 +11,7 @@
 # This configure script is free software; the Free Software Foundation
 # gives unlimited permission to copy, distribute and modify it.
 #
-# Copyright (c) 2010-2017, 2ndQuadrant Ltd.
+# Copyright (c) 2010-2018, 2ndQuadrant Ltd.
 ## -------------------- ##
 ## M4sh Initialization. ##
 ## -------------------- ##
@@ -582,8 +582,8 @@ MAKEFLAGS=
 # Identity of this package.
 PACKAGE_NAME='repmgr'
 PACKAGE_TARNAME='repmgr'
-PACKAGE_VERSION='4.0.1'
+PACKAGE_VERSION='4.0.2'
-PACKAGE_STRING='repmgr 4.0.1'
+PACKAGE_STRING='repmgr 4.0.2'
 PACKAGE_BUGREPORT='pgsql-bugs@postgresql.org'
 PACKAGE_URL='https://2ndquadrant.com/en/resources/repmgr/'
 
@@ -1179,7 +1179,7 @@ if test "$ac_init_help" = "long"; then
   # Omit some internal or obsolete options to make the list less imposing.
   # This message is too long to be a string in the A/UX 3.1 sh.
   cat <<_ACEOF
-\`configure' configures repmgr 4.0.1 to adapt to many kinds of systems.
+\`configure' configures repmgr 4.0.2 to adapt to many kinds of systems.
 
 Usage: $0 [OPTION]... [VAR=VALUE]...
 
@@ -1240,7 +1240,7 @@ fi
 
 if test -n "$ac_init_help"; then
   case $ac_init_help in
-     short | recursive ) echo "Configuration of repmgr 4.0.1:";;
+     short | recursive ) echo "Configuration of repmgr 4.0.2:";;
    esac
   cat <<\_ACEOF
 
@@ -1319,14 +1319,14 @@ fi
 test -n "$ac_init_help" && exit $ac_status
 if $ac_init_version; then
   cat <<\_ACEOF
-repmgr configure 4.0.1
+repmgr configure 4.0.2
 generated by GNU Autoconf 2.69
 
 Copyright (C) 2012 Free Software Foundation, Inc.
 This configure script is free software; the Free Software Foundation
 gives unlimited permission to copy, distribute and modify it.
 
-Copyright (c) 2010-2017, 2ndQuadrant Ltd.
+Copyright (c) 2010-2018, 2ndQuadrant Ltd.
 _ACEOF
   exit
 fi
@@ -1338,7 +1338,7 @@ cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.
 
-It was created by repmgr $as_me 4.0.1, which was
+It was created by repmgr $as_me 4.0.2, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   $ $0 $@
@@ -2379,7 +2379,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 # report actual input values of CONFIG_FILES etc. instead of their
 # values after options handling.
 ac_log="
-This file was extended by repmgr $as_me 4.0.1, which was
+This file was extended by repmgr $as_me 4.0.2, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   CONFIG_FILES    = $CONFIG_FILES
@@ -2442,7 +2442,7 @@ _ACEOF
 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
 ac_cs_version="\\
-repmgr config.status 4.0.1
+repmgr config.status 4.0.2
 configured by $0, generated by GNU Autoconf 2.69,
   with options \\"\$ac_cs_config\\"
 

configure.in

@@ -1,6 +1,6 @@
-AC_INIT([repmgr], [4.0.1], [pgsql-bugs@postgresql.org], [repmgr], [https://2ndquadrant.com/en/resources/repmgr/])
+AC_INIT([repmgr], [4.0.2], [pgsql-bugs@postgresql.org], [repmgr], [https://2ndquadrant.com/en/resources/repmgr/])
 
-AC_COPYRIGHT([Copyright (c) 2010-2017, 2ndQuadrant Ltd.])
+AC_COPYRIGHT([Copyright (c) 2010-2018, 2ndQuadrant Ltd.])
 
 AC_CONFIG_HEADER(config.h)
 

controldata.c

@@ -1,6 +1,6 @@
 /*
  * controldata.c
- * Copyright (c) 2ndQuadrant, 2010-2017
+ * Copyright (c) 2ndQuadrant, 2010-2018
  *
  * Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California

controldata.h

@@ -1,6 +1,6 @@
 /*
  * controldata.h
- * Copyright (c) 2ndQuadrant, 2010-2017
+ * Copyright (c) 2ndQuadrant, 2010-2018
  *
  * Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California

dbutils.c

@@ -1,7 +1,7 @@
 /*
  * dbutils.c - Database connection/management functions
  *
- * Copyright (c) 2ndQuadrant, 2010-2017
+ * Copyright (c) 2ndQuadrant, 2010-2018
  *
  *
  * This program is free software: you can redistribute it and/or modify
@@ -1608,7 +1608,12 @@ repmgrd_get_local_node_id(PGconn *conn)
 
 	res = PQexec(conn, "SELECT repmgr.get_local_node_id()");
 
-	if (!PQgetisnull(res, 0, 0))
+	if (PQresultStatus(res) != PGRES_TUPLES_OK)
+	{
+		log_error(_("unable to execute \"SELECT repmgr.get_local_node_id()\""));
+		log_detail("%s", PQerrorMessage(conn));
+	}
+	else if (!PQgetisnull(res, 0, 0))
 	{
 		local_node_id = atoi(PQgetvalue(res, 0, 0));
 	}
@@ -2592,6 +2597,36 @@ truncate_node_records(PGconn *conn)
 	return true;
 }
 
+
+bool
+update_node_record_slot_name(PGconn *primary_conn, int node_id, char *slot_name)
+{
+	PQExpBufferData query;
+	PGresult   *res = NULL;
+
+	initPQExpBuffer(&query);
+
+	appendPQExpBuffer(&query,
+					  " UPDATE repmgr.nodes "
+					  "    SET slot_name = '%s' "
+					  "  WHERE node_id = %i ",
+					  slot_name,
+					  node_id);
+	res = PQexec(primary_conn, query.data);
+	termPQExpBuffer(&query);
+
+	if (PQresultStatus(res) != PGRES_COMMAND_OK)
+	{
+		log_error(_("unable to set node record slot name:\n  %s"),
+				  PQerrorMessage(primary_conn));
+		PQclear(res);
+		return false;
+	}
+
+	PQclear(res);
+	return true;
+}
+
 void
 get_node_replication_stats(PGconn *conn, int server_version_num, t_node_info *node_info)
 {
@@ -2962,7 +2997,6 @@ create_event_record(PGconn *conn, t_configuration_options *options, int node_id,
 }
 
 
-
 /*
  * create_event_notification()
  *
@@ -3063,7 +3097,7 @@ _create_event(PGconn *conn, t_configuration_options *options, int node_id, char
 
 		if (PQresultStatus(res) != PGRES_TUPLES_OK)
 		{
-			/* we don't treat this as an error */
+			/* we don't treat this as a fatal error */
 			log_warning(_("unable to create event record:\n  %s"),
 						PQerrorMessage(conn));
 
@@ -3216,6 +3250,20 @@ _create_event(PGconn *conn, t_configuration_options *options, int node_id, char
 							dst_ptr += strlen(dst_ptr);
 						}
 						break;
+					case 'p':
+						/* %p: former primary id ("repmgr standby switchover") */
+						src_ptr++;
+						if (event_info->former_primary_id != UNKNOWN_NODE_ID)
+						{
+							PQExpBufferData node_id;
+							initPQExpBuffer(&node_id);
+							appendPQExpBuffer(&node_id,
+											  "%i", event_info->former_primary_id);
+							strlcpy(dst_ptr, node_id.data, end_ptr - dst_ptr);
+							dst_ptr += strlen(dst_ptr);
+							termPQExpBuffer(&node_id);
+						}
+						break;
 					default:
 						/* otherwise treat the % as not special */
 						if (dst_ptr < end_ptr)
@@ -3249,10 +3297,102 @@ _create_event(PGconn *conn, t_configuration_options *options, int node_id, char
 }
 
 
+PGresult *
+get_event_records(PGconn *conn, int node_id, const char *node_name, const char *event, bool all, int limit)
+{
+	PGresult   *res;
+
+	PQExpBufferData query;
+	PQExpBufferData where_clause;
+
+
+	initPQExpBuffer(&query);
+	initPQExpBuffer(&where_clause);
+
+	/* LEFT JOIN used here as a node record may have been removed */
+	appendPQExpBuffer(&query,
+					  "   SELECT e.node_id, n.node_name, e.event, e.successful, "
+					  "          TO_CHAR(e.event_timestamp, 'YYYY-MM-DD HH24:MI:SS') AS timestamp, "
+					  "          e.details "
+					  "     FROM repmgr.events e "
+					  "LEFT JOIN repmgr.nodes n ON e.node_id = n.node_id ");
+
+	if (node_id != UNKNOWN_NODE_ID)
+	{
+		append_where_clause(&where_clause,
+							"n.node_id=%i", node_id);
+	}
+	else if (node_name[0] != '\0')
+	{
+		char	   *escaped = escape_string(conn, node_name);
+
+		if (escaped == NULL)
+		{
+			log_error(_("unable to escape value provided for node name"));
+			log_detail(_("node name is: \"%s\""), node_name);
+		}
+		else
+		{
+			append_where_clause(&where_clause,
+								"n.node_name='%s'",
+								escaped);
+			pfree(escaped);
+		}
+	}
+
+	if (event[0] != '\0')
+	{
+		char	   *escaped = escape_string(conn, event);
+
+		if (escaped == NULL)
+		{
+			log_error(_("unable to escape value provided for event"));
+			log_detail(_("event is: \"%s\""), event);
+		}
+		else
+		{
+			append_where_clause(&where_clause,
+								"e.event='%s'",
+								escaped);
+			pfree(escaped);
+		}
+	}
+
+	appendPQExpBuffer(&query, "\n%s\n",
+					  where_clause.data);
+
+	appendPQExpBuffer(&query,
+					  " ORDER BY e.event_timestamp DESC");
+
+	if (all == false && limit > 0)
+	{
+		appendPQExpBuffer(&query, " LIMIT %i",
+						  limit);
+	}
+
+	log_debug("do_cluster_event():\n%s", query.data);
+	res = PQexec(conn, query.data);
+
+	termPQExpBuffer(&query);
+	termPQExpBuffer(&where_clause);
+
+
+	return res;
+}
+
+
 /* ========================== */
 /* replication slot functions */
 /* ========================== */
 
+
+void
+create_slot_name(char *slot_name, int node_id)
+{
+	maxlen_snprintf(slot_name, "repmgr_slot_%i", node_id);
+}
+
+
 bool
 create_replication_slot(PGconn *conn, char *slot_name, int server_version_num, PQExpBufferData *error_msg)
 {
@@ -4079,20 +4219,23 @@ is_active_bdr_node(PGconn *conn, const char *node_name)
 					  "    SELECT COALESCE(s.active, TRUE) AS active"
 					  "      FROM bdr.bdr_nodes n "
 					  " LEFT JOIN pg_catalog.pg_replication_slots s "
-					  "        ON slot_name=bdr.bdr_format_slot_name(n.node_sysid, n.node_timeline, n.node_dboid, (SELECT oid FROM pg_database WHERE datname = current_database())) "
+					  "        ON s.slot_name=bdr.bdr_format_slot_name(n.node_sysid, n.node_timeline, n.node_dboid, (SELECT oid FROM pg_catalog.pg_database WHERE datname = pg_catalog.current_database())) "
-					  "     WHERE node_name='%s' ",
+					  "     WHERE n.node_name='%s' ",
 					  node_name);
 
+	log_verbose(LOG_DEBUG, "is_active_bdr_node():\n  %s", query.data);
+
 	res = PQexec(conn, query.data);
 	termPQExpBuffer(&query);
 
+	/* we don't care if the query fails */
 	if (PQresultStatus(res) != PGRES_TUPLES_OK || PQntuples(res) == 0)
 	{
 		is_active_bdr_node = false;
 	}
 	else
 	{
-		is_active_bdr_node = atoi(PQgetvalue(res, 0, 0)) == 1 ? true : false;
+		is_active_bdr_node = atobool(PQgetvalue(res, 0, 0));
 	}
 
 	PQclear(res);
@@ -4202,7 +4345,7 @@ add_table_to_bdr_replication_set(PGconn *conn, const char *tablename, const char
 
 
 bool
-bdr_node_exists(PGconn *conn, const char *node_name)
+bdr_node_name_matches(PGconn *conn, const char *node_name, PQExpBufferData *bdr_local_node_name)
 {
 	PQExpBufferData query;
 	PGresult   *res = NULL;
@@ -4211,10 +4354,7 @@ bdr_node_exists(PGconn *conn, const char *node_name)
 	initPQExpBuffer(&query);
 
 	appendPQExpBuffer(&query,
-					  "SELECT COUNT(*)"
+					  "SELECT bdr.bdr_get_local_node_name() AS node_name");
-					  "  FROM bdr.bdr_nodes"
-					  " WHERE node_name = '%s'",
-					  node_name);
 
 	res = PQexec(conn, query.data);
 	termPQExpBuffer(&query);
@@ -4225,7 +4365,9 @@ bdr_node_exists(PGconn *conn, const char *node_name)
 	}
 	else
 	{
-		node_exists = atoi(PQgetvalue(res, 0, 0)) == 1 ? true : false;
+		node_exists = true;
+		appendPQExpBuffer(bdr_local_node_name,
+						  "%s", PQgetvalue(res, 0, 0));
 	}
 
 	PQclear(res);
@@ -4510,3 +4652,115 @@ unset_bdr_failover_handler(PGconn *conn)
 	PQclear(res);
 	return;
 }
+
+
+bool
+bdr_node_has_repmgr_set(PGconn *conn, const char *node_name)
+{
+	PQExpBufferData query;
+	PGresult   *res = NULL;
+	bool		has_repmgr_set = false;
+
+	initPQExpBuffer(&query);
+
+	appendPQExpBuffer(&query,
+					  " SELECT COUNT(*) "
+					  "   FROM UNNEST(bdr.connection_get_replication_sets('%s') AS repset "
+					  "  WHERE repset = 'repmgr'",
+					  node_name);
+
+	res = PQexec(conn, query.data);
+	termPQExpBuffer(&query);
+
+	if (PQresultStatus(res) != PGRES_TUPLES_OK || PQntuples(res) == 0)
+	{
+		has_repmgr_set = false;
+	}
+	else
+	{
+		has_repmgr_set = atoi(PQgetvalue(res, 0, 0)) == 1 ? true : false;
+	}
+
+	PQclear(res);
+
+	return has_repmgr_set;
+}
+
+
+bool
+bdr_node_set_repmgr_set(PGconn *conn, const char *node_name)
+{
+	PQExpBufferData query;
+	PGresult   *res = NULL;
+	bool		success = true;
+
+	initPQExpBuffer(&query);
+
+	appendPQExpBuffer(&query,
+					  " SELECT bdr.connection_set_replication_sets( "
+					  "   ARRAY( "
+					  "     SELECT repset::TEXT "
+					  "       FROM UNNEST(bdr.connection_get_replication_sets('%s')) AS repset "
+					  "         UNION "
+					  "     SELECT 'repmgr'::TEXT "
+					  "   ), "
+					  "   '%s' "
+					  " ) ",
+					  node_name,
+					  node_name);
+
+	res = PQexec(conn, query.data);
+	termPQExpBuffer(&query);
+
+	if (PQresultStatus(res) != PGRES_TUPLES_OK)
+	{
+		success = false;
+	}
+
+	PQclear(res);
+
+	return success;
+}
+
+
+
+/* miscellaneous debugging functions */
+
+const char *
+print_node_status(NodeStatus node_status)
+{
+	switch (node_status)
+	{
+		case NODE_STATUS_UNKNOWN:
+			return "UNKNOWN";
+		case NODE_STATUS_UP:
+			return "UP";
+		case NODE_STATUS_SHUTTING_DOWN:
+			return "SHUTTING_DOWN";
+		case NODE_STATUS_DOWN:
+			return "DOWN";
+		case NODE_STATUS_UNCLEAN_SHUTDOWN:
+			return "UNCLEAN_SHUTDOWN";
+	}
+
+	return "UNIDENTIFIED_STATUS";
+}
+
+
+const char *
+print_pqping_status(PGPing ping_status)
+{
+	switch (ping_status)
+	{
+		case PQPING_OK:
+			return "PQPING_OK";
+		case PQPING_REJECT:
+			return "PQPING_REJECT";
+		case PQPING_NO_RESPONSE:
+			return "PQPING_NO_RESPONSE";
+		case PQPING_NO_ATTEMPT:
+			return "PQPING_NO_ATTEMPT";
+	}
+
+	return "PQPING_UNKNOWN_STATUS";
+}

dbutils.h

@@ -1,7 +1,7 @@
 /*
  * dbutils.h
  *
- * Copyright (c) 2ndQuadrant, 2010-2017
+ * Copyright (c) 2ndQuadrant, 2010-2018
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -74,6 +74,7 @@ typedef enum
 {
 	NODE_STATUS_UNKNOWN = -1,
 	NODE_STATUS_UP,
+	NODE_STATUS_SHUTTING_DOWN,
 	NODE_STATUS_DOWN,
 	NODE_STATUS_UNCLEAN_SHUTDOWN
 } NodeStatus;
@@ -174,11 +175,13 @@ typedef struct s_event_info
 {
 	char	   *node_name;
 	char	   *conninfo_str;
+	int			former_primary_id;
 } t_event_info;
 
 #define T_EVENT_INFO_INITIALIZER { \
 	NULL, \
- 	NULL \
+	NULL, \
+	UNKNOWN_NODE_ID \
 }
 
 
@@ -369,10 +372,8 @@ bool		check_cluster_schema(PGconn *conn);
 /* GUC manipulation functions */
 bool		set_config(PGconn *conn, const char *config_param, const char *config_value);
 bool		set_config_bool(PGconn *conn, const char *config_param, bool state);
-int guc_set(PGconn *conn, const char *parameter, const char *op,
+int		    guc_set(PGconn *conn, const char *parameter, const char *op, const char *value);
-		const char *value);
+int			guc_set_typed(PGconn *conn, const char *parameter, const char *op, const char *value, const char *datatype);
-int guc_set_typed(PGconn *conn, const char *parameter, const char *op,
-			  const char *value, const char *datatype);
 bool		get_pg_setting(PGconn *conn, const char *setting, char *output);
 
 /* server information functions */
@@ -421,10 +422,10 @@ bool		update_node_record_set_primary(PGconn *conn, int this_node_id);
 bool		update_node_record_set_upstream(PGconn *conn, int this_node_id, int new_upstream_node_id);
 bool		update_node_record_status(PGconn *conn, int this_node_id, char *type, int upstream_node_id, bool active);
 bool		update_node_record_conn_priority(PGconn *conn, t_configuration_options *options);
+bool		update_node_record_slot_name(PGconn *primary_conn, int node_id, char *slot_name);
 
 bool		witness_copy_node_records(PGconn *primary_conn, PGconn *witness_conn);
 
-
 void		clear_node_info_list(NodeInfoList *nodes);
 
 /* PostgreSQL configuration file location functions */
@@ -437,8 +438,10 @@ void		config_file_list_add(t_configfile_list *list, const char *file, const char
 bool		create_event_record(PGconn *conn, t_configuration_options *options, int node_id, char *event, bool successful, char *details);
 bool		create_event_notification(PGconn *conn, t_configuration_options *options, int node_id, char *event, bool successful, char *details);
 bool		create_event_notification_extended(PGconn *conn, t_configuration_options *options, int node_id, char *event, bool successful, char *details, t_event_info *event_info);
+PGresult   *get_event_records(PGconn *conn, int node_id, const char *node_name, const char *event, bool all, int limit);
 
 /* replication slot functions */
+void		create_slot_name(char *slot_name, int node_id);
 bool		create_replication_slot(PGconn *conn, char *slot_name, int server_version_num, PQExpBufferData *error_msg);
 bool		drop_replication_slot(PGconn *conn, char *slot_name);
 RecordStatus get_slot_record(PGconn *conn, char *slot_name, t_replication_slot *record);
@@ -499,12 +502,17 @@ bool		is_bdr_repmgr(PGconn *conn);
 bool		is_table_in_bdr_replication_set(PGconn *conn, const char *tablename, const char *set);
 bool		add_table_to_bdr_replication_set(PGconn *conn, const char *tablename, const char *set);
 void		add_extension_tables_to_bdr_replication_set(PGconn *conn);
-
+bool		bdr_node_name_matches(PGconn *conn, const char *node_name, PQExpBufferData *bdr_local_node_name);
-bool		bdr_node_exists(PGconn *conn, const char *node_name);
 ReplSlotStatus get_bdr_node_replication_slot_status(PGconn *conn, const char *node_name);
 void		get_bdr_other_node_name(PGconn *conn, int node_id, char *name_buf);
 
 bool		am_bdr_failover_handler(PGconn *conn, int node_id);
 void		unset_bdr_failover_handler(PGconn *conn);
+bool		bdr_node_has_repmgr_set(PGconn *conn, const char *node_name);
+bool		bdr_node_set_repmgr_set(PGconn *conn, const char *node_name);
+
+/* miscellaneous debugging functions */
+const char *print_node_status(NodeStatus node_status);
+const char *print_pqping_status(PGPing ping_status);
 
 #endif							/* _REPMGR_DBUTILS_H_ */

dirutil.c

@@ -3,7 +3,7 @@
  * dirmod.c
  *	  directory handling functions
  *
- * Copyright (c) 2ndQuadrant, 2010-2017
+ * Copyright (c) 2ndQuadrant, 2010-2018
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by

dirutil.h

@@ -1,6 +1,6 @@
 /*
  * dirutil.h
- * Copyright (c) 2ndQuadrant, 2010-2017
+ * Copyright (c) 2ndQuadrant, 2010-2018
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by

doc/.gitignore

@@ -2,4 +2,6 @@ HTML.index
 bookindex.sgml
 html-stamp
 html/
+nochunks.dsl
+repmgr.html
 version.sgml

doc/Makefile.in

@@ -10,7 +10,7 @@ SGMLINCLUDE = -D . -D ${srcdir}
 
 SPFLAGS += -wall -wno-unused-param -wno-empty -wfully-tagged
 
-JADE.html.call = $(JADE) $(JADEFLAGS) $(SPFLAGS) $(SGMLINCLUDE) $(CATALOG) -d stylesheet.dsl -t sgml -i output-html
+JADE.html.call = $(JADE) $(JADEFLAGS) $(SPFLAGS) $(SGMLINCLUDE) $(CATALOG) -t sgml -i output-html
 
 ALLSGML := $(wildcard $(srcdir)/*.sgml)
 # to build bookindex
@@ -26,10 +26,15 @@ html: html-stamp
 
 html-stamp: repmgr.sgml $(ALLSGML) $(GENERATED_SGML) stylesheet.dsl website-docs.css
 	$(MKDIR_P) html
-	$(JADE.html.call) -i include-index $<
+	$(JADE.html.call) -d stylesheet.dsl -i include-index $<
 	cp $(srcdir)/stylesheet.css $(srcdir)/website-docs.css html/
 	touch $@
 
+repmgr.html: repmgr.sgml $(ALLSGML) $(GENERATED_SGML) stylesheet.dsl website-docs.css
+	sed '/html-index-filename/a\
+(define nochunks  #t)' <stylesheet.dsl >nochunks.dsl
+	$(JADE.html.call) -d nochunks.dsl -i include-index $< >repmgr.html
+
 version.sgml: ${repmgr_top_builddir}/repmgr_version.h
 	{ \
 	  echo "<!ENTITY repmgrversion \"$(REPMGR_VERSION)\">"; \
@@ -37,7 +42,7 @@ version.sgml: ${repmgr_top_builddir}/repmgr_version.h
 
 HTML.index: repmgr.sgml $(ALMOSTALLSGML) stylesheet.dsl
 	@$(MKDIR_P) html
-	$(JADE.html.call) -V html-index $<
+	$(JADE.html.call) -d stylesheet.dsl -V html-index $<
 
 website-docs.css:
 	@$(MKDIR_P) html

doc/appendix-faq.sgml

@@ -1,5 +1,5 @@
 <appendix id="appendix-faq" xreflabel="FAQ">
-<indexterm>
+ <indexterm>
   <primary>FAQ (Frequently Asked Questions)</primary>
  </indexterm>
 

doc/appendix-packages.sgml

@@ -0,0 +1,158 @@
+<appendix id="appendix-packages" xreflabel="Package details">
+ <indexterm>
+  <primary>packages</primary>
+ </indexterm>
+
+ <title>&repmgr; package details</title>
+ <para>
+   This section provides technical details about various &repmgr; binary
+   packages, such as location of the installed binaries and
+   configuration files.
+ </para>
+
+ <sect1 id="packages-centos" xreflabel="CentOS packages">
+  <title>CentOS, RHEL, Scientific Linux etc.</title>
+  <para>
+    Currently packages are provided for versions 6.x and 7.x of CentOS et al.
+  </para>
+
+  <note>
+    <para>
+      For PostgreSQL 9.6 and lower, the CentOS packages use a mixture of <literal>9.6</literal>
+      and <literal>96</literal> in various places to designate the major version;
+      from PostgreSQL 10, the first part of the version number (e.g. <literal>10</literal>) is
+      the major version, so there is more consistency in file/path/package naming.
+    </para>
+  </note>
+
+  <table id="centos-7-packages">
+   <title>CentOS 7 packages</title>
+
+   <tgroup cols="2">
+    <tbody>
+     <row>
+      <entry>Repository URL:</entry>
+      <entry><ulink url="https://yum.postgresql.org/repopackages.php">https://yum.postgresql.org/repopackages.php</ulink></entry>
+     </row>
+
+     <row>
+      <entry>Repository documentation:</entry>
+      <entry><ulink url="https://yum.postgresql.org/">https://yum.postgresql.org/</ulink></entry>
+     </row>
+
+     <row>
+      <entry>Package name example:</entry>
+      <entry><filename>repmgr10-4.0.0-1.rhel7.x86_64</filename></entry>
+     </row>
+
+     <row>
+      <entry>Metapackage:</entry>
+      <entry>(none)</entry>
+     </row>
+
+     <row>
+      <entry>Installation command:</entry>
+      <entry><literal>yum install -y repmgr10</literal></entry>
+     </row>
+
+     <row>
+      <entry>Binary location:</entry>
+      <entry><filename>/usr/pgsql-10/bin</filename></entry>
+     </row>
+
+     <row>
+      <entry>In default path:</entry>
+      <entry>NO</entry>
+     </row>
+
+     <row>
+      <entry>Configuration file location:</entry>
+      <entry><filename>/etc/repmgr/10/repmgr.conf</filename></entry>
+     </row>
+
+     <row>
+      <entry>repmgrd service command:</entry>
+      <entry><literal>service repmgr10</literal></entry>
+     </row>
+
+     <row>
+      <entry>repmgrd service file location:</entry>
+      <entry><filename>/usr/lib/systemd/system/repmgr10.service</filename></entry>
+     </row>
+
+     <row>
+      <entry>repmgrd log file location:</entry>
+      <entry>(not specified)</entry>
+     </row>
+
+    </tbody>
+   </tgroup>
+  </table>
+
+  <table id="centos-6-packages">
+   <title>CentOS 6 packages</title>
+
+   <tgroup cols="2">
+    <tbody>
+     <row>
+      <entry>Repository URL:</entry>
+      <entry><ulink url="https://yum.postgresql.org/repopackages.php">https://yum.postgresql.org/repopackages.php</ulink></entry>
+     </row>
+
+     <row>
+      <entry>Repository documentation:</entry>
+      <entry><ulink url="https://yum.postgresql.org/">https://yum.postgresql.org/</ulink></entry>
+     </row>
+
+     <row>
+      <entry>Package name example:</entry>
+      <entry><filename>repmgr96-4.0.0-1.rhel6.x86_64</filename></entry>
+     </row>
+
+     <row>
+      <entry>Metapackage:</entry>
+      <entry>NO</entry>
+     </row>
+
+     <row>
+      <entry>Installation command:</entry>
+      <entry><literal>yum install -y repmgr96</literal></entry>
+     </row>
+
+     <row>
+      <entry>Binary location:</entry>
+      <entry><filename>/usr/pgsql-9.6/bin</filename></entry>
+     </row>
+
+     <row>
+      <entry>In default path:</entry>
+      <entry>NO</entry>
+     </row>
+
+     <row>
+      <entry>Configuration file location:</entry>
+      <entry><filename>/etc/repmgr/9.6/repmgr.conf</filename></entry>
+     </row>
+
+     <row>
+      <entry>repmgrd service command:</entry>
+      <entry>service repmgr-9.6</entry>
+     </row>
+
+     <row>
+      <entry>repmgrd service file location:</entry>
+      <entry><literal>/etc/init.d/repmgr-9.6</literal></entry>
+     </row>
+
+     <row>
+      <entry>repmgrd log file location:</entry>
+      <entry><filename>/var/log/repmgr/repmgrd-9.6.log</filename></entry>
+     </row>
+
+    </tbody>
+   </tgroup>
+  </table>
+
+
+ </sect1>
+</appendix>

doc/appendix-release-notes.sgml

@@ -1,25 +1,124 @@
 <appendix id="appendix-release-notes">
- <title>Release notes</title>
+  <title>Release notes</title>
- <indexterm>
+  <indexterm>
-  <primary>Release notes</primary>
+    <primary>Release notes</primary>
- </indexterm>
+  </indexterm>
 
- <para>
+  <para>
-  Changes to each &repmgr; release are documented in the release notes.
+    Changes to each &repmgr; release are documented in the release notes.
-  Please read the release notes for all versions between
+    Please read the release notes for all versions between
-  your current version and the version you are plan to upgrade to
+    your current version and the version you are plan to upgrade to
-  before performing an upgrade, as there may be version-specific upgrade steps.
+    before performing an upgrade, as there may be version-specific upgrade steps.
- </para>
+  </para>
 
 
- <para>
+  <para>
-  See also: <xref linkend="upgrading-repmgr">
+    See also: <xref linkend="upgrading-repmgr">
- </para>
+  </para>
+
+  <sect1 id="release-4.0.2">
+    <title>Release 4.0.2</title>
+    <para><emphasis>Thu Jan 18, 2018</emphasis></para>
+
+    <para>
+      repmgr 4.0.2 contains some bug fixes and minor usability enhancements.
+    </para>
+    <sect2>
+      <title>Usability enhancements</title>
+
+      <para>
+        <itemizedlist>
+          <listitem>
+            <para>
+              Recognize the <option>-t</option>/<option>--terse</option> option for
+              <command><link linkend="repmgr-cluster-event">repmgr cluster event</link></command> to hide
+              the <literal>Details</literal> column (GitHub #360)
+            </para>
+          </listitem>
+
+          <listitem>
+            <para>
+              Add "--wait-start" option for
+              <command><link linkend="repmgr-standby-register">repmgr standby register</link></command>
+              (GitHub #356)
+            </para>
+          </listitem>
+
+          <listitem>
+            <para>
+              Add <literal>%p</literal> <link linkend="event-notifications">event notification parameter</link>
+              for <command><link linkend="repmgr-standby-switchover">repmgr standby switchover</link></command>
+            </para>
+          </listitem>
+        </itemizedlist>
+      </para>
+
+    </sect2>
+
+    <sect2>
+      <title>Bug fixes</title>
+
+      <para>
+        <itemizedlist>
+          <listitem>
+            <para>
+              Add missing -W option to <literal>getopt_long()</literal> invocation (GitHub #350)
+            </para>
+          </listitem>
+
+          <listitem>
+            <para>
+              Automatically create slot name if missing (GitHub #343)
+            </para>
+          </listitem>
+
+          <listitem>
+            <para>
+              Fixes to parsing output of remote repmgr invocations (GitHub #349)
+            </para>
+          </listitem>
+
+
+          <listitem>
+            <para>
+              When registering BDR nodes, automatically create missing connection replication set (GitHub #347)
+            </para>
+          </listitem>
+
+
+          <listitem>
+            <para>
+              Handle missing node record in <command><link linkend="repmgr-node-rejoin">repmgr node rejoin</link></command>
+              (GitHub #358)
+            </para>
+          </listitem>
+
+
+        </itemizedlist>
+      </para>
+
+    </sect2>
+
+    <sect2>
+      <title>Documentation</title>
+
+      <para>
+        <itemizedlist>
+          <listitem>
+            <para>
+              The documentation can now be built as a single HTML file (GitHub pull request #353)
+            </para>
+          </listitem>
+        </itemizedlist>
+      </para>
+    </sect2>
+
+  </sect1>
 
  <sect1 id="release-4.0.1">
   <title>Release 4.0.1</title>
 
-  <para><emphasis>Mon Dec 4, 2017</emphasis></para>
+  <para><emphasis>Wed Dec 13, 2017</emphasis></para>
 
   <para>
     repmgr 4.0.1 is a bugfix release.
@@ -416,7 +515,7 @@
   <sect2>
     <title>repmgrd</title>
     <para>
-      The `repmgr` shared library has been renamed from <literal>repmgr_funcs</literal> to
+      The shared library has been renamed from <literal>repmgr_funcs</literal> to
       <literal>repmgr</literal>,  meaning <varname>shared_preload_libraries</varname>
       in <filename>postgresql.conf</filename> needs to be updated to the new name:
       <programlisting>

doc/event-notifications.sgml

@@ -60,10 +60,10 @@
   </varlistentry>
 
   <varlistentry>
-   <term><option>%t</option></term>
+   <term><option>%s</option></term>
    <listitem>
     <para>
-     success (1 or 0)
+     success (1) or failure (0)
     </para>
    </listitem>
   </varlistentry>
@@ -84,6 +84,17 @@
     </para>
    </listitem>
   </varlistentry>
+
+
+  <varlistentry>
+   <term><option>%p</option></term>
+   <listitem>
+    <para>
+     node ID of the demoted standby (<xref linkend="repmgr-standby-switchover"> only)
+    </para>
+   </listitem>
+  </varlistentry>
+
  </variablelist>
  <para>
   The values provided for <literal>%t</literal> and <literal>%d</literal>
@@ -121,6 +132,7 @@
  <para>
   By default, all notification types will be passed to the designated script;
   the notification types can be filtered to explicitly named ones:
+
   <itemizedlist spacing="compact" mark="bullet">
 
    <listitem>
@@ -147,6 +159,12 @@
    <listitem>
     <simpara><literal>standby_disconnect_manual</literal></simpara>
    </listitem>
+   <listitem>
+    <simpara><literal>standby_failure</literal></simpara>
+   </listitem>
+   <listitem>
+    <simpara><literal>standby_recovery</literal></simpara>
+   </listitem>
    <listitem>
     <simpara><literal>witness_register</literal></simpara>
    </listitem>

doc/filelist.sgml

@@ -80,6 +80,7 @@
 <!ENTITY appendix-release-notes  SYSTEM "appendix-release-notes.sgml">
 <!ENTITY appendix-faq      SYSTEM "appendix-faq.sgml">
 <!ENTITY appendix-signatures      SYSTEM "appendix-signatures.sgml">
+<!ENTITY appendix-packages      SYSTEM "appendix-packages.sgml">
 
 <!ENTITY bookindex  SYSTEM "bookindex.sgml">
 

doc/install-packages.sgml

@@ -81,7 +81,7 @@
           <para>
             Install the repmgr version appropriate for your PostgreSQL version (e.g. <literal>repmgr96</literal>), e.g.:
             <programlisting>
-              $ yum install repmg96</programlisting>
+              $ yum install repmgr96</programlisting>
           </para>
         </listitem>
       </itemizedlist>

doc/install-source.sgml

@@ -155,6 +155,20 @@
      The generated HTML files will be placed in the <filename>doc/html</filename>
      subdirectory of your source tree.
    </para>
+
+   <para>
+     To build the documentation as a single HTML file, execute:
+   <programlisting>
+    cd doc/ && make repmgr.html</programlisting>
+   </para>
+
+   <note>
+     <simpara>
+       Due to changes in PostgreSQL's documentation build system from PostgreSQL 10,
+       the documentation can currently only be built agains PostgreSQL 9.6 or earlier.
+       This limitation will be fixed when time and resources permit.
+     </simpara>
+   </note>
  </sect2>
 
 

doc/legal.sgml

@@ -3,7 +3,7 @@
 <date>2017</date>
 
 <copyright>
- <year>2010-2017</year>
+ <year>2010-2018</year>
  <holder>2ndQuadrant, Ltd.</holder>
 </copyright>
 
@@ -11,7 +11,7 @@
  <title>Legal Notice</title>
 
  <para>
-  <productname>repmgr</productname> is Copyright &copy; 2010-2017
+  <productname>repmgr</productname> is Copyright &copy; 2010-2018
   by 2ndQuadrant, Ltd. All rights reserved.
  </para>
 

doc/repmgr-cluster-event.sgml

@@ -40,10 +40,13 @@
           <simpara><literal>--node-name</literal>: restrict entries to node with this name</simpara>
         </listitem>
         <listitem>
-          <simpara><literal>--event</literal>: filter specific event</simpara>
+          <simpara><literal>--event</literal>: filter specific event (see <xref linkend="event-notifications"> for a full list)</simpara>
         </listitem>
       </itemizedlist>
     </para>
+    <para>
+      The "Details" column can be omitted by providing <literal>--terse</literal>.
+    </para>
   </refsect1>
 
   <refsect1>

doc/repmgr-node-rejoin.sgml

@@ -48,7 +48,7 @@
   <refsect1>
     <title>Event notifications</title>
     <para>
-      A <literal>node_rejoin</literal> event notification will be generated.
+      A <literal>node_rejoin</literal> <link linkend="event-notifications">event notification</link> will be generated.
     </para>
   </refsect1>
 

doc/repmgr-primary-register.sgml

@@ -51,7 +51,7 @@
   <refsect1>
     <title>Event notifications</title>
     <para>
-      A <literal>primary_register</literal> event notification will be generated.
+      A <literal>primary_register</literal> <link linkend="event-notifications">event notification</link> will be generated.
     </para>
   </refsect1>
 

doc/repmgr-primary-unregister.sgml

@@ -13,7 +13,7 @@
   <refsect1>
     <title>Description</title>
     <para>
-      <command>repmgr primary register</command> unregisters an inactive primary node
+      <command>repmgr primary unregister</command> unregisters an inactive primary node
       from the &repmgr; metadata. This is typically when the primary has failed and is
       being removed from the cluster after a new primary has been promoted.
     </para>
@@ -28,14 +28,14 @@
 
     <para>
       <command>repmgr master unregister</command> can be used as an alias for
-      <command>repmgr primary unregister</command>/
+      <command>repmgr primary unregister</command>.
     </para>
   </refsect1>
 
   <refsect1>
     <title>Event notifications</title>
     <para>
-      A <literal>primary_unregister</literal> event notification will be generated.
+      A <literal>primary_unregister</literal> <link linkend="event-notifications">event notification</link> will be generated.
     </para>
   </refsect1>
 

doc/repmgr-standby-clone.sgml

@@ -103,7 +103,7 @@
   <refsect1>
     <title>Event notifications</title>
     <para>
-      A <literal>standby_clone</literal> event notification will be generated.
+      A <literal>standby_clone</literal> <link linkend="event-notifications">event notification</link> will be generated.
     </para>
   </refsect1>
 

doc/repmgr-standby-follow.sgml

@@ -51,7 +51,7 @@
   <refsect1>
     <title>Event notifications</title>
     <para>
-      A <literal>standby_follow</literal> event notification will be generated.
+      A <literal>standby_follow</literal> <link linkend="event-notifications">event notification</link> will be generated.
     </para>
   </refsect1>
 

doc/repmgr-standby-promote.sgml

@@ -45,7 +45,7 @@
   <refsect1>
     <title>Event notifications</title>
     <para>
-      A <literal>standby_promote</literal> event notification will be generated.
+      A <literal>standby_promote</literal> <link linkend="event-notifications">event notification</link> will be generated.
     </para>
   </refsect1>
 

doc/repmgr-standby-register.sgml

@@ -37,7 +37,24 @@
     </note>
   </refsect1>
 
-  <refsect1 id="repmgr-standby-register-wait" xreflabel="repmgr standby register --wait">
+  <refsect1 id="repmgr-standby-register-wait-start" xreflabel="repmgr standby register --wait-start">
+   <title>Waiting for the the standby to start</title>
+   <para>
+     By default, &repmgr; will wait 30 seconds for the standby to become available before
+     aborting with a connection error. This is useful when setting up a standby from a script,
+     as the standby may not have fully started up by the time <command>repmgr standby register</command>
+     is executed.
+   </para>
+   <para>
+     To change the timeout, pass the desired value with the <literal>--wait-start</literal> option.
+     A value of <literal>0</literal> will disable the timeout.
+   </para>
+   <para>
+     The timeout will be ignored if <literal>-F/--force</literal> was provided.
+   </para>
+  </refsect1>
+
+  <refsect1 id="repmgr-standby-register-wait-sync" xreflabel="repmgr standby register --wait-sync">
    <title>Waiting for the registration to propagate to the standby</title>
    <para>
      Depending on your environment and workload, it may take some time for
@@ -75,11 +92,11 @@
    </para>
   </refsect1>
 
-
   <refsect1>
     <title>Event notifications</title>
     <para>
-      A <literal>standby_register</literal> event notification will be generated.
+      A <literal>standby_register</literal> <link linkend="event-notifications">event notification</link>
+      will be generated.
     </para>
   </refsect1>
 

doc/repmgr-standby-switchover.sgml

@@ -27,6 +27,71 @@
     </para>
   </refsect1>
 
+  <refsect1>
+    <title>Options</title>
+    <variablelist>
+
+      <varlistentry>
+        <term><option>--always-promote</option></term>
+        <listitem>
+          <para>
+            Promote standby to primary, even if it is behind original primary
+            (original primary will be shut down in any case).
+          </para>
+        </listitem>
+      </varlistentry>
+
+      <varlistentry>
+        <term><option>--dry-run</option></term>
+        <listitem>
+          <para>
+            Check prerequisites but don't actually execute a switchover.
+          </para>
+        </listitem>
+      </varlistentry>
+
+      <varlistentry>
+        <term><option>-F</option></term>
+        <term><option>--force</option></term>
+        <listitem>
+          <para>
+            Ignore warnings and continue anyway.
+          </para>
+        </listitem>
+      </varlistentry>
+
+      <varlistentry>
+        <term><option>--force-rewind</option></term>
+        <listitem>
+          <para>
+            Use <application>pg_rewind</application> to reintegrate the old primary if necessary
+            (PostgreSQL 9.5 and later).
+          </para>
+        </listitem>
+      </varlistentry>
+
+      <varlistentry>
+        <term><option>-R</option></term>
+        <term><option>--remote-user</option></term>
+        <listitem>
+          <para>
+            System username for remote SSH operations (defaults to local system user).
+          </para>
+        </listitem>
+      </varlistentry>
+
+     <varlistentry>
+        <term><option>--siblings-follow</option></term>
+        <listitem>
+          <para>
+            Have standbys attached to the old primary follow the new primary.
+          </para>
+        </listitem>
+      </varlistentry>
+    </variablelist>
+
+  </refsect1>
+
   <refsect1>
     <title>Execution</title>
 
@@ -43,8 +108,14 @@
   <refsect1>
     <title>Event notifications</title>
     <para>
-      A <literal>standby_promote</literal> event notification will be generated on the new primary,
+      <literal>standby_switchover</literal> and <literal>standby_promote</literal>
-      and a <literal>node_rejoin</literal> event notification on the former primary (new standby).
+      <link linkend="event-notifications">event notifications</link> will be generated for the new primary,
+      and a <literal>node_rejoin</literal> event notification for the former primary (new standby).
+    </para>
+    <para>
+      If using an event notification script, <literal>standby_switchover</literal>
+      will populate the placeholder parameter <literal>%p</literal> with the node ID of
+      the former standby.
     </para>
   </refsect1>
 

doc/repmgr-standby-unregister.sgml

@@ -46,7 +46,7 @@
   <refsect1>
     <title>Event notifications</title>
     <para>
-      A <literal>standby_unregister</literal> event notification will be generated.
+      A <literal>standby_unregister</literal> <link linkend="event-notifications">event notification</link> will be generated.
     </para>
   </refsect1>
 

doc/repmgr-witness-register.sgml

@@ -53,7 +53,7 @@
   <refsect1>
     <title>Event notifications</title>
     <para>
-      A <literal>witness_register</literal> event notification will be generated.
+      A <literal>witness_register</literal> <link linkend="event-notifications">event notification</link> will be generated.
     </para>
   </refsect1>
 

doc/repmgr-witness-unregister.sgml

@@ -66,7 +66,7 @@
   <refsect1>
     <title>Event notifications</title>
     <para>
-      A <literal>witness_unregister</literal> event notification will be generated.
+      A <literal>witness_unregister</literal> <link linkend="event-notifications">event notification</link> will be generated.
     </para>
   </refsect1>
 

doc/repmgr.sgml

@@ -117,6 +117,7 @@
  &appendix-release-notes;
  &appendix-signatures;
  &appendix-faq;
+ &appendix-packages;
 
  <![%include-index;[&bookindex;]]>
  <![%include-xslt-index;[<index id="bookindex"></index>]]>

doc/switchover.sgml

@@ -54,45 +54,77 @@
      <secondary>preparation</secondary>
    </indexterm>
    <title>Preparing for switchover</title>
+
    <para>
-    As mentioned above, success of the switchover operation depends on &repmgr;
+    As mentioned in the previous section, success of the switchover operation depends on
-    being able to shut down the current primary server quickly and cleanly.
+    &repmgr;  being able to shut down the current primary server quickly and cleanly.
    </para>
+
    <para>
     Double-check which commands will be used to stop/start/restart the current
     primary; on the primary execute:
     <programlisting>
      repmgr -f /etc/repmgr.conf node service --list --action=stop
      repmgr -f /etc/repmgr.conf node service --list --action=start
-     repmgr -f /etc/repmgr.conf node service --list --action=restart
+     repmgr -f /etc/repmgr.conf node service --list --action=restart</programlisting>
-    </programlisting>
    </para>
+
+   <para>
+     These commands can be defined in <filename>repmgr.conf</filename> with
+     <option>service_start_command</option>, <option>service_stop_command</option>
+     and <option>service_restart_command</option>.
+   </para>
+
+   <important>
+     <para>
+       If &repmgr; is installed from a package. you should set these commands
+       to use the appropriate service commands defined by the package/operating
+       system as these will ensure PostgreSQL is stopped/started properly
+       taking into account configuration and log file locations etc.
+     </para>
+     <para>
+       If the <option>service_*_command</option> options aren't defined, &repmgr; will
+       fall back to using <application>pg_ctl</application> to stop/start/restart
+       PostgreSQL, which may not work properly.
+     </para>
+   </important>
+
    <note>
     <simpara>
      On <literal>systemd</literal> systems we strongly recommend using the appropriate
      <command>systemctl</command> commands (typically run via <command>sudo</command>) to ensure
-     <literal>systemd</literal> informed about the status of the PostgreSQL service.
+     <literal>systemd</literal> is informed about the status of the PostgreSQL service.
+    </simpara>
+    <simpara>
+     If using <command>sudo</command> for the <command>systemctl</command> calls, make sure the
+     <command>sudo</command> specification doesn't require a real tty for the user. If not set
+     this way, <command>repmgr</command> will fail to stop the primary.
     </simpara>
    </note>
+
    <para>
     Check that access from applications is minimalized or preferably blocked
     completely, so applications are not unexpectedly interrupted.
    </para>
+
    <para>
     Check there is no significant replication lag on standbys attached to the
     current primary.
    </para>
+
    <para>
     If WAL file archiving is set up, check that there is no backlog of files waiting
-    to be archived, as PostgreSQL will not finally shut down until all these have been
+    to be archived, as PostgreSQL will not finally shut down until all of these have been
     archived. If there is a backlog exceeding <varname>archive_ready_warning</varname> WAL files,
     &repmgr; will emit a warning before attempting to perform a switchover; you can also check
     manually with <command>repmgr node check --archive-ready</command>.
    </para>
+
    <para>
     Ensure that <application>repmgrd</application> is *not* running anywhere to prevent it unintentionally
     promoting a node.
    </para>
+
    <para>
     Finally, consider executing <command>repmgr standby switchover</command> with the
     <literal>--dry-run</literal> option; this will perform any necessary checks and inform you about
@@ -110,6 +142,48 @@
         "pg_ctl -l /var/log/postgresql/startup.log -D '/var/lib/postgresql/data' -m fast -W stop"
     </programlisting>
    </para>
+
+   <important>
+     <para>
+       Be aware that <option>--dry-run</option> checks the prerequisites
+       for performing the switchover and some basic sanity checks on the
+       state of the database which might effect the switchover operation
+       (e.g. replication lag); it cannot however guarantee the switchover
+       operation will succeed. In particular, if the current primary
+       does not shut down cleanly, &repmgr; will not be able to reliably
+       execute the switchover (as there would be a danger of divergence
+       between the former and new primary nodes).
+     </para>
+   </important>
+
+   <para>
+     Note that following parameters in <filename>repmgr.conf</filename> are relevant to the
+     switchover operation:
+     <itemizedlist spacing="compact" mark="bullet">
+       <listitem>
+         <simpara>
+           <literal>reconnect_attempts</literal>: number of times to check the original primary
+           for a clean shutdown after executing the shutdown command, before aborting
+         </simpara>
+       </listitem>
+       <listitem>
+         <simpara>
+           <literal>reconnect_interval</literal>: interval (in seconds) to check the original
+           primary for a clean shutdown after executing the shutdown command (up to a maximum
+           of <literal>reconnect_attempts</literal> tries)
+         </simpara>
+       </listitem>
+       <listitem>
+         <simpara>
+           <literal>replication_lag_critical</literal>:
+           if replication lag (in seconds) on the standby exceeds this value, the
+           switchover will be aborted (unless the <literal>-F/--force</literal> option
+           is provided)
+         </simpara>
+       </listitem>
+
+     </itemizedlist>
+   </para>
  </sect1>
 
  <sect1 id="switchover-execution" xreflabel="Executing the switchover command">

doc/upgrading-repmgr.sgml

@@ -45,7 +45,7 @@
     </listitem>
     <listitem>
      <simpara>
-       upgrading the repmgr schema
+       upgrading the repmgr schema using <command>CREATE EXTENSION</command>
      </simpara>
     </listitem>
    </orderedlist>
@@ -58,11 +58,19 @@
    a packaged PostgreSQL extension) is normally carried out
    automatically when the &repmgr; extension is created.
   </para>
+  <para>
+   The shared library has been renamed from <literal>repmgr_funcs</literal> to
+   <literal>repmgr</literal> - if it's set in <varname>shared_preload_libraries</varname>
+   in <filename>postgresql.conf</filename> it will need to be updated to the new name:
+   <programlisting>
+    shared_preload_libraries = 'repmgr'</programlisting>
+  </para>
+
   <sect2 id="converting-repmgr-conf">
    <title>Converting repmgr.conf configuration files</title>
    <para>
     With a completely new repmgr version, we've taken the opportunity
-    to rename some configuration items have had their names changed for
+    to rename some configuration items for
     clarity and consistency, both between the configuration file and
     the column names in <structname>repmgr.nodes</structname>
     (e.g. <varname>node</varname> to <varname>node_id</varname>), and

doc/version.sgml

@@ -1 +1 @@
-<!ENTITY repmgrversion "4.0.1">
+<!ENTITY repmgrversion "4.0.2">

errcode.h

@@ -1,6 +1,6 @@
 /*
  * errcode.h
- * Copyright (c) 2ndQuadrant, 2010-2017
+ * Copyright (c) 2ndQuadrant, 2010-2018
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by

log.c

@@ -1,6 +1,6 @@
 /*
  * log.c - Logging methods
- * Copyright (c) 2ndQuadrant, 2010-2017
+ * Copyright (c) 2ndQuadrant, 2010-2018
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by

log.h

@@ -1,6 +1,6 @@
 /*
  * log.h
- * Copyright (c) 2ndQuadrant, 2010-2017
+ * Copyright (c) 2ndQuadrant, 2010-2018
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by

repmgr-action-bdr.c

@@ -1,9 +1,9 @@
 /*
- * repmgr-action-standby.c
+ * repmgr-action-bdr.c
  *
  * Implements BDR-related actions for the repmgr command line utility
  *
- * Copyright (c) 2ndQuadrant, 2010-2017
+ * Copyright (c) 2ndQuadrant, 2010-2018
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -92,7 +92,39 @@ do_bdr_register(void)
 		exit(ERR_BAD_CONFIG);
 	}
 
-	/* check whether repmgr extension exists, and that any other nodes are BDR */
+	/* check for a matching BDR node */
+	{
+		PQExpBufferData bdr_local_node_name;
+		bool		node_match = false;
+
+		initPQExpBuffer(&bdr_local_node_name);
+		node_match = bdr_node_name_matches(conn, config_file_options.node_name, &bdr_local_node_name);
+
+		if (node_match == false)
+		{
+			if (strlen(bdr_local_node_name.data))
+			{
+				log_error(_("local node BDR node name is \"%s\", expected: \"%s\""),
+						  bdr_local_node_name.data,
+						  config_file_options.node_name);
+				log_hint(_("\"node_name\" in repmgr.conf must match \"node_name\" in bdr.bdr_nodes"));
+			}
+			else
+			{
+				log_error(_("local node does not report BDR node name"));
+				log_hint(_("ensure this is an active BDR node"));
+			}
+
+			PQfinish(conn);
+			pfree(dbname);
+			termPQExpBuffer(&bdr_local_node_name);
+			exit(ERR_BAD_CONFIG);
+		}
+
+		termPQExpBuffer(&bdr_local_node_name);
+	}
+
+	/* check whether repmgr extension exists, and there are no non-BDR nodes registered */
 	extension_status = get_repmgr_extension_status(conn);
 
 	if (extension_status == REPMGR_UNKNOWN)
@@ -142,17 +174,9 @@ do_bdr_register(void)
 
 	pfree(dbname);
 
-	/* check for a matching BDR node */
+	if (bdr_node_has_repmgr_set(conn, config_file_options.node_name) == false)
 	{
-		bool		node_exists = bdr_node_exists(conn, config_file_options.node_name);
+		bdr_node_set_repmgr_set(conn, config_file_options.node_name);
-
-		if (node_exists == false)
-		{
-			log_error(_("no BDR node with node_name \"%s\" found"), config_file_options.node_name);
-			log_hint(_("\"node_name\" in repmgr.conf must match \"node_name\" in bdr.bdr_nodes"));
-			PQfinish(conn);
-			exit(ERR_BAD_CONFIG);
-		}
 	}
 
 	/*

repmgr-action-bdr.h

@@ -1,6 +1,6 @@
 /*
  * repmgr-action-bdr.h
- * Copyright (c) 2ndQuadrant, 2010-2017
+ * Copyright (c) 2ndQuadrant, 2010-2018
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by

repmgr-action-cluster.c

@@ -3,7 +3,7 @@
  *
  * Implements cluster information actions for the repmgr command line utility
  *
- * Copyright (c) 2ndQuadrant, 2010-2017
+ * Copyright (c) 2ndQuadrant, 2010-2018
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -436,82 +436,18 @@ void
 do_cluster_event(void)
 {
 	PGconn	   *conn = NULL;
-	PQExpBufferData query;
-	PQExpBufferData where_clause;
 	PGresult   *res;
 	int			i = 0;
+	int			column_count = EVENT_HEADER_COUNT;
 
 	conn = establish_db_connection(config_file_options.conninfo, true);
 
-	initPQExpBuffer(&query);
+	res = get_event_records(conn,
-	initPQExpBuffer(&where_clause);
+							runtime_options.node_id,
-
+							runtime_options.node_name,
-	/* LEFT JOIN used here as a node record may have been removed */
+							runtime_options.event,
-	appendPQExpBuffer(
+							runtime_options.all,
-					  &query,
+							runtime_options.limit);
-					  "   SELECT e.node_id, n.node_name, e.event, e.successful, \n"
-					  "          TO_CHAR(e.event_timestamp, 'YYYY-MM-DD HH24:MI:SS') AS timestamp, \n"
-					  "          e.details \n"
-					  "     FROM repmgr.events e \n"
-					  "LEFT JOIN repmgr.nodes n ON e.node_id = n.node_id ");
-
-	if (runtime_options.node_id != UNKNOWN_NODE_ID)
-	{
-
-		append_where_clause(&where_clause,
-							"n.node_id=%i", runtime_options.node_id);
-	}
-	else if (runtime_options.node_name[0] != '\0')
-	{
-		char	   *escaped = escape_string(conn, runtime_options.node_name);
-
-		if (escaped == NULL)
-		{
-			log_error(_("unable to escape value provided for node name"));
-		}
-		else
-		{
-			append_where_clause(&where_clause,
-								"n.node_name='%s'",
-								escaped);
-			pfree(escaped);
-		}
-	}
-
-	if (runtime_options.event[0] != '\0')
-	{
-		char	   *escaped = escape_string(conn, runtime_options.event);
-
-		if (escaped == NULL)
-		{
-			log_error(_("unable to escape value provided for event"));
-		}
-		else
-		{
-			append_where_clause(&where_clause,
-								"e.event='%s'",
-								escaped);
-			pfree(escaped);
-		}
-	}
-
-	appendPQExpBuffer(&query, "\n%s\n",
-					  where_clause.data);
-
-	appendPQExpBuffer(&query,
-					  " ORDER BY e.event_timestamp DESC");
-
-	if (runtime_options.all == false && runtime_options.limit > 0)
-	{
-		appendPQExpBuffer(&query, " LIMIT %i",
-						  runtime_options.limit);
-	}
-
-	log_debug("do_cluster_event():\n%s", query.data);
-	res = PQexec(conn, query.data);
-
-	termPQExpBuffer(&query);
-	termPQExpBuffer(&where_clause);
 
 	if (PQresultStatus(res) != PGRES_TUPLES_OK)
 	{
@@ -538,7 +474,11 @@ do_cluster_event(void)
 	strncpy(headers_event[EV_TIMESTAMP].title, _("Timestamp"), MAXLEN);
 	strncpy(headers_event[EV_DETAILS].title, _("Details"), MAXLEN);
 
-	for (i = 0; i < EVENT_HEADER_COUNT; i++)
+	/* if --terse provided, simply omit the "Details" column */
+	if (runtime_options.terse == true)
+		column_count --;
+
+	for (i = 0; i < column_count; i++)
 	{
 		headers_event[i].max_length = strlen(headers_event[i].title);
 	}
@@ -547,7 +487,7 @@ do_cluster_event(void)
 	{
 		int			j;
 
-		for (j = 0; j < EVENT_HEADER_COUNT; j++)
+		for (j = 0; j < column_count; j++)
 		{
 			headers_event[j].cur_length = strlen(PQgetvalue(res, i, j));
 			if (headers_event[j].cur_length > headers_event[j].max_length)
@@ -558,7 +498,7 @@ do_cluster_event(void)
 
 	}
 
-	for (i = 0; i < EVENT_HEADER_COUNT; i++)
+	for (i = 0; i < column_count; i++)
 	{
 		if (i == 0)
 			printf(" ");
@@ -571,14 +511,14 @@ do_cluster_event(void)
 	}
 	printf("\n");
 	printf("-");
-	for (i = 0; i < EVENT_HEADER_COUNT; i++)
+	for (i = 0; i < column_count; i++)
 	{
 		int			j;
 
 		for (j = 0; j < headers_event[i].max_length; j++)
 			printf("-");
 
-		if (i < (EVENT_HEADER_COUNT - 1))
+		if (i < (column_count - 1))
 			printf("-+-");
 		else
 			printf("-");
@@ -591,13 +531,13 @@ do_cluster_event(void)
 		int			j;
 
 		printf(" ");
-		for (j = 0; j < EVENT_HEADER_COUNT; j++)
+		for (j = 0; j < column_count; j++)
 		{
 			printf("%-*s",
 				   headers_event[j].max_length,
 				   PQgetvalue(res, i, j));
 
-			if (j < (EVENT_HEADER_COUNT - 1))
+			if (j < (column_count - 1))
 				printf(" | ");
 		}
 

repmgr-action-cluster.h

@@ -1,6 +1,6 @@
 /*
  * repmgr-action-cluster.h
- * Copyright (c) 2ndQuadrant, 2010-2017
+ * Copyright (c) 2ndQuadrant, 2010-2018
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by

repmgr-action-node.c

@@ -3,7 +3,7 @@
  *
  * Implements actions available for any kind of node
  *
- * Copyright (c) 2ndQuadrant, 2010-2017
+ * Copyright (c) 2ndQuadrant, 2010-2018
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -496,16 +496,19 @@ _do_node_status_is_shutdown_cleanly(void)
 
 	db_state = get_db_state(config_file_options.data_directory);
 
+	log_verbose(LOG_DEBUG, "db state now: %s", describe_db_state(db_state));
+
 	if (db_state != DB_SHUTDOWNED && db_state != DB_SHUTDOWNED_IN_RECOVERY)
 	{
-		/*
-		 * node is not running, but pg_controldata says it is - unclean
-		 * shutdown
-		 */
 		if (node_status != NODE_STATUS_UP)
 		{
 			node_status = NODE_STATUS_UNCLEAN_SHUTDOWN;
 		}
+		/* server is still responding but shutting down */
+		else if (db_state == DB_SHUTDOWNING)
+		{
+			node_status = NODE_STATUS_SHUTTING_DOWN;
+		}
 	}
 
 	checkPoint = get_latest_checkpoint_location(config_file_options.data_directory);
@@ -525,19 +528,24 @@ _do_node_status_is_shutdown_cleanly(void)
 		node_status = NODE_STATUS_DOWN;
 	}
 
+	log_verbose(LOG_DEBUG, "node status determined as: %s", print_node_status(node_status));
+
 	switch (node_status)
 	{
 		case NODE_STATUS_UP:
 			appendPQExpBuffer(&output, "RUNNING");
 			break;
-		case NODE_STATUS_UNCLEAN_SHUTDOWN:
+		case NODE_STATUS_SHUTTING_DOWN:
-			appendPQExpBuffer(&output, "UNCLEAN_SHUTDOWN");
+			appendPQExpBuffer(&output, "SHUTTING_DOWN");
 			break;
 		case NODE_STATUS_DOWN:
 			appendPQExpBuffer(&output,
 							  "SHUTDOWN --last-checkpoint-lsn=%X/%X",
 							  format_lsn(checkPoint));
 			break;
+		case NODE_STATUS_UNCLEAN_SHUTDOWN:
+			appendPQExpBuffer(&output, "UNCLEAN_SHUTDOWN");
+			break;
 		case NODE_STATUS_UNKNOWN:
 			appendPQExpBuffer(&output, "UNKNOWN");
 			break;
@@ -728,8 +736,7 @@ do_node_check_role(PGconn *conn, OutputMode mode, t_node_info *node_info, CheckS
 			}
 			else
 			{
-				appendPQExpBuffer(
+				appendPQExpBuffer(&details,
-								  &details,
 								  _("node is primary"));
 			}
 			break;
@@ -754,8 +761,7 @@ do_node_check_role(PGconn *conn, OutputMode mode, t_node_info *node_info, CheckS
 				if (is_bdr_db(conn, &output) == false)
 				{
 					status = CHECK_STATUS_CRITICAL;
-					appendPQExpBuffer(
+					appendPQExpBuffer(&details,
-									  &details,
 									  "%s", output.data);
 				}
 				termPQExpBuffer(&output);
@@ -768,6 +774,11 @@ do_node_check_role(PGconn *conn, OutputMode mode, t_node_info *node_info, CheckS
 						appendPQExpBuffer(&details,
 										  _("node is not an active BDR node"));
 					}
+					else
+					{
+						appendPQExpBuffer(&details,
+										  _("node is an active BDR node"));
+					}
 				}
 			}
 		default:
@@ -1556,13 +1567,11 @@ parse_server_action(const char *action_name)
 
 
 /*
- * Intended mainly for "internal" use by "standby switchover", which
+ * Rejoin a dormant (shut down) node to the replication cluster; this
- * calls this on the target server to excute pg_rewind on a demoted
+ * is typically a former primary which needs to be demoted to a standby.
- * primary with a forked (sic) timeline. This function does not
- * currently check whether this is a useful thing to do (however
- * "standby switchover" will perform a check before calling it).
  *
- * TODO: make this into a more generally useful function.
+ * Note that "repmgr node rejoin" is also executed by
+ * "repmgr standby switchover" after promoting the new primary.
  */
 void
 do_node_rejoin(void)
@@ -1615,21 +1624,28 @@ do_node_rejoin(void)
 	/* check if cleanly shut down */
 	if (db_state != DB_SHUTDOWNED && db_state != DB_SHUTDOWNED_IN_RECOVERY)
 	{
-		log_error(_("database is not shut down cleanly"));
+		if (db_state == DB_SHUTDOWNING)
-
-		if (runtime_options.force_rewind == true)
 		{
-			log_detail(_("pg_rewind will not be able to run"));
+			log_error(_("database is still shutting down"));
+		}
+		else
+		{
+			log_error(_("database is not shut down cleanly"));
+
+			if (runtime_options.force_rewind == true)
+			{
+				log_detail(_("pg_rewind will not be able to run"));
+			}
+			log_hint(_("database should be restarted then shut down cleanly after crash recovery completes"));
+			exit(ERR_BAD_CONFIG);
 		}
-		log_hint(_("database should be restarted and shut down cleanly after crash recovery completes"));
-		exit(ERR_BAD_CONFIG);
 	}
 
 
 	/* check provided upstream connection */
 	upstream_conn = establish_db_connection_by_params(&source_conninfo, true);
 
-	/* sanity-checks for 9.3 */
+	/* sanity checks for 9.3 */
 	server_version_num = get_server_version(upstream_conn, NULL);
 
 	if (server_version_num < 90400)
@@ -1638,7 +1654,9 @@ do_node_rejoin(void)
 	if (get_primary_node_record(upstream_conn, &primary_node_record) == false)
 	{
 		log_error(_("unable to retrieve primary node record"));
+		log_hint(_("check the provided database connection string is for a \"repmgr\" database"));
 		PQfinish(upstream_conn);
+		exit(ERR_BAD_CONFIG);
 	}
 
 	PQfinish(upstream_conn);

repmgr-action-node.h

@@ -1,6 +1,6 @@
 /*
  * repmgr-action-node.h
- * Copyright (c) 2ndQuadrant, 2010-2017
+ * Copyright (c) 2ndQuadrant, 2010-2018
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by

repmgr-action-primary.c

@@ -3,7 +3,7 @@
  *
  * Implements primary actions for the repmgr command line utility
  *
- * Copyright (c) 2ndQuadrant, 2010-2017
+ * Copyright (c) 2ndQuadrant, 2010-2018
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by

repmgr-action-primary.h

@@ -1,6 +1,6 @@
 /*
  * repmgr-action-primary.h
- * Copyright (c) 2ndQuadrant, 2010-2017
+ * Copyright (c) 2ndQuadrant, 2010-2018
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by

repmgr-action-standby.c

@@ -3,7 +3,7 @@
  *
  * Implements standby actions for the repmgr command line utility
  *
- * Copyright (c) 2ndQuadrant, 2010-2017
+ * Copyright (c) 2ndQuadrant, 2010-2018
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -99,7 +99,8 @@ static bool create_recovery_file(t_node_info *node_record, t_conninfo_param_list
 static void write_primary_conninfo(char *line, t_conninfo_param_list *param_list);
 static bool write_recovery_file_line(FILE *recovery_file, char *recovery_file_path, char *line);
 
-
+static int parse_output_to_argv(const char *string, char ***argv_array);
+static void free_parsed_argv(char ***argv_array);
 static NodeStatus parse_node_status_is_shutdown_cleanly(const char *node_status_output, XLogRecPtr *checkPoint);
 static CheckStatus parse_node_check_archiver(const char *node_check_output, int *files, int *threshold);
 
@@ -783,9 +784,52 @@ do_standby_register(void)
 
 	conn = establish_db_connection_quiet(config_file_options.conninfo);
 
+	/*
+	 * if --force provided, don't wait for the node to start, as the
+	 * normal use case will be re-registering an existing node, or
+	 * registering an inactive/not-yet-extant one; we'll do the
+	 * error handling for those cases in the next code block
+	 */
+	if (PQstatus(conn) != CONNECTION_OK && runtime_options.force == false)
+	{
+		bool		conn_ok = false;
+		int			timer = 0;
+
+		for (;;)
+		{
+			if (timer == runtime_options.wait_start)
+				break;
+
+			sleep(1);
+
+			log_verbose(LOG_INFO, _("%i of %i connection attempts"),
+						timer + 1,
+						runtime_options.wait_start);
+
+			conn = establish_db_connection_quiet(config_file_options.conninfo);
+
+			if (PQstatus(conn) == CONNECTION_OK)
+			{
+				conn_ok = true;
+				break;
+			}
+
+			timer++;
+		}
+
+		if (conn_ok == true)
+		{
+			log_info(_("connected to local node \"%s\" (ID: %i) after %i seconds"),
+					 config_file_options.node_name,
+					 config_file_options.node_id,
+					 timer);
+		}
+
+	}
+
 	if (PQstatus(conn) != CONNECTION_OK)
 	{
-		if (!runtime_options.force)
+		if (runtime_options.force == false)
 		{
 			log_error(_("unable to connect to local node \"%s\" (ID: %i)"),
 					  config_file_options.node_name,
@@ -797,7 +841,7 @@ do_standby_register(void)
 			exit(ERR_BAD_CONFIG);
 		}
 
-		if (!runtime_options.connection_param_provided)
+		if (runtime_options.connection_param_provided == false)
 		{
 			log_error(_("unable to connect to local node \"%s\" (ID: %i) and no primary connection parameters provided"),
 					  config_file_options.node_name,
@@ -821,8 +865,8 @@ do_standby_register(void)
 	}
 
 	/*
-	 * User is forcing a registration and must have supplied primary
+	 * otherwise user is forcing a registration of a (potentially) inactive (or
-	 * connection info
+	 * not-yet-extant) node and must have supplied primary connection info
 	 */
 	else
 	{
@@ -1731,6 +1775,26 @@ do_standby_follow_internal(PGconn *primary_conn, t_node_info *primary_node_recor
 	{
 		int			primary_server_version_num = get_server_version(primary_conn, NULL);
 
+		/*
+		 * Here we add a sanity check for the "slot_name" field - it's possible
+		 * the node was initially registered with "use_replication_slots=false"
+		 * but the configuration was subsequently changed, leaving the field NULL.
+		 *
+		 * To avoid annoying failures we can just update the node record and proceed.
+		 */
+
+		if (!strlen(local_node_record.slot_name))
+		{
+			create_slot_name(local_node_record.slot_name, config_file_options.node_id);
+
+			log_notice(_("setting node %i's slot name to \"%s\""),
+					   config_file_options.node_id,
+					   local_node_record.slot_name);
+
+			update_node_record_slot_name(primary_conn, config_file_options.node_id, local_node_record.slot_name);
+		}
+
+
 		if (create_replication_slot(primary_conn,
 									local_node_record.slot_name,
 									primary_server_version_num,
@@ -1823,8 +1887,8 @@ do_standby_follow_internal(PGconn *primary_conn, t_node_info *primary_node_recor
 	/* Set the replication user from the primary node record */
 	param_set(&recovery_conninfo, "user", primary_node_record->repluser);
 
-	log_info(_("setting node %i's primary to node %i"),
+	log_notice(_("setting node %i's primary to node %i"),
-			 config_file_options.node_id, primary_node_record->node_id);
+			   config_file_options.node_id, primary_node_record->node_id);
 
 	if (!create_recovery_file(&local_node_record, &recovery_conninfo, config_file_options.data_directory))
 	{
@@ -1927,14 +1991,11 @@ do_standby_follow_internal(PGconn *primary_conn, t_node_info *primary_node_recor
  *  - promoting this standby node to primary
  *  - forcing previous primary node to follow this node
  *
- * Caveats:
+ * Caveat:
  *  - repmgrd must not be running, otherwise it may
  *    attempt a failover
  *    (TODO: find some way of notifying repmgrd of planned
  *     activity like this)
- *  - currently only set up for two-node operation; any other
- *    standbys will probably become downstream cascaded standbys
- *    of the old primary once it's restarted
  *
  * TODO:
  *  - make connection test timeouts/intervals configurable (see below)
@@ -1975,6 +2036,8 @@ do_standby_switchover(void)
 	NodeInfoList sibling_nodes = T_NODE_INFO_LIST_INITIALIZER;
 	int			unreachable_sibling_node_count = 0;
 
+	t_event_info event_info = T_EVENT_INFO_INITIALIZER;
+
 	/*
 	 * SANITY CHECKS
 	 *
@@ -2073,6 +2136,8 @@ do_standby_switchover(void)
 
 	log_verbose(LOG_DEBUG, "remote node name is \"%s\"", remote_node_record.node_name);
 
+	/* this will fill the %p event notification parameter */
+	event_info.former_primary_id = remote_node_record.node_id;
 
 	/*
 	 * If --force-rewind specified, check pg_rewind can be used, and
@@ -2603,6 +2668,7 @@ do_standby_switchover(void)
 				 i + 1, config_file_options.reconnect_attempts);
 		ping_res = PQping(remote_conninfo);
 
+		log_debug("ping status is: %s", print_pqping_status(ping_res));
 
 		/* database server could not be contacted */
 		if (ping_res == PQPING_NO_RESPONSE || ping_res == PQPING_NO_ATTEMPT)
@@ -2623,8 +2689,7 @@ do_standby_switchover(void)
 
 			initPQExpBuffer(&command_output);
 
-			command_success = remote_command(
+			command_success = remote_command(remote_host,
-											 remote_host,
 											 runtime_options.remote_user,
 											 remote_command_str.data,
 											 &command_output);
@@ -2635,6 +2700,8 @@ do_standby_switchover(void)
 			{
 				NodeStatus	status = parse_node_status_is_shutdown_cleanly(command_output.data, &remote_last_checkpoint_lsn);
 
+				log_verbose(LOG_DEBUG, "remote node status is: %s", print_node_status(status));
+
 				if (status == NODE_STATUS_DOWN && remote_last_checkpoint_lsn != InvalidXLogRecPtr)
 				{
 					shutdown_success = true;
@@ -2644,11 +2711,30 @@ do_standby_switchover(void)
 
 					break;
 				}
+				/* remote node did not shut down cleanly */
+				else if (status == NODE_STATUS_UNCLEAN_SHUTDOWN)
+				{
+					if (!runtime_options.force)
+					{
+						log_error(_("current primary did not shut down cleanly, aborting"));
+						log_hint(_("use -F/--force to promote current standby"));
+						termPQExpBuffer(&command_output);
+						exit(ERR_SWITCHOVER_FAIL);
+					}
+					log_error(_("current primary did not shut down cleanly, continuing anyway"));
+					shutdown_success = true;
+					break;
+				}
+				else if (status == NODE_STATUS_SHUTTING_DOWN)
+				{
+					log_info(_("remote node is still shutting down"));
+				}
 			}
 
 			termPQExpBuffer(&command_output);
 		}
 
+		log_debug("sleeping %i seconds until next check", config_file_options.reconnect_interval);
 		sleep(config_file_options.reconnect_interval);
 	}
 
@@ -2662,7 +2748,7 @@ do_standby_switchover(void)
 	/* this is unlikely to happen, but check and handle gracefully anyway */
 	if (PQstatus(local_conn) != CONNECTION_OK)
 	{
-		log_warning(_("connection to local node lost, reconnecting.."));
+		log_warning(_("connection to local node lost, reconnecting..."));
 		local_conn = establish_db_connection(config_file_options.conninfo, false);
 
 		if (PQstatus(local_conn) != CONNECTION_OK)
@@ -2749,8 +2835,7 @@ do_standby_switchover(void)
 	log_debug("executing:\n  %s", remote_command_str.data);
 	initPQExpBuffer(&command_output);
 
-	command_success = remote_command(
+	command_success = remote_command(remote_host,
-									 remote_host,
 									 runtime_options.remote_user,
 									 remote_command_str.data,
 									 &command_output);
@@ -2766,21 +2851,33 @@ do_standby_switchover(void)
 		if (strlen(command_output.data) > 2)
 			log_detail("%s", command_output.data);
 
-		create_event_record(local_conn,
+		create_event_notification_extended(local_conn,
-							&config_file_options,
+										   &config_file_options,
-							config_file_options.node_id,
+										   config_file_options.node_id,
-							"standby_switchover",
+										   "standby_switchover",
-							false,
+										   false,
-							command_output.data);
+										   command_output.data,
+										   &event_info);
 	}
 	else
 	{
-		create_event_record(local_conn,
+		PQExpBufferData event_details;
-							&config_file_options,
+
-							config_file_options.node_id,
+		initPQExpBuffer(&event_details);
-							"standby_switchover",
+
-							true,
+		appendPQExpBuffer(&event_details,
-							NULL);
+						  "node %i promoted to primary, node %i demoted to standby",
+						  config_file_options.node_id,
+						  remote_node_record.node_id);
+
+		create_event_notification_extended(local_conn,
+										   &config_file_options,
+										   config_file_options.node_id,
+										   "standby_switchover",
+										   true,
+										   event_details.data,
+										   &event_info);
+		termPQExpBuffer(&event_details);
 	}
 
 	termPQExpBuffer(&command_output);
@@ -2850,8 +2947,7 @@ do_standby_switchover(void)
 
 			initPQExpBuffer(&command_output);
 
-			success = remote_command(
+			success = remote_command(host,
-									 host,
 									 runtime_options.remote_user,
 									 remote_command_str.data,
 									 &command_output);
@@ -3680,8 +3776,7 @@ initialise_direct_clone(t_node_info *node_record)
 		{
 			log_error("%s", event_details.data);
 
-			create_event_notification(
+			create_event_notification(primary_conn,
-									  primary_conn,
 									  &config_file_options,
 									  config_file_options.node_id,
 									  "standby_clone",
@@ -4989,32 +5084,18 @@ write_primary_conninfo(char *line, t_conninfo_param_list *param_list)
 }
 
 
-/* TODO: consolidate code in below functions */
 static NodeStatus
 parse_node_status_is_shutdown_cleanly(const char *node_status_output, XLogRecPtr *checkPoint)
 {
-	int			options_len = 0;
-	char	   *options_string = NULL;
-	char	   *options_string_ptr = NULL;
 	NodeStatus	node_status = NODE_STATUS_UNKNOWN;
 
-	/*
+	int			c = 0,
-	 * Add parsed options to this list, then copy to an array to pass to
+				argc_item = 0;
-	 * getopt
+	char	  **argv_array = NULL;
-	 */
-	static ItemList option_argv = {NULL, NULL};
-
-	char	   *argv_item;
-	int			c,
-				argc_item = 1;
-
-	char	  **argv_array;
-	ItemListCell *cell;
-
 	int			optindex = 0;
 
 	/* We're only interested in these options */
-	static struct option long_options[] =
+	struct option node_status_options[] =
 	{
 		{"last-checkpoint-lsn", required_argument, NULL, 'L'},
 		{"state", required_argument, NULL, 'S'},
@@ -5028,51 +5109,7 @@ parse_node_status_is_shutdown_cleanly(const char *node_status_output, XLogRecPtr
 		return node_status;
 	}
 
-	options_len = strlen(node_status_output) + 1;
+	argc_item = parse_output_to_argv(node_status_output, &argv_array);
-	options_string = pg_malloc(options_len);
-	options_string_ptr = options_string;
-
-	/* Copy the string before operating on it with strtok() */
-	strncpy(options_string, node_status_output, options_len);
-
-	/* Extract arguments into a list and keep a count of the total */
-	while ((argv_item = strtok(options_string_ptr, " ")) != NULL)
-	{
-		item_list_append(&option_argv, argv_item);
-
-		argc_item++;
-
-		if (options_string_ptr != NULL)
-			options_string_ptr = NULL;
-	}
-
-	/*
-	 * Array of argument values to pass to getopt_long - this will need to
-	 * include an empty string as the first value (normally this would be the
-	 * program name)
-	 */
-	argv_array = pg_malloc0(sizeof(char *) * (argc_item + 2));
-
-	/* Insert a blank dummy program name at the start of the array */
-	argv_array[0] = pg_malloc0(1);
-
-	c = 1;
-
-	/*
-	 * Copy the previously extracted arguments from our list to the array
-	 */
-	for (cell = option_argv.head; cell; cell = cell->next)
-	{
-		int			argv_len = strlen(cell->string) + 1;
-
-		argv_array[c] = pg_malloc0(argv_len);
-
-		strncpy(argv_array[c], cell->string, argv_len);
-
-		c++;
-	}
-
-	argv_array[c] = NULL;
 
 	/* Reset getopt's optind variable */
 	optind = 0;
@@ -5080,7 +5117,7 @@ parse_node_status_is_shutdown_cleanly(const char *node_status_output, XLogRecPtr
 	/* Prevent getopt from emitting errors */
 	opterr = 0;
 
-	while ((c = getopt_long(argc_item, argv_array, "L:S:", long_options,
+	while ((c = getopt_long(argc_item, argv_array, "L:S:", node_status_options,
 							&optindex)) != -1)
 	{
 		switch (c)
@@ -5113,16 +5150,7 @@ parse_node_status_is_shutdown_cleanly(const char *node_status_output, XLogRecPtr
 		}
 	}
 
-	pfree(options_string);
+	free_parsed_argv(&argv_array);
-
-	{
-		int			i;
-
-		for (i = 0; i < argc_item + 2; i++)
-			pfree(argv_array[i]);
-	}
-	pfree(argv_array);
-
 
 	return node_status;
 }
@@ -5131,30 +5159,15 @@ parse_node_status_is_shutdown_cleanly(const char *node_status_output, XLogRecPtr
 static CheckStatus
 parse_node_check_archiver(const char *node_check_output, int *files, int *threshold)
 {
-	int			options_len = 0;
-	char	   *options_string = NULL;
-	char	   *options_string_ptr = NULL;
-
 	CheckStatus status = CHECK_STATUS_UNKNOWN;
 
-
+	int			c = 0,
-	/*
+				argc_item = 0;
-	 * Add parsed options to this list, then copy to an array to pass to
+	char	  **argv_array = NULL;
-	 * getopt
-	 */
-	static ItemList option_argv = {NULL, NULL};
-
-	char	   *argv_item;
-	int			c,
-				argc_item = 1;
-
-	char	  **argv_array;
-	ItemListCell *cell;
-
 	int			optindex = 0;
 
 	/* We're only interested in these options */
-	static struct option long_options[] =
+	struct option node_check_options[] =
 	{
 		{"status", required_argument, NULL, 'S'},
 		{"files", required_argument, NULL, 'f'},
@@ -5171,51 +5184,8 @@ parse_node_check_archiver(const char *node_check_output, int *files, int *thresh
 		return status;
 	}
 
-	options_len = strlen(node_check_output) + 1;
+	argc_item = parse_output_to_argv(node_check_output, &argv_array);
-	options_string = pg_malloc(options_len);
-	options_string_ptr = options_string;
 
-	/* Copy the string before operating on it with strtok() */
-	strncpy(options_string, node_check_output, options_len);
-
-	/* Extract arguments into a list and keep a count of the total */
-	while ((argv_item = strtok(options_string_ptr, " ")) != NULL)
-	{
-		item_list_append(&option_argv, argv_item);
-
-		argc_item++;
-
-		if (options_string_ptr != NULL)
-			options_string_ptr = NULL;
-	}
-
-	/*
-	 * Array of argument values to pass to getopt_long - this will need to
-	 * include an empty string as the first value (normally this would be the
-	 * program name)
-	 */
-	argv_array = pg_malloc0(sizeof(char *) * (argc_item + 2));
-
-	/* Insert a blank dummy program name at the start of the array */
-	argv_array[0] = pg_malloc0(1);
-
-	c = 1;
-
-	/*
-	 * Copy the previously extracted arguments from our list to the array
-	 */
-	for (cell = option_argv.head; cell; cell = cell->next)
-	{
-		int			argv_len = strlen(cell->string) + 1;
-
-		argv_array[c] = pg_malloc0(argv_len);
-
-		strncpy(argv_array[c], cell->string, argv_len);
-
-		c++;
-	}
-
-	argv_array[c] = NULL;
 
 	/* Reset getopt's optind variable */
 	optind = 0;
@@ -5223,7 +5193,7 @@ parse_node_check_archiver(const char *node_check_output, int *files, int *thresh
 	/* Prevent getopt from emitting errors */
 	opterr = 0;
 
-	while ((c = getopt_long(argc_item, argv_array, "f:S:t:", long_options,
+	while ((c = getopt_long(argc_item, argv_array, "f:S:t:", node_check_options,
 							&optindex)) != -1)
 	{
 		switch (c)
@@ -5265,11 +5235,99 @@ parse_node_check_archiver(const char *node_check_output, int *files, int *thresh
 		}
 	}
 
+	free_parsed_argv(&argv_array);
+
 	return status;
 }
 
 
+static int
+parse_output_to_argv(const char *string, char ***argv_array)
+{
+	int			options_len = 0;
+	char	   *options_string = NULL;
+	char	   *options_string_ptr = NULL;
+	int			c = 1,
+	   			argc_item = 1;
+	char	   *argv_item = NULL;
+	char **local_argv_array = NULL;
+	ItemListCell *cell;
 
+	/*
+	 * Add parsed options to this list, then copy to an array to pass to
+	 * getopt
+	 */
+	ItemList option_argv = {NULL, NULL};
+
+	options_len = strlen(string) + 1;
+	options_string = pg_malloc(options_len);
+	options_string_ptr = options_string;
+
+	/* Copy the string before operating on it with strtok() */
+	strncpy(options_string, string, options_len);
+
+	/* Extract arguments into a list and keep a count of the total */
+	while ((argv_item = strtok(options_string_ptr, " ")) != NULL)
+	{
+		item_list_append(&option_argv, trim(argv_item));
+
+		argc_item++;
+
+		if (options_string_ptr != NULL)
+			options_string_ptr = NULL;
+	}
+
+	pfree(options_string);
+
+	/*
+	 * Array of argument values to pass to getopt_long - this will need to
+	 * include an empty string as the first value (normally this would be the
+	 * program name)
+	 */
+	local_argv_array = pg_malloc0(sizeof(char *) * (argc_item + 2));
+
+	/* Insert a blank dummy program name at the start of the array */
+	local_argv_array[0] = pg_malloc0(1);
+
+	/*
+	 * Copy the previously extracted arguments from our list to the array
+	 */
+	for (cell = option_argv.head; cell; cell = cell->next)
+	{
+		int			argv_len = strlen(cell->string) + 1;
+
+		local_argv_array[c] = (char *)pg_malloc0(argv_len);
+
+		strncpy(local_argv_array[c], cell->string, argv_len);
+
+		c++;
+	}
+
+	local_argv_array[c] = NULL;
+
+	item_list_free(&option_argv);
+
+	*argv_array = local_argv_array;
+
+	return argc_item;
+}
+
+
+static void
+free_parsed_argv(char ***argv_array)
+{
+	char	  **local_argv_array = *argv_array;
+	int			i = 0;
+
+	while (local_argv_array[i] != NULL)
+	{
+		pfree((char *)local_argv_array[i]);
+		i++;
+	}
+
+	pfree((char **)local_argv_array);
+	*argv_array = NULL;
+}
 
 
 void
@@ -5313,6 +5371,8 @@ do_standby_help(void)
 	printf(_("  -F, --force                         overwrite an existing node record, or if primary connection\n" \
 			 "                                        parameters supplied, create record even if standby offline\n"));
 	printf(_("  --upstream-node-id                  ID of the upstream node to replicate from (optional)\n"));
+	printf(_("  --wait-start=VALUE                  wait for the standby to start (timeout in seconds, default %i)\n"), DEFAULT_WAIT_START);
+
 	printf(_("  --wait-sync[=VALUE]                 wait for the node record to synchronise to the standby\n" \
 			 "                                        (optional timeout in seconds)\n"));
 

repmgr-action-standby.h

@@ -1,6 +1,6 @@
 /*
  * repmgr-action-standby.h
- * Copyright (c) 2ndQuadrant, 2010-2017
+ * Copyright (c) 2ndQuadrant, 2010-2018
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by

repmgr-action-witness.c

@@ -3,7 +3,7 @@
  *
  * Implements witness actions for the repmgr command line utility
  *
- * Copyright (c) 2ndQuadrant, 2010-2017
+ * Copyright (c) 2ndQuadrant, 2010-2018
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by

repmgr-action-witness.h

@@ -1,6 +1,6 @@
 /*
  * repmgr-action-witness.h
- * Copyright (c) 2ndQuadrant, 2010-2017
+ * Copyright (c) 2ndQuadrant, 2010-2018
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by

repmgr-client-global.h

@@ -1,6 +1,6 @@
 /*
  * repmgr-client-global.h
- * Copyright (c) 2ndQuadrant, 2010-2017
+ * Copyright (c) 2ndQuadrant, 2010-2018
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -86,6 +86,7 @@ typedef struct
 	/* "standby register" options */
 	bool		wait_register_sync;
 	int			wait_register_sync_seconds;
+	int			wait_start;
 
 	/* "standby switchover" options */
 	bool		always_promote;
@@ -146,7 +147,7 @@ typedef struct
 		/* "standby clone"/"standby follow" options */ \
 		NO_UPSTREAM_NODE, \
 		/* "standby register" options */ \
-		false, 0, \
+		false, 0, DEFAULT_WAIT_START,   \
 		/* "standby switchover" options */ \
 		false, false, false, \
 		/* "node status" options */ \

repmgr-client.c

@@ -1,7 +1,7 @@
 /*
  * repmgr-client.c - Command interpreter for the repmgr package
  *
- * Copyright (c) 2ndQuadrant, 2010-2017
+ * Copyright (c) 2ndQuadrant, 2010-2018
  *
  * This module is a command-line utility to easily setup a cluster of
  * hot standby servers for an HA environment
@@ -177,7 +177,7 @@ main(int argc, char **argv)
 		strncpy(runtime_options.username, pw->pw_name, MAXLEN);
 	}
 
-	while ((c = getopt_long(argc, argv, "?Vb:f:Fd:h:p:U:R:S:L:vtD:crC:", long_options,
+	while ((c = getopt_long(argc, argv, "?Vb:f:FWd:h:p:U:R:S:D:ckL:tvC:", long_options,
 							&optindex)) != -1)
 	{
 		/*
@@ -389,7 +389,11 @@ main(int argc, char **argv)
 				 *---------------------------
 				 */
 
-			case OPT_REGISTER_WAIT:
+			case OPT_WAIT_START:
+				runtime_options.wait_start = repmgr_atoi(optarg, "--wait-start", &cli_errors, false);
+				break;
+
+			case OPT_WAIT_SYNC:
 				runtime_options.wait_register_sync = true;
 				if (optarg != NULL)
 				{
@@ -2725,6 +2729,6 @@ init_node_record(t_node_info *node_record)
 
 	if (config_file_options.use_replication_slots == true)
 	{
-		maxlen_snprintf(node_record->slot_name, "repmgr_slot_%i", config_file_options.node_id);
+		create_slot_name(node_record->slot_name, config_file_options.node_id);
 	}
 }

repmgr-client.h

@@ -1,6 +1,6 @@
 /*
  * repmgr-client.h
- * Copyright (c) 2ndQuadrant, 2010-2017
+ * Copyright (c) 2ndQuadrant, 2010-2018
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -56,7 +56,7 @@
 #define OPT_NODE_NAME					   1007
 #define OPT_WITHOUT_BARMAN				   1008
 #define OPT_NO_UPSTREAM_CONNECTION		   1009
-#define OPT_REGISTER_WAIT				   1010
+#define OPT_WAIT_SYNC				       1010
 #define OPT_LOG_TO_FILE					   1011
 #define OPT_UPSTREAM_CONNINFO			   1012
 #define OPT_REPLICATION_USER			   1013
@@ -82,6 +82,8 @@
 #define OPT_SLOTS						   1033
 #define OPT_CONFIG_ARCHIVE_DIR			   1034
 #define OPT_HAS_PASSFILE                   1035
+#define OPT_WAIT_START                     1036
+
 /* deprecated since 3.3 */
 #define OPT_DATA_DIR						999
 #define OPT_NO_CONNINFO_PASSWORD			998
@@ -136,7 +138,8 @@ static struct option long_options[] =
 	{"without-barman", no_argument, NULL, OPT_WITHOUT_BARMAN},
 
 /* "standby register" options */
-	{"wait-sync", optional_argument, NULL, OPT_REGISTER_WAIT},
+	{"wait-start", required_argument, NULL, OPT_WAIT_START},
+	{"wait-sync", optional_argument, NULL, OPT_WAIT_SYNC},
 
 /* "standby switchover" options
  *

repmgr.c

@@ -1,7 +1,7 @@
 /*
  * repmgr.c - repmgr extension
  *
- * Copyright (c) 2ndQuadrant, 2010-2017
+ * Copyright (c) 2ndQuadrant, 2010-2018
  *
  * This is the actual extension code; see repmgr-client.c for the code which
  * generates the repmgr binary

repmgr.conf.sample

@@ -13,35 +13,35 @@
 # repmgr and repmgrd require the following items to be explicitly configured.
 
 
-#node_id=                               # A unique integer greater than zero
+#node_id=    	     	     	 # A unique integer greater than zero
-#node_name=''                           # An arbitrary (but unique) string; we recommend
+#node_name=''			 # An arbitrary (but unique) string; we recommend
-                                        # using the server's hostname or another identifier
+				 # using the server's hostname or another identifier
-                                        # unambiguously associated with the server to avoid
+				 # unambiguously associated with the server to avoid
-                                        # confusion. Avoid choosing names which reflect the
+				 # confusion. Avoid choosing names which reflect the
-                                        # node's current role, e.g. "primary" or "standby1",
+				 # node's current role, e.g. "primary" or "standby1",
-                                        # as roles can change and it will be confusing if
+				 # as roles can change and it will be confusing if
-                                        # the current primary is called "standby1".
+				 # the current primary is called "standby1".
 
-#conninfo=''                            # Database connection information as a conninfo string.
+#conninfo=''			 # Database connection information as a conninfo string.
-                                        # All servers in the cluster must be able to connect to
+				 # All servers in the cluster must be able to connect to
-                                        # the local node using this string.
+				 # the local node using this string.
-                                        #
+				 #
-                                        # For details on conninfo strings, see:
+				 # For details on conninfo strings, see:
-                                        #  https://www.postgresql.org/docs/current/static/libpq-connect.html#LIBPQ-CONNSTRING
+				 #  https://www.postgresql.org/docs/current/static/libpq-connect.html#LIBPQ-CONNSTRING
-                                        #
+				 #
-                                        # If repmgrd is in use, consider explicitly setting
+				 # If repmgrd is in use, consider explicitly setting
-                                        # "connect_timeout" in the conninfo string to determine
+				 # "connect_timeout" in the conninfo string to determine
-                                        # the length of time which elapses before a network
+				 # the length of time which elapses before a network
-                                        # connection attempt is abandoned; for details see:
+				 # connection attempt is abandoned; for details see:
-                                        #  https://www.postgresql.org/docs/current/static/libpq-connect.html#LIBPQ-CONNECT-CONNECT-TIMEOUT
+				 #  https://www.postgresql.org/docs/current/static/libpq-connect.html#LIBPQ-CONNECT-CONNECT-TIMEOUT
 
-#data_directory                         # The node's data directory. This is needed by repmgr
+#data_directory=''		 # The node's data directory. This is needed by repmgr
-                                        # when performing operations when the PostgreSQL instance
+				 # when performing operations when the PostgreSQL instance
-                                        # is not running and there's no other way of determining
+				 # is not running and there's no other way of determining
-                                        # the data directory.
+				 # the data directory.
 
-#replication_user                       # User to make replication connections with, if not set defaults
+#replication_user='repmgr'	 # User to make replication connections with, if not set defaults
-                                        # to the user defined in "conninfo".
+				 # to the user defined in "conninfo".
 
 # =============================================================================
 
@@ -52,28 +52,28 @@
 # Replication settings
 #------------------------------------------------------------------------------
 
-#replication_type=physical              # Must be one of 'physical' or 'bdr'.
+#replication_type=physical	 # Must be one of 'physical' or 'bdr'.
 
-#location=default                       # arbitrary string defining the location of the node; this
+#location=default		 # arbitrary string defining the location of the node; this
-                                        # is used during failover to check visibilty of the
+				 # is used during failover to check visibilty of the
-                                        # current primary node. See the 'repmgrd' documentation
+				 # current primary node. See the 'repmgrd' documentation
-                                        # in README.md for further details.
+				 # in README.md for further details.
 
-#use_replication_slots=no               # whether to use physical replication slots
+#use_replication_slots=no	 # whether to use physical replication slots
-                                        # NOTE: when using replication slots,
+				 # NOTE: when using replication slots,
-                                        # 'max_replication_slots' should be configured for
+				 # 'max_replication_slots' should be configured for
-                                        # at least the number of standbys which will connect
+				 # at least the number of standbys which will connect
-                                        # to the primary.
+				 # to the primary.
 
-#recovery_min_apply_delay=              # If provided, "recovery_min_apply_delay" in recovery.conf
+#recovery_min_apply_delay=	 # If provided, "recovery_min_apply_delay" in recovery.conf
-                                        # will be set to this value.
+				 # will be set to this value.
 
 #------------------------------------------------------------------------------
 # Witness server settings
 #------------------------------------------------------------------------------
 
-#witness_sync_interval=15               # interval (in seconds) to synchronise node records
+#witness_sync_interval=15	 # interval (in seconds) to synchronise node records
-                                        # to the witness server
+				 # to the witness server
 
 #------------------------------------------------------------------------------
 # Logging settings
@@ -85,14 +85,14 @@
 # This is mainly intended for those cases when `repmgr` is executed directly
 # by `repmgrd`.
 
-#log_level=INFO                         # Log level: possible values are DEBUG, INFO, NOTICE,
+#log_level=INFO			 # Log level: possible values are DEBUG, INFO, NOTICE,
-                                        # WARNING, ERROR, ALERT, CRIT or EMERG
+				 # WARNING, ERROR, ALERT, CRIT or EMERG
 
-#log_facility=STDERR                    # Logging facility: possible values are STDERR, or for
+#log_facility=STDERR		 # Logging facility: possible values are STDERR, or for
-                                        # syslog integration, one of LOCAL0, LOCAL1, ..., LOCAL7, USER
+				 # syslog integration, one of LOCAL0, LOCAL1, ..., LOCAL7, USER
 
-#log_file=''                            # stderr can be redirected to an arbitrary file:
+#log_file=''			 # stderr can be redirected to an arbitrary file:
-#log_status_interval=300                # interval (in seconds) for repmgrd to log a status message
+#log_status_interval=300	 # interval (in seconds) for repmgrd to log a status message
 
 
 #------------------------------------------------------------------------------
@@ -118,28 +118,28 @@
 #
 #   event_notifications=primary_register,standby_register
 
-#event_notification_command=''          # An external program or script which
+#event_notification_command=''		# An external program or script which
-                                        # can be executed by the user under which
+					# can be executed by the user under which
-                                        # repmgr/repmgrd are run.
+					# repmgr/repmgrd are run.
 
-#event_notifications=''                 # A commas-separated list of notification
+#event_notifications=''			# A commas-separated list of notification
-                                        # types
+					# types
 
 #------------------------------------------------------------------------------
 # Environment/command settings
 #------------------------------------------------------------------------------
 
-#pg_bindir=''                           # Path to PostgreSQL binary directory (location
+#pg_bindir=''				# Path to PostgreSQL binary directory (location
-                                        # of pg_ctl, pg_basebackup etc.). Only needed
+					# of pg_ctl, pg_basebackup etc.). Only needed
-                                        # if these files are not in the system $PATH.
+					# if these files are not in the system $PATH.
-                                        #
+					#
-                                        # Debian/Ubuntu users: you will probably need to
+					# Debian/Ubuntu users: you will probably need to
-                                        # set this to the directory where `pg_ctl` is located,
+					# set this to the directory where `pg_ctl` is located,
-                                        # e.g. /usr/lib/postgresql/9.6/bin/
+					# e.g. /usr/lib/postgresql/9.6/bin/
-#use_primary_conninfo_password=false    # explicitly set "password" in recovery.conf's
+#use_primary_conninfo_password=false	# explicitly set "password" in recovery.conf's
-                                        # "primary_conninfo" parameter using the value contained
+					# "primary_conninfo" parameter using the value contained
-                                        # in the environment variable PGPASSWORD
+					# in the environment variable PGPASSWORD
-#passfile=''                            # path to .pgpass file to include in "primary_conninfo"
+#passfile=''				# path to .pgpass file to include in "primary_conninfo"
 #------------------------------------------------------------------------------
 # external command options
 #------------------------------------------------------------------------------
@@ -153,11 +153,10 @@
 #   rsync_options=--archive --checksum --compress --progress --rsh="ssh -o \"StrictHostKeyChecking no\""
 #   ssh_options=-o "StrictHostKeyChecking no"
 
-#pg_ctl_options=''                      # Options to append to "pg_ctl"
+#pg_ctl_options='' 			# Options to append to "pg_ctl"
-#pg_basebackup_options=''               # Options to append to "pg_basebackup"
+#pg_basebackup_options=''		# Options to append to "pg_basebackup"
-#rsync_options=''                       # Options to append to "rsync"
+#rsync_options=''			# Options to append to "rsync"
-ssh_options='-q -o ConnectTimeout=10'   # Options to append to "ssh"
+ssh_options='-q -o ConnectTimeout=10'	# Options to append to "ssh"
-
 
 
 
@@ -172,12 +171,12 @@ ssh_options='-q -o ConnectTimeout=10'   # Options to append to "ssh"
 #   tablespace_mapping=/path/to/original/tablespace=/path/to/new/tablespace
 #   restore_command = 'cp /path/to/archived/wals/%f %p'
 
-#tablespace_mapping=''                  # Tablespaces can be remapped from one
+#tablespace_mapping=''	  		# Tablespaces can be remapped from one
-                                        # file system location to another. This
+					# file system location to another. This
-                                        # parameter can be provided multiple times.
+					# parameter can be provided multiple times.
 
-#restore_command=''                     # This will be placed in the recovery.conf
+#restore_command=''			# This will be placed in the recovery.conf
-                                        # file generated by repmgr
+					# file generated by repmgr
 
 #------------------------------------------------------------------------------
 # Standby follow settings
@@ -186,19 +185,19 @@ ssh_options='-q -o ConnectTimeout=10'   # Options to append to "ssh"
 # These settings apply when instructing a standby to follow the new primary
 # ("repmgr standby follow").
 
-#primary_follow_timeout=60              # The length of time (in seconds) to wait
+#primary_follow_timeout=60		# The length of time (in seconds) to wait
-                                        # for the new primary to become available
+					# for the new primary to become available
 
 
 #------------------------------------------------------------------------------
 # Barman options
 #------------------------------------------------------------------------------
 
-#barman_server=''                       # The barman configuration section
+#barman_server=''			# The barman configuration section
-#barman_host=''                         # The host name of the barman server
+#barman_host=''				# The host name of the barman server
-#barman_config=''                       # The Barman configuration file on the
+#barman_config=''			# The Barman configuration file on the
-                                        # Barman server (needed if the file is
+					# Barman server (needed if the file is
-                                        # in a non-standard location)
+					# in a non-standard location)
 
 #------------------------------------------------------------------------------
 # Failover and monitoring settings (repmgrd)
@@ -207,42 +206,42 @@ ssh_options='-q -o ConnectTimeout=10'   # Options to append to "ssh"
 # These settings are only applied when repmgrd is running. Values shown
 # are defaults.
 
-#failover=manual                        # one of 'automatic', 'manual'.
+#failover=manual			# one of 'automatic', 'manual'.
-                                        # determines what action to take in the event of upstream failure
+					# determines what action to take in the event of upstream failure
-                                        #
+					#
-                                        # 'automatic': repmgrd will automatically attempt to promote the
+					# 'automatic': repmgrd will automatically attempt to promote the
-                                        #    node or follow the new upstream node
+					#    node or follow the new upstream node
-                                        # 'manual': repmgrd will take no action and the node will require
+					# 'manual': repmgrd will take no action and the node will require
-                                        #    manual attention to reattach it to replication
+					#    manual attention to reattach it to replication
-                                        # (does not apply to BDR mode)
+					# (does not apply to BDR mode)
 
-#priority=100                           # indicate a preferred priorty for promoting nodes;
+#priority=100				# indicate a preferred priorty for promoting nodes;
-                                        # a value of zero prevents the node being promoted to primary
+					# a value of zero prevents the node being promoted to primary
-                                        # (default: 100)
+					# (default: 100)
 
-#reconnect_attempts=6                   # Number attempts which will be made to reconnect to an unreachable
+#reconnect_attempts=6			# Number attempts which will be made to reconnect to an unreachable
-                                        # primary (or other upstream node)
+					# primary (or other upstream node)
-#reconnect_interval=10                  # Interval between attempts  to reconnect to an unreachable
+#reconnect_interval=10			# Interval between attempts  to reconnect to an unreachable
-                                        # primary (or other upstream node)
+					# primary (or other upstream node)
-#promote_command=                       # command to execute when promoting a new primary; use something like:
+#promote_command=			# command to execute when promoting a new primary; use something like:
-                                        #
+					#
-                                        #     repmgr standby promote -f /etc/repmgr.conf
+					#     repmgr standby promote -f /etc/repmgr.conf
-                                        #
+					#
-#follow_command=                        # command to execute when instructing a standby to follow a new primary;
+#follow_command=			# command to execute when instructing a standby to follow a new primary;
-                                        # use something like:
+					# use something like:
-                                        #
+					#
-                                        #     repmgr standby follow -f /etc/repmgr.conf -W --upstream-node-id=%n
+					#     repmgr standby follow -f /etc/repmgr.conf -W --upstream-node-id=%n
-                                        #
+					#
-#primary_notification_timeout=60        # Interval (in seconds) which repmgrd on a standby
+#primary_notification_timeout=60	# Interval (in seconds) which repmgrd on a standby
-                                        # will wait for a notification from the new primary,
+					# will wait for a notification from the new primary,
-                                        # before falling back to degraded monitoring
+					# before falling back to degraded monitoring
 #monitoring_history=no
 
-#degraded_monitoring_timeout=-1         # Interval (in seconds) after which repmgrd will terminate if the
+#degraded_monitoring_timeout=-1		# Interval (in seconds) after which repmgrd will terminate if the
-                                        # server being monitored is no longer available. -1 (default)
+					# server being monitored is no longer available. -1 (default)
-                                        # disables the timeout completely.
+					# disables the timeout completely.
-#async_query_timeout=60                 # Interval (in seconds) which repmgrd will wait before
+#async_query_timeout=60			# Interval (in seconds) which repmgrd will wait before
-                                        # cancelling an asynchronous query.
+					# cancelling an asynchronous query.
 
 #------------------------------------------------------------------------------
 # service control commands
@@ -275,10 +274,10 @@ ssh_options='-q -o ConnectTimeout=10'   # Options to append to "ssh"
 #service_stop_command = ''
 #service_restart_command = ''
 #service_reload_command = ''
-#service_promote_command = ''            # Note: this overrides any value contained in the setting
+#service_promote_command = ''		# Note: this overrides any value contained in the setting
-                                         # "promote_command". This is intended for systems which
+					# "promote_command". This is intended for systems which
-                                         # provide a package-level promote command, such as Debian's
+					# provide a package-level promote command, such as Debian's
-                                         # "pg_ctlcluster"
+					# "pg_ctlcluster"
 
 
 #------------------------------------------------------------------------------
@@ -287,25 +286,25 @@ ssh_options='-q -o ConnectTimeout=10'   # Options to append to "ssh"
 
 # Various warning/critical thresholds used by "repmgr node check".
 
-#archive_ready_warning=16                # repmgr node check --archiver
+#archive_ready_warning=16  	      	# repmgr node check --archive-ready
-#archive_ready_critical=128              #
+#archive_ready_critical=128		#
-                                         # Numbers of files pending archiving via PostgreSQL's
+					# Numbers of files pending archiving via PostgreSQL's
-                                         # "archive_command" configuration parameter. If
+					# "archive_command" configuration parameter. If
-                                         # files can't be archived fast enough, or the archive
+					# files can't be archived fast enough, or the archive
-                                         # command is failing, the buildup of files can
+					# command is failing, the buildup of files can
-                                         # cause various issues, such as server shutdown being
+					# cause various issues, such as server shutdown being
-                                         # delayed until all files are archived, or excessive
+					# delayed until all files are archived, or excessive
-                                         # space being occupied by unarchived files.
+					# space being occupied by unarchived files.
-                                         #
+					#
-                                         # Note that these values will be checked when executing
+					# Note that these values will be checked when executing
-                                         # "repmgr standby switchover" to warn about potential
+					# "repmgr standby switchover" to warn about potential
-                                         # issues with shutting down the demotion candidate.
+					# issues with shutting down the demotion candidate.
 
-#replication_lag_warning=300             # repmgr node check --replication-lag
+#replication_lag_warning=300		# repmgr node check --replication-lag
-#replication_lag_critical=600            #
+#replication_lag_critical=600		#
-                                         # Note that these values will be checked when executing
+					# Note that these values will be checked when executing
-                                         # "repmgr standby switchover" to warn about potential
+					# "repmgr standby switchover" to warn about potential
-                                         # issues with shutting down the demotion candidate.
+					# issues with shutting down the demotion candidate.
 
 
 #------------------------------------------------------------------------------

repmgr.h

@@ -1,6 +1,6 @@
 /*
  * repmgr.h
- * Copyright (c) 2ndQuadrant, 2010-2017
+ * Copyright (c) 2ndQuadrant, 2010-2018
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -76,6 +76,7 @@
 #define	DEFAULT_REPLICATION_LAG_WARNING      300 /* seconds */
 #define DEFAULT_REPLICATION_LAG_CRITICAL     600 /* seconds */
 #define DEFAULT_WITNESS_SYNC_INTERVAL        15  /* seconds */
+#define DEFAULT_WAIT_START                   30  /* seconds */
 
 #ifndef RECOVERY_COMMAND_FILE
 #define RECOVERY_COMMAND_FILE "recovery.conf"

repmgr_version.h.in

@@ -1,3 +1,3 @@
 #define REPMGR_VERSION_DATE ""
-#define REPMGR_VERSION "4.0.1"
+#define REPMGR_VERSION "4.0.2"
 

repmgrd-bdr.c

@@ -1,7 +1,7 @@
 /*
  * repmgrd-bdr.c - BDR functionality for repmgrd
  *
- * Copyright (c) 2ndQuadrant, 2010-2017
+ * Copyright (c) 2ndQuadrant, 2010-2018
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -124,9 +124,9 @@ monitor_bdr(void)
 		exit(ERR_BAD_CONFIG);
 	}
 
-	if (is_active_bdr_node(local_conn, local_node_info.node_name))
+	if (is_active_bdr_node(local_conn, local_node_info.node_name) == false)
 	{
-		log_error(_("BDR node %s is not active, terminating"),
+		log_error(_("BDR node \"%s\" is not active, terminating"),
 				  local_node_info.node_name);
 		PQfinish(local_conn);
 		exit(ERR_BAD_CONFIG);

repmgrd-bdr.h

@@ -1,6 +1,6 @@
 /*
  * repmgrd-bdr.h
- * Copyright (c) 2ndQuadrant, 2010-2017
+ * Copyright (c) 2ndQuadrant, 2010-2018
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by

repmgrd-physical.c

@@ -1,7 +1,7 @@
 /*
- * repmgrd-physical.c - physical replication functionality for repmgrd
+ * repmgrd-physical.c - physical (streaming) replication functionality for repmgrd
  *
- * Copyright (c) 2ndQuadrant, 2010-2017
+ * Copyright (c) 2ndQuadrant, 2010-2018
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -107,11 +107,11 @@ do_physical_node_check(void)
 
 	if (local_node_info.active == false)
 	{
-		char	   *hint = "Check that 'repmgr (primary|standby) register' was executed for this node";
+		char	   *hint = "Check that \"repmgr (primary|standby) register\" was executed for this node";
 
 		switch (config_file_options.failover)
 		{
-				/* "failover" is an enum, all values should be covered here */
+			/* "failover" is an enum, all values should be covered here */
 
 			case FAILOVER_AUTOMATIC:
 				log_error(_("this node is marked as inactive and cannot be used as a failover target"));
@@ -935,22 +935,19 @@ loop:
 
 						local_node_info.active = false;
 
-						appendPQExpBuffer(
+						appendPQExpBuffer(&event_details,
-										  &event_details,
 										  _("unable to connect to local node \"%s\" (ID: %i), marking inactive"),
 										  local_node_info.node_name,
 										  local_node_info.node_id);
 
-						log_warning("%s", event_details.data)
+						log_warning("%s", event_details.data);
 
-
+						create_event_notification(primary_conn,
-							create_event_notification(
+												  &config_file_options,
-													  primary_conn,
+												  local_node_info.node_id,
-													  &config_file_options,
+												  "standby_failure",
-													  local_node_info.node_id,
+												  false,
-													  "standby_failure",
+												  event_details.data);
-													  false,
-													  event_details.data);
 
 						termPQExpBuffer(&event_details);
 					}
@@ -971,8 +968,7 @@ loop:
 
 						local_node_info.active = true;
 
-						appendPQExpBuffer(
+						appendPQExpBuffer(&event_details,
-										  &event_details,
 										  _("reconnected to local node \"%s\" (ID: %i), marking active"),
 										  local_node_info.node_name,
 										  local_node_info.node_id);

repmgrd-physical.h

@@ -1,6 +1,6 @@
 /*
  * repmgrd-physical.h
- * Copyright (c) 2ndQuadrant, 2010-2017
+ * Copyright (c) 2ndQuadrant, 2010-2018
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by

repmgrd.c

@@ -1,7 +1,7 @@
 /*
  * repmgrd.c - Replication manager daemon
  *
- * Copyright (c) 2ndQuadrant, 2010-2017
+ * Copyright (c) 2ndQuadrant, 2010-2018
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by

repmgrd.h

@@ -1,6 +1,6 @@
 /*
  * repmgrd.h
- * Copyright (c) 2ndQuadrant, 2010-2017
+ * Copyright (c) 2ndQuadrant, 2010-2018
  */
 
 

strutil.c

@@ -1,7 +1,7 @@
 /*
  * strutil.c
  *
- * Copyright (c) 2ndQuadrant, 2010-2017
+ * Copyright (c) 2ndQuadrant, 2010-2018
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by

strutil.h

@@ -1,6 +1,6 @@
 /*
  * strutil.h
- * Copyright (c) 2ndQuadrant, 2010-2017
+ * Copyright (c) 2ndQuadrant, 2010-2018
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by

voting.h

@@ -1,6 +1,6 @@
 /*
  * voting.h
- * Copyright (c) 2ndQuadrant, 2010-2017
+ * Copyright (c) 2ndQuadrant, 2010-2018
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by