Update HISTORY

Fixes GitHub #280

CONTRIBUTING.md

@@ -2,7 +2,7 @@ License and Contributions
 =========================
 
 `repmgr` is licensed under the GPL v3.  All of its code and documentation is
-Copyright 2010-2016, 2ndQuadrant Limited.  See the files COPYRIGHT and LICENSE for
+Copyright 2010-2017, 2ndQuadrant Limited.  See the files COPYRIGHT and LICENSE for
 details.
 
 The development of repmgr has primarily been sponsored by 2ndQuadrant customers.

COPYRIGHT

@@ -1,4 +1,4 @@
-Copyright (c) 2010-2016, 2ndQuadrant Limited
+Copyright (c) 2010-2017, 2ndQuadrant Limited
 All rights reserved.
 
 This program is free software: you can redistribute it and/or modify

FAQ.md

@@ -38,7 +38,7 @@ General
 
   No. Hash indexes and replication do not mix well and their use is
   explicitly discouraged; see:
-    http://www.postgresql.org/docs/current/interactive/sql-createindex.html#AEN74175
+    https://www.postgresql.org/docs/current/interactive/sql-createindex.html#AEN74175
 
 `repmgr`
 --------
@@ -137,6 +137,7 @@ General
   of events which includes servers removed from the replication cluster
   which no longer have an entry in the `repl_nodes` table.
 
+
 `repmgrd`
 ---------
 
@@ -151,6 +152,9 @@ General
 
   In `repmgr.conf`, set its priority to a value of 0 or less.
 
+  Additionally, if `failover` is set to `manual`, the node will never
+  be considered as a promotion candidate.
+
 - Does `repmgrd` support delayed standbys?
 
   `repmgrd` can monitor delayed standbys - those set up with
@@ -169,3 +173,11 @@ General
 
   Configure your system's `logrotate` service to do this; see example
   in README.md
+
+- I've recloned a failed master as a standby, but `repmgrd` refuses to start?
+
+  Check you registered the standby after recloning. If unregistered the standby
+  cannot be considered as a promotion candidate even if `failover` is set to
+  `automatic`, which is probably not what you want. `repmgrd` will start if
+  `failover` is set to `manual` so the node's replication status can still
+  be monitored, if desired.

HISTORY

@@ -1,3 +1,99 @@
+3.3.1   2017-03-
+        repmgrd: prevent invalid apply lag value being written to the
+          monitoring table (Ian)
+        repmgrd: fix error in XLogRecPtr conversion when calculating
+          monitoring statistics (Ian)
+        repmgr: if replication slots in use, where possible delete slot on old
+          upstream node after following new upstream (Ian)
+        repmgr: improve logging of rsync actions (Ian)
+        repmgr: improve `standby clone` when synchronous replication in use (Ian)
+        repmgr: stricter checking of allowed node id values
+        repmgr: enable `master register --force` when there is a foreign key
+          dependency from a standby node (Ian)
+
+3.3     2016-12-27
+        repmgr: always log to STDERR even if log facility defined (Ian)
+        repmgr: add --log-to-file to log repmgr output to the defined
+          log facility (Ian)
+        repmgr: improve handling of command line parameter errors (Ian)
+        repmgr: add option --upstream-conninfo to explicitly set
+          'primary_conninfo' in recovery.conf (Ian)
+        repmgr: enable a standby to be registered which isn't running (Ian)
+        repmgr: enable `standby register --force` to update a node record
+          with cascaded downstream node records (Ian)
+        repmgr: add option `--no-conninfo-password` (Abhijit, Ian)
+        repmgr: add initial support for PostgreSQL 10.0 (Ian)
+        repmgr: escape values in primary_conninfo if needed (Ian)
+
+3.2.1   2016-10-24
+        repmgr: require a valid repmgr cluster name unless -F/--force
+          supplied (Ian)
+        repmgr: check master server is registered with repmgr before
+          cloning (Ian)
+        repmgr: ensure data directory defaults to that of the source node (Ian)
+        repmgr: various fixes to Barman cloning mode (Gianni, Ian)
+        repmgr: fix `repmgr cluster crosscheck` output (Ian)
+
+3.2     2016-10-05
+        repmgr: add support for cloning from a Barman backup (Gianni)
+        repmgr: add commands `standby matrix` and `standby crosscheck` (Gianni)
+        repmgr: suppress connection error display in `repmgr cluster show`
+          unless `--verbose` supplied (Ian)
+        repmgr: add commands `witness register` and `witness unregister` (Ian)
+        repmgr: enable `standby unregister` / `witness unregister` to be
+          executed for a node which is not running (Ian)
+        repmgr: remove deprecated command line options --initdb-no-pwprompt and
+           -l/--local-port (Ian)
+        repmgr: before cloning with pg_basebackup, check that sufficient free
+           walsenders are available (Ian)
+        repmgr: add option `--wait-sync` for `standby register` which causes
+           repmgr to wait for the registered node record to synchronise to
+           the standby (Ian)
+        repmgr: add option `--copy-external-config-files` for files outside
+           of the data directory (Ian)
+        repmgr: only require `wal_keep_segments` to be set in certain corner
+           cases (Ian)
+        repmgr: better support cloning from a node other than the one to
+           stream from (Ian)
+        repmgrd: add configuration options to override the default pg_ctl
+           commands (Jarkko Oranen, Ian)
+        repmgrd: don't start if node is inactive and failover=automatic (Ian)
+        packaging: improve "repmgr-auto" Debian package (Gianni)
+
+
+3.1.5   2016-08-15
+        repmgrd: in a failover situation, prevent endless looping when
+          attempting to establish the status of a node with
+          `failover=manual` (Ian)
+        repmgrd: improve handling of failover events on standbys with
+          `failover=manual`, and create a new event notification
+          for this, `standby_disconnect_manual` (Ian)
+        repmgr: add further event notifications (Gianni)
+        repmgr: when executing `standby switchover`, don't collect remote
+          command output unless required (Gianni, Ian)
+        repmgrd: improve standby monitoring query (Ian, based on suggestion
+          from  Álvaro)
+        repmgr: various command line handling improvements (Ian)
+
+3.1.4   2016-07-12
+        repmgr: new configuration option for setting "restore_command"
+          in the recovery.conf file generated by repmgr (Martín)
+        repmgr: add --csv option to "repmgr cluster show" (Gianni)
+        repmgr: enable provision of a conninfo string as the -d/--dbname
+          parameter, similar to other PostgreSQL utilities (Ian)
+        repmgr: during switchover operations improve detection of
+          demotion candidate shutdown (Ian)
+        various bugfixes and documentation updates (Ian, Martín)
+
+3.1.3   2016-05-17
+        repmgrd: enable monitoring when a standby is catching up by
+          replaying archived WAL (Ian)
+        repmgrd: when upstream_node_id is NULL, assume upstream node
+          to be current master (Ian)
+        repmgrd: check for reappearance of the master node if standby
+          promotion fails (Ian)
+        improve handling of rsync failure conditions (Martín)
+
 3.1.2   2016-04-12
         Fix pg_ctl path generation in do_standby_switchover() (Ian)
         Regularly sync witness server repl_nodes table (Ian)

Makefile

@@ -1,11 +1,11 @@
 #
 # Makefile
-# Copyright (c) 2ndQuadrant, 2010-2016
+# Copyright (c) 2ndQuadrant, 2010-2017
 
 HEADERS = $(wildcard *.h)
 
 repmgrd_OBJS = dbutils.o config.o repmgrd.o log.o strutil.o
-repmgr_OBJS = dbutils.o check_dir.o config.o repmgr.o log.o strutil.o
+repmgr_OBJS = dbutils.o check_dir.o config.o repmgr.o log.o strutil.o dirmod.o compat.o
 
 DATA = repmgr.sql uninstall_repmgr.sql
 
@@ -87,10 +87,12 @@ PG_VERSION = $(shell pg_config --version | cut -d ' ' -f 2 | cut -d '.' -f 1,2)
 REPMGR_VERSION = $(shell grep REPMGR_VERSION version.h | cut -d ' ' -f 3 | cut -d '"' -f 2)
 PKGLIBDIR = $(shell pg_config --pkglibdir)
 SHAREDIR = $(shell pg_config --sharedir)
+PGBINDIR = /usr/lib/postgresql/$(PG_VERSION)/bin
 
 deb: repmgrd repmgr
-	mkdir -p ./debian/usr/bin
+	mkdir -p ./debian/usr/bin ./debian$(PGBINDIR)
-	cp repmgrd repmgr ./debian/usr/bin/
+	cp repmgrd repmgr ./debian$(PGBINDIR)
+	ln -s ../..$(PGBINDIR)/repmgr ./debian/usr/bin/repmgr
 	mkdir -p ./debian$(SHAREDIR)/contrib/
 	cp sql/repmgr_funcs.sql ./debian$(SHAREDIR)/contrib/
 	cp sql/uninstall_repmgr_funcs.sql ./debian$(SHAREDIR)/contrib/

RHEL/repmgr3-93.spec

@@ -1,61 +0,0 @@
-Summary: repmgr
-Name: repmgr
-Version: 3.0
-Release: 1
-License: GPLv3
-Group: System Environment/Daemons
-URL: http://repmgr.org
-Packager: Ian Barwick <ian@2ndquadrant.com>
-Vendor: 2ndQuadrant Limited
-Distribution: centos
-Source0: %{name}-%{version}.tar.gz
-BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root
-
-%description
-repmgr is a utility suite which greatly simplifies
-the process of setting up and managing replication
-using streaming replication within a cluster of
-PostgreSQL servers.
-
-%prep
-%setup
-
-%build
-export PATH=$PATH:/usr/pgsql-9.3/bin/
-%{__make} USE_PGXS=1
-
-%install
-[ "%{buildroot}" != "/" ] && %{__rm} -rf %{buildroot}
-
-export PATH=$PATH:/usr/pgsql-9.3/bin/
-%{__make} USE_PGXS=1 install DESTDIR=%{buildroot} INSTALL="install -p"
-%{__make} USE_PGXS=1 install_prog DESTDIR=%{buildroot} INSTALL="install -p"
-%{__make} USE_PGXS=1 install_rhel DESTDIR=%{buildroot} INSTALL="install -p"
-
-
-%clean
-[ "%{buildroot}" != "/" ] && %{__rm} -rf %{buildroot}
-
-
-%files
-%defattr(-,root,root)
-/usr/bin/repmgr
-/usr/bin/repmgrd
-/usr/pgsql-9.3/bin/repmgr
-/usr/pgsql-9.3/bin/repmgrd
-/usr/pgsql-9.3/lib/repmgr_funcs.so
-/usr/pgsql-9.3/share/contrib/repmgr.sql
-/usr/pgsql-9.3/share/contrib/repmgr_funcs.sql
-/usr/pgsql-9.3/share/contrib/uninstall_repmgr.sql
-/usr/pgsql-9.3/share/contrib/uninstall_repmgr_funcs.sql
-%attr(0755,root,root)/etc/init.d/repmgrd
-%attr(0644,root,root)/etc/sysconfig/repmgrd
-%attr(0644,root,root)/etc/repmgr/repmgr.conf.sample
-
-%changelog
-* Tue Mar 10 2015 Ian Barwick ian@2ndquadrant.com>
-- build for repmgr 3.0
-* Thu Jun 05 2014 Nathan Van Overloop <nathan.van.overloop@nexperteam.be> 2.0.2
-- fix witness creation to create db and user if needed
-* Fri Apr 04 2014 Nathan Van Overloop <nathan.van.overloop@nexperteam.be> 2.0.1
-- initial build for RHEL6

RHEL/repmgrd.init

@@ -1,133 +0,0 @@
-#!/bin/sh
-#
-# chkconfig: - 75 16
-# description: Enable repmgrd replication management and monitoring daemon for PostgreSQL
-# processname: repmgrd
-# pidfile="/var/run/${NAME}.pid"
-
-# Source function library.
-INITD=/etc/rc.d/init.d
-. $INITD/functions
-
-# Get function listing for cross-distribution logic.
-TYPESET=`typeset -f|grep "declare"`
-
-# Get network config.
-. /etc/sysconfig/network
-
-DESC="PostgreSQL replication management and monitoring daemon"
-NAME=repmgrd
-
-REPMGRD_ENABLED=no
-REPMGRD_OPTS=
-REPMGRD_USER=postgres
-REPMGRD_BIN=/usr/pgsql-9.3/bin/repmgrd
-REPMGRD_PIDFILE=/var/run/repmgrd.pid
-REPMGRD_LOCK=/var/lock/subsys/${NAME}
-REPMGRD_LOG=/var/lib/pgsql/9.3/data/pg_log/repmgrd.log
-
-# Read configuration variable file if it is present
-[ -r /etc/sysconfig/$NAME ] && . /etc/sysconfig/$NAME
-
-# For SELinux we need to use 'runuser' not 'su'
-if [ -x /sbin/runuser ]
-then
-    SU=runuser
-else
-    SU=su
-fi
-
-test -x $REPMGRD_BIN || exit 0
-
-case "$REPMGRD_ENABLED" in
-    [Yy]*)
-	break
-	;;
-    *)
-	exit 0
-	;;
-esac
-
-
-if [ -z "${REPMGRD_OPTS}" ]
-then
-    echo "Not starting ${NAME}, REPMGRD_OPTS not set in /etc/sysconfig/${NAME}"
-    exit 0
-fi
-
-start()
-{
-    REPMGRD_START=$"Starting ${NAME} service: "
-
-    # Make sure startup-time log file is valid
-    if [ ! -e "${REPMGRD_LOG}" -a ! -h "${REPMGRD_LOG}" ]
-    then
-        touch "${REPMGRD_LOG}" || exit 1
-        chown ${REPMGRD_USER}:postgres "${REPMGRD_LOG}"
-        chmod go-rwx "${REPMGRD_LOG}"
-        [ -x /sbin/restorecon ] && /sbin/restorecon "${REPMGRD_LOG}"
-    fi
-
-    echo -n "${REPMGRD_START}"
-    $SU -l $REPMGRD_USER -c "${REPMGRD_BIN} ${REPMGRD_OPTS} -p ${REPMGRD_PIDFILE} &" >> "${REPMGRD_LOG}" 2>&1 < /dev/null
-    sleep 2
-    pid=`head -n 1 "${REPMGRD_PIDFILE}" 2>/dev/null`
-    if [ "x${pid}" != "x" ]
-    then
-        success "${REPMGRD_START}"
-        touch "${REPMGRD_LOCK}"
-        echo $pid > "${REPMGRD_PIDFILE}"
-        echo
-    else
-        failure "${REPMGRD_START}"
-        echo
-        script_result=1
-    fi
-}
-
-stop()
-{
-    echo -n $"Stopping ${NAME} service: "
-    if [ -e "${REPMGRD_LOCK}" ]
-    then
-        killproc ${NAME}
-        ret=$? 
-        if [ $ret -eq 0 ]
-        then
-            echo_success
-            rm -f "${REPMGRD_PIDFILE}"
-            rm -f "${REPMGRD_LOCK}"
-        else
-            echo_failure
-            script_result=1
-        fi
-    else
-        # not running; per LSB standards this is "ok"   
-        echo_success
-    fi
-    echo
-}
-
-
-# See how we were called.
-case "$1" in
-  start)
-        start
-        ;;
-  stop)
-        stop
-        ;;
-  status)
-        status -p $REPMGRD_PIDFILE $NAME
-        script_result=$?
-        ;;
-  restart)
-        stop
-	start
-        ;;
-  *)
-        echo $"Usage: $0 {start|stop|status|restart}"
-        exit 2
-esac
-
-exit $script_result

RHEL/repmgrd.sysconfig

@@ -1,21 +0,0 @@
-# default settings for repmgrd. This file is source by /bin/sh from
-# /etc/init.d/repmgrd
-
-# disable repmgrd by default so it won't get started upon installation
-# valid values: yes/no
-REPMGRD_ENABLED=no
-
-# Options for repmgrd (required)
-#REPMGRD_OPTS="--verbose -d -f /var/lib/pgsql/repmgr/repmgr.conf"
-
-# User to run repmgrd as
-#REPMGRD_USER=postgres
-
-# repmgrd binary
-#REPMGRD_BIN=/usr/bin/repmgrd
-
-# pid file
-#REPMGRD_PIDFILE=/var/lib/pgsql/repmgr/repmgrd.pid
-
-# log file
-#REPMGRD_LOG=/var/lib/pgsql/repmgr/repmgrd.log

TODO

@@ -53,8 +53,9 @@ Planned feature improvements
   requested, activate the replication slot using pg_receivexlog to negate the
   need to set `wal_keep_segments` just for the initial clone (9.4 and 9.5).
 
-* Take into account the fact that a standby can obtain WAL from an archive,
+* repmgr: enable "standby follow" to point a standby at another standby, not
-  so even if direct streaming replication is interrupted, it may be up-to-date
+  just the replication cluster master (see GitHub #130)
+
 
 Usability improvements
 ======================

check_dir.c

@@ -1,6 +1,6 @@
 /*
  * check_dir.c - Directories management functions
- * Copyright (C) 2ndQuadrant, 2010-2016
+ * Copyright (c) 2ndQuadrant, 2010-2017
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by

check_dir.h

@@ -1,6 +1,6 @@
 /*
  * check_dir.h
- * Copyright (c) 2ndQuadrant, 2010-2016
+ * Copyright (c) 2ndQuadrant, 2010-2017
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by

compat.c

@@ -0,0 +1,107 @@
+/*
+ *
+ * compat.c
+ *	  Provides a couple of useful string utility functions adapted
+ *	  from the backend code, which are not publicly exposed. They're
+ *	  unlikely to change but it would be worth keeping an eye on them
+ *	  for any fixes/improvements
+ *
+ * Copyright (c) 2ndQuadrant, 2010-2017
+ *
+ * Portions Copyright (c) 1996-2013, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "repmgr.h"
+#include "compat.h"
+
+/*
+ * Append the given string to the buffer, with suitable quoting for passing
+ * the string as a value, in a keyword/pair value in a libpq connection
+ * string
+ *
+ * This function is adapted from src/fe_utils/string_utils.c (before 9.6
+ * located in: src/bin/pg_dump/dumputils.c)
+ */
+void
+appendConnStrVal(PQExpBuffer buf, const char *str)
+{
+	const char *s;
+	bool		needquotes;
+
+	/*
+	 * If the string is one or more plain ASCII characters, no need to quote
+	 * it. This is quite conservative, but better safe than sorry.
+	 */
+	needquotes = true;
+	for (s = str; *s; s++)
+	{
+		if (!((*s >= 'a' && *s <= 'z') || (*s >= 'A' && *s <= 'Z') ||
+			  (*s >= '0' && *s <= '9') || *s == '_' || *s == '.'))
+		{
+			needquotes = true;
+			break;
+		}
+		needquotes = false;
+	}
+
+	if (needquotes)
+	{
+		appendPQExpBufferChar(buf, '\'');
+		while (*str)
+		{
+			/* ' and \ must be escaped by to \' and \\ */
+			if (*str == '\'' || *str == '\\')
+				appendPQExpBufferChar(buf, '\\');
+
+			appendPQExpBufferChar(buf, *str);
+			str++;
+		}
+		appendPQExpBufferChar(buf, '\'');
+	}
+	else
+		appendPQExpBufferStr(buf, str);
+}
+
+/*
+ * Adapted from: src/fe_utils/string_utils.c
+ */
+void
+appendShellString(PQExpBuffer buf, const char *str)
+{
+	const char *p;
+
+	appendPQExpBufferChar(buf, '\'');
+	for (p = str; *p; p++)
+	{
+		if (*p == '\n' || *p == '\r')
+		{
+			fprintf(stderr,
+					_("shell command argument contains a newline or carriage return: \"%s\"\n"),
+					str);
+			exit(ERR_BAD_CONFIG);
+		}
+
+		if (*p == '\'')
+			appendPQExpBufferStr(buf, "'\"'\"'");
+		else
+			appendPQExpBufferChar(buf, *p);
+	}
+
+	appendPQExpBufferChar(buf, '\'');
+}
+

compat.h

@@ -0,0 +1,29 @@
+/*
+ * compat.h
+ * Copyright (c) 2ndQuadrant, 2010-2017
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef _COMPAT_H_
+#define _COMPAT_H_
+
+extern void
+appendConnStrVal(PQExpBuffer buf, const char *str);
+
+extern void
+appendShellString(PQExpBuffer buf, const char *str);
+
+#endif

config.c

@@ -1,6 +1,7 @@
 /*
  * config.c - Functions to parse the config file
- * Copyright (C) 2ndQuadrant, 2010-2016
+ *
+ * Copyright (c) 2ndQuadrant, 2010-2017
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -9,11 +10,11 @@
  *
  * This program is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ * along with this program.	 If not, see <http://www.gnu.org/licenses/>.
  *
  */
 
@@ -26,9 +27,9 @@
 
 static void parse_event_notifications_list(t_configuration_options *options, const char *arg);
 static void tablespace_list_append(t_configuration_options *options, const char *arg);
-static void exit_with_errors(ErrorList *config_errors);
+static void exit_with_errors(ItemList *config_errors);
 
-const static char *_progname = '\0';
+const static char *_progname = NULL;
 static char config_file_path[MAXPGPATH];
 static bool config_file_provided = false;
 bool config_file_found = false;
@@ -54,8 +55,8 @@ progname(void)
  *
  * Returns true if a configuration file could be parsed, otherwise false.
  *
- * Any configuration options changed in this function must also be changed in
+ * Any *repmgrd-specific* configuration options added/changed in this function must also be
- * reload_config()
+ * added/changed in reload_config()
  *
  * NOTE: this function is called before the logger is set up, so we need
  * to handle the verbose option ourselves; also the default log level is NOTICE,
@@ -98,9 +99,9 @@ load_config(const char *config_file, bool verbose, t_configuration_options *opti
 	/*
 	 * If no configuration file was provided, attempt to find a default file
 	 * in this order:
-	 *  - current directory
+	 *	- current directory
-	 *  - /etc/repmgr.conf
+	 *	- /etc/repmgr.conf
-	 *  - default sysconfdir
+	 *	- default sysconfdir
 	 *
 	 * here we just check for the existence of the file; parse_config()
 	 * will handle read errors etc.
@@ -180,6 +181,23 @@ load_config(const char *config_file, bool verbose, t_configuration_options *opti
 }
 
 
+bool
+parse_config(t_configuration_options *options)
+{
+	/* Collate configuration file errors here for friendlier reporting */
+	static ItemList config_errors = { NULL, NULL };
+
+	_parse_config(options, &config_errors);
+
+	if (config_errors.head != NULL)
+	{
+		exit_with_errors(&config_errors);
+	}
+
+	return true;
+}
+
+
 /*
  * Parse configuration file; if any errors are encountered,
  * list them and exit.
@@ -187,8 +205,8 @@ load_config(const char *config_file, bool verbose, t_configuration_options *opti
  * Ensure any default values set here are synced with repmgr.conf.sample
  * and any other documentation.
  */
-bool
+void
-parse_config(t_configuration_options *options)
+_parse_config(t_configuration_options *options, ItemList *error_list)
 {
 	FILE	   *fp;
 	char	   *s,
@@ -200,9 +218,6 @@ parse_config(t_configuration_options *options)
 	PQconninfoOption *conninfo_options;
 	char	   *conninfo_errmsg = NULL;
 
-	/* Collate configuration file errors here for friendlier reporting */
-	static ErrorList config_errors = { NULL, NULL };
-
 	bool		node_found = false;
 
 	/* Initialize configuration options with sensible defaults
@@ -210,20 +225,28 @@ parse_config(t_configuration_options *options)
 	 * to be initialised here
 	 */
 	memset(options->cluster_name, 0, sizeof(options->cluster_name));
-	options->node = -1;
+	options->node = UNKNOWN_NODE_ID;
 	options->upstream_node = NO_UPSTREAM_NODE;
 	options->use_replication_slots = 0;
 	memset(options->conninfo, 0, sizeof(options->conninfo));
+	memset(options->barman_server, 0, sizeof(options->barman_server));
+	memset(options->barman_config, 0, sizeof(options->barman_config));
 	options->failover = MANUAL_FAILOVER;
 	options->priority = DEFAULT_PRIORITY;
 	memset(options->node_name, 0, sizeof(options->node_name));
 	memset(options->promote_command, 0, sizeof(options->promote_command));
 	memset(options->follow_command, 0, sizeof(options->follow_command));
+	memset(options->service_stop_command, 0, sizeof(options->service_stop_command));
+	memset(options->service_start_command, 0, sizeof(options->service_start_command));
+	memset(options->service_restart_command, 0, sizeof(options->service_restart_command));
+	memset(options->service_reload_command, 0, sizeof(options->service_reload_command));
+	memset(options->service_promote_command, 0, sizeof(options->service_promote_command));
 	memset(options->rsync_options, 0, sizeof(options->rsync_options));
 	memset(options->ssh_options, 0, sizeof(options->ssh_options));
 	memset(options->pg_bindir, 0, sizeof(options->pg_bindir));
 	memset(options->pg_ctl_options, 0, sizeof(options->pg_ctl_options));
 	memset(options->pg_basebackup_options, 0, sizeof(options->pg_basebackup_options));
+	memset(options->restore_command, 0, sizeof(options->restore_command));
 
 	/* default master_response_timeout is 60 seconds */
 	options->master_response_timeout = 60;
@@ -239,6 +262,8 @@ parse_config(t_configuration_options *options)
 	options->witness_repl_nodes_sync_interval_secs = 30;
 
 	memset(options->event_notification_command, 0, sizeof(options->event_notification_command));
+	options->event_notifications.head = NULL;
+	options->event_notifications.tail = NULL;
 
 	options->tablespace_mapping.head = NULL;
 	options->tablespace_mapping.tail = NULL;
@@ -251,7 +276,7 @@ parse_config(t_configuration_options *options)
 	{
 		log_verbose(LOG_NOTICE, _("no configuration file provided and no default file found - "
 					 "continuing with default values\n"));
-		return true;
+		return;
 	}
 
 	fp = fopen(config_file_path, "r");
@@ -296,13 +321,17 @@ parse_config(t_configuration_options *options)
 			strncpy(options->cluster_name, value, MAXLEN);
 		else if (strcmp(name, "node") == 0)
 		{
-			options->node = repmgr_atoi(value, "node", &config_errors, false);
+			options->node = repmgr_atoi(value, "node", error_list, false);
 			node_found = true;
 		}
 		else if (strcmp(name, "upstream_node") == 0)
-			options->upstream_node = repmgr_atoi(value, "upstream_node", &config_errors, false);
+			options->upstream_node = repmgr_atoi(value, "upstream_node", error_list, false);
 		else if (strcmp(name, "conninfo") == 0)
 			strncpy(options->conninfo, value, MAXLEN);
+		else if (strcmp(name, "barman_server") == 0)
+			strncpy(options->barman_server, value, MAXLEN);
+		else if (strcmp(name, "barman_config") == 0)
+			strncpy(options->barman_config, value, MAXLEN);
 		else if (strcmp(name, "rsync_options") == 0)
 			strncpy(options->rsync_options, value, QUERY_STR_LEN);
 		else if (strcmp(name, "ssh_options") == 0)
@@ -327,28 +356,39 @@ parse_config(t_configuration_options *options)
 			}
 			else
 			{
-				error_list_append(&config_errors,_("value for 'failover' must be 'automatic' or 'manual'\n"));
+				item_list_append(error_list, _("value for 'failover' must be 'automatic' or 'manual'\n"));
 			}
 		}
 		else if (strcmp(name, "priority") == 0)
-			options->priority = repmgr_atoi(value, "priority", &config_errors, true);
+			options->priority = repmgr_atoi(value, "priority", error_list, true);
 		else if (strcmp(name, "node_name") == 0)
 			strncpy(options->node_name, value, MAXLEN);
 		else if (strcmp(name, "promote_command") == 0)
 			strncpy(options->promote_command, value, MAXLEN);
 		else if (strcmp(name, "follow_command") == 0)
 			strncpy(options->follow_command, value, MAXLEN);
+		else if (strcmp(name, "service_stop_command") == 0)
+			strncpy(options->service_stop_command, value, MAXLEN);
+		else if (strcmp(name, "service_start_command") == 0)
+			strncpy(options->service_start_command, value, MAXLEN);
+		else if (strcmp(name, "service_restart_command") == 0)
+			strncpy(options->service_restart_command, value, MAXLEN);
+		else if (strcmp(name, "service_reload_command") == 0)
+			strncpy(options->service_reload_command, value, MAXLEN);
+		else if (strcmp(name, "service_promote_command") == 0)
+			strncpy(options->service_promote_command, value, MAXLEN);
 		else if (strcmp(name, "master_response_timeout") == 0)
-			options->master_response_timeout = repmgr_atoi(value, "master_response_timeout", &config_errors, false);
+			options->master_response_timeout = repmgr_atoi(value, "master_response_timeout", error_list, false);
-		/* 'primary_response_timeout' as synonym for 'master_response_timeout' -
+		/*
+		 * 'primary_response_timeout' as synonym for 'master_response_timeout' -
 		 * we'll switch terminology in a future release (3.1?)
 		 */
 		else if (strcmp(name, "primary_response_timeout") == 0)
-			options->master_response_timeout = repmgr_atoi(value, "primary_response_timeout", &config_errors, false);
+			options->master_response_timeout = repmgr_atoi(value, "primary_response_timeout", error_list, false);
 		else if (strcmp(name, "reconnect_attempts") == 0)
-			options->reconnect_attempts = repmgr_atoi(value, "reconnect_attempts", &config_errors, false);
+			options->reconnect_attempts = repmgr_atoi(value, "reconnect_attempts", error_list, false);
 		else if (strcmp(name, "reconnect_interval") == 0)
-			options->reconnect_interval = repmgr_atoi(value, "reconnect_interval", &config_errors, false);
+			options->reconnect_interval = repmgr_atoi(value, "reconnect_interval", error_list, false);
 		else if (strcmp(name, "pg_bindir") == 0)
 			strncpy(options->pg_bindir, value, MAXLEN);
 		else if (strcmp(name, "pg_ctl_options") == 0)
@@ -358,20 +398,22 @@ parse_config(t_configuration_options *options)
 		else if (strcmp(name, "logfile") == 0)
 			strncpy(options->logfile, value, MAXLEN);
 		else if (strcmp(name, "monitor_interval_secs") == 0)
-			options->monitor_interval_secs = repmgr_atoi(value, "monitor_interval_secs", &config_errors, false);
+			options->monitor_interval_secs = repmgr_atoi(value, "monitor_interval_secs", error_list, false);
 		else if (strcmp(name, "retry_promote_interval_secs") == 0)
-			options->retry_promote_interval_secs = repmgr_atoi(value, "retry_promote_interval_secs", &config_errors, false);
+			options->retry_promote_interval_secs = repmgr_atoi(value, "retry_promote_interval_secs", error_list, false);
 		else if (strcmp(name, "witness_repl_nodes_sync_interval_secs") == 0)
-			options->witness_repl_nodes_sync_interval_secs = repmgr_atoi(value, "witness_repl_nodes_sync_interval_secs", &config_errors, false);
+			options->witness_repl_nodes_sync_interval_secs = repmgr_atoi(value, "witness_repl_nodes_sync_interval_secs", error_list, false);
 		else if (strcmp(name, "use_replication_slots") == 0)
 			/* XXX we should have a dedicated boolean argument format */
-			options->use_replication_slots = repmgr_atoi(value, "use_replication_slots", &config_errors, false);
+			options->use_replication_slots = repmgr_atoi(value, "use_replication_slots", error_list, false);
 		else if (strcmp(name, "event_notification_command") == 0)
 			strncpy(options->event_notification_command, value, MAXLEN);
 		else if (strcmp(name, "event_notifications") == 0)
 			parse_event_notifications_list(options, value);
 		else if (strcmp(name, "tablespace_mapping") == 0)
 			tablespace_list_append(options, value);
+		else if (strcmp(name, "restore_command") == 0)
+			strncpy(options->restore_command, value, MAXLEN);
 		else
 		{
 			known_parameter = false;
@@ -391,7 +433,7 @@ parse_config(t_configuration_options *options)
 					 _("no value provided for parameter \"%s\""),
 					 name);
 
-			error_list_append(&config_errors, error_message_buf);
+			item_list_append(error_list, error_message_buf);
 		}
 	}
 
@@ -400,11 +442,15 @@ parse_config(t_configuration_options *options)
 
 	if (node_found == false)
 	{
-		error_list_append(&config_errors, _("\"node\": parameter was not found"));
+		item_list_append(error_list, _("\"node\": parameter was not found"));
 	}
 	else if (options->node == 0)
 	{
-		error_list_append(&config_errors, _("\"node\": must be greater than zero"));
+		item_list_append(error_list, _("\"node\": must be greater than zero"));
+	}
+	else if (options->node < 0)
+	{
+		item_list_append(error_list, _("\"node\": must be a positive signed 32 bit integer, i.e. 2147483647 or less"));
 	}
 
 	if (strlen(options->conninfo))
@@ -424,18 +470,11 @@ parse_config(t_configuration_options *options)
 					 _("\"conninfo\": %s"),
 					 conninfo_errmsg);
 
-			error_list_append(&config_errors, error_message_buf);
+			item_list_append(error_list, error_message_buf);
 		}
 
 		PQconninfoFree(conninfo_options);
 	}
-
-	if (config_errors.head != NULL)
-	{
-		exit_with_errors(&config_errors);
-	}
-
-	return true;
 }
 
 
@@ -525,70 +564,85 @@ parse_line(char *buf, char *name, char *value)
 	trim(value);
 }
 
+
+/*
+ * reload_config()
+ *
+ * This is only called by repmgrd after receiving a SIGHUP or when a monitoring
+ * loop is started up; it therefore only needs to reload options required
+ * by repmgrd, which are as follows:
+ *
+ * changeable options:
+ * - failover
+ * - follow_command
+ * - logfacility
+ * - logfile
+ * - loglevel
+ * - master_response_timeout
+ * - monitor_interval_secs
+ * - priority
+ * - promote_command
+ * - reconnect_attempts
+ * - reconnect_interval
+ * - retry_promote_interval_secs
+ * - witness_repl_nodes_sync_interval_secs
+ *
+ * non-changeable options:
+ * - cluster_name
+ * - conninfo
+ * - node
+ * - node_name
+ *
+ * extract with something like:
+ *	 grep local_options\\. repmgrd.c | perl -n -e '/local_options\.([\w_]+)/ && print qq|$1\n|;' | sort | uniq
+
+ */
 bool
 reload_config(t_configuration_options *orig_options)
 {
 	PGconn	   *conn;
-	t_configuration_options new_options;
+	t_configuration_options new_options = T_CONFIGURATION_OPTIONS_INITIALIZER;
 	bool	  config_changed = false;
+	bool	  log_config_changed = false;
+
+	static ItemList config_errors = { NULL, NULL };
 
 	/*
 	 * Re-read the configuration file: repmgr.conf
 	 */
-	log_info(_("reloading configuration file and updating repmgr tables\n"));
+	log_info(_("reloading configuration file\n"));
 
-	parse_config(&new_options);
+	_parse_config(&new_options, &config_errors);
-	if (new_options.node == -1)
+
+	if (config_errors.head != NULL)
 	{
+		/* XXX dump errors to log */
 		log_warning(_("unable to parse new configuration, retaining current configuration\n"));
 		return false;
 	}
 
+	/* The following options cannot be changed */
 	if (strcmp(new_options.cluster_name, orig_options->cluster_name) != 0)
 	{
-		log_warning(_("unable to change cluster name, retaining current configuration\n"));
+		log_warning(_("cluster_name cannot be changed, retaining current configuration\n"));
 		return false;
 	}
 
 	if (new_options.node != orig_options->node)
 	{
-		log_warning(_("unable to change node ID, retaining current configuration\n"));
+		log_warning(_("node ID cannot be changed, retaining current configuration\n"));
 		return false;
 	}
 
 	if (strcmp(new_options.node_name, orig_options->node_name) != 0)
 	{
-		log_warning(_("unable to change standby name, keeping current configuration\n"));
+		log_warning(_("node_name cannot be changed, keeping current configuration\n"));
-		return false;
-	}
-
-	if (new_options.failover != MANUAL_FAILOVER && new_options.failover != AUTOMATIC_FAILOVER)
-	{
-		log_warning(_("new value for 'failover' must be 'automatic' or 'manual'\n"));
-		return false;
-	}
-
-	if (new_options.master_response_timeout <= 0)
-	{
-		log_warning(_("new value for 'master_response_timeout' must be greater than zero\n"));
-		return false;
-	}
-
-	if (new_options.reconnect_attempts < 0)
-	{
-		log_warning(_("new value for 'reconnect_attempts' must be zero or greater\n"));
-		return false;
-	}
-
-	if (new_options.reconnect_interval < 0)
-	{
-		log_warning(_("new value for 'reconnect_interval' must be zero or greater\n"));
 		return false;
 	}
 
 	if (strcmp(orig_options->conninfo, new_options.conninfo) != 0)
 	{
-		/* Test conninfo string */
+		/* Test conninfo string works*/
 		conn = establish_db_connection(new_options.conninfo, false);
 		if (!conn || (PQstatus(conn) != CONNECTION_OK))
 		{
@@ -605,27 +659,6 @@ reload_config(t_configuration_options *orig_options)
 	 * to manage them
 	 */
 
-	/* cluster_name */
-	if (strcmp(orig_options->cluster_name, new_options.cluster_name) != 0)
-	{
-		strcpy(orig_options->cluster_name, new_options.cluster_name);
-		config_changed = true;
-	}
-
-	/* conninfo */
-	if (strcmp(orig_options->conninfo, new_options.conninfo) != 0)
-	{
-		strcpy(orig_options->conninfo, new_options.conninfo);
-		config_changed = true;
-	}
-
-	/* node */
-	if (orig_options->node != new_options.node)
-	{
-		orig_options->node = new_options.node;
-		config_changed = true;
-	}
-
 	/* failover */
 	if (orig_options->failover != new_options.failover)
 	{
@@ -633,27 +666,6 @@ reload_config(t_configuration_options *orig_options)
 		config_changed = true;
 	}
 
-	/* priority */
-	if (orig_options->priority != new_options.priority)
-	{
-		orig_options->priority = new_options.priority;
-		config_changed = true;
-	}
-
-	/* node_name */
-	if (strcmp(orig_options->node_name, new_options.node_name) != 0)
-	{
-		strcpy(orig_options->node_name, new_options.node_name);
-		config_changed = true;
-	}
-
-	/* promote_command */
-	if (strcmp(orig_options->promote_command, new_options.promote_command) != 0)
-	{
-		strcpy(orig_options->promote_command, new_options.promote_command);
-		config_changed = true;
-	}
-
 	/* follow_command */
 	if (strcmp(orig_options->follow_command, new_options.follow_command) != 0)
 	{
@@ -661,30 +673,6 @@ reload_config(t_configuration_options *orig_options)
 		config_changed = true;
 	}
 
-	/*
-	 * XXX These ones can change with a simple SIGHUP?
-	 *
-	 * strcpy (orig_options->loglevel, new_options.loglevel); strcpy
-	 * (orig_options->logfacility, new_options.logfacility);
-	 *
-	 * logger_shutdown(); XXX do we have progname here ? logger_init(progname,
-	 * orig_options.loglevel, orig_options.logfacility);
-	 */
-
-	/* rsync_options */
-	if (strcmp(orig_options->rsync_options, new_options.rsync_options) != 0)
-	{
-		strcpy(orig_options->rsync_options, new_options.rsync_options);
-		config_changed = true;
-	}
-
-	/* ssh_options */
-	if (strcmp(orig_options->ssh_options, new_options.ssh_options) != 0)
-	{
-		strcpy(orig_options->ssh_options, new_options.ssh_options);
-		config_changed = true;
-	}
-
 	/* master_response_timeout */
 	if (orig_options->master_response_timeout != new_options.master_response_timeout)
 	{
@@ -692,6 +680,27 @@ reload_config(t_configuration_options *orig_options)
 		config_changed = true;
 	}
 
+	/* monitor_interval_secs */
+	if (orig_options->monitor_interval_secs != new_options.monitor_interval_secs)
+	{
+		orig_options->monitor_interval_secs = new_options.monitor_interval_secs;
+		config_changed = true;
+	}
+
+	/* priority */
+	if (orig_options->priority != new_options.priority)
+	{
+		orig_options->priority = new_options.priority;
+		config_changed = true;
+	}
+
+	/* promote_command */
+	if (strcmp(orig_options->promote_command, new_options.promote_command) != 0)
+	{
+		strcpy(orig_options->promote_command, new_options.promote_command);
+		config_changed = true;
+	}
+
 	/* reconnect_attempts */
 	if (orig_options->reconnect_attempts != new_options.reconnect_attempts)
 	{
@@ -706,27 +715,6 @@ reload_config(t_configuration_options *orig_options)
 		config_changed = true;
 	}
 
-	/* pg_ctl_options */
-	if (strcmp(orig_options->pg_ctl_options, new_options.pg_ctl_options) != 0)
-	{
-		strcpy(orig_options->pg_ctl_options, new_options.pg_ctl_options);
-		config_changed = true;
-	}
-
-	/* pg_basebackup_options */
-	if (strcmp(orig_options->pg_basebackup_options, new_options.pg_basebackup_options) != 0)
-	{
-		strcpy(orig_options->pg_basebackup_options, new_options.pg_basebackup_options);
-		config_changed = true;
-	}
-
-	/* monitor_interval_secs */
-	if (orig_options->monitor_interval_secs != new_options.monitor_interval_secs)
-	{
-		orig_options->monitor_interval_secs = new_options.monitor_interval_secs;
-		config_changed = true;
-	}
-
 	/* retry_promote_interval_secs */
 	if (orig_options->retry_promote_interval_secs != new_options.retry_promote_interval_secs)
 	{
@@ -734,20 +722,54 @@ reload_config(t_configuration_options *orig_options)
 		config_changed = true;
 	}
 
-	/* use_replication_slots */
+
-	if (orig_options->use_replication_slots != new_options.use_replication_slots)
+	/* witness_repl_nodes_sync_interval_secs */
+	if (orig_options->witness_repl_nodes_sync_interval_secs != new_options.witness_repl_nodes_sync_interval_secs)
 	{
-		orig_options->use_replication_slots = new_options.use_replication_slots;
+		orig_options->witness_repl_nodes_sync_interval_secs = new_options.witness_repl_nodes_sync_interval_secs;
 		config_changed = true;
 	}
 
+	/*
+	 * Handle changes to logging configuration
+	 */
+	if (strcmp(orig_options->logfacility, new_options.logfacility) != 0)
+	{
+		strcpy(orig_options->logfacility, new_options.logfacility);
+		log_config_changed = true;
+	}
+
+	if (strcmp(orig_options->logfile, new_options.logfile) != 0)
+	{
+		strcpy(orig_options->logfile, new_options.logfile);
+		log_config_changed = true;
+	}
+
+
+	if (strcmp(orig_options->loglevel, new_options.loglevel) != 0)
+	{
+		strcpy(orig_options->loglevel, new_options.loglevel);
+		log_config_changed = true;
+	}
+
+	if (log_config_changed == true)
+	{
+		log_notice(_("restarting logging with changed parameters\n"));
+		logger_shutdown();
+		logger_init(orig_options, progname());
+	}
+
 	if (config_changed == true)
 	{
-		log_debug(_("reload_config(): configuration has changed\n"));
+		log_notice(_("configuration file reloaded with changed parameters\n"));
 	}
-	else
+	/*
+	 * if logging configuration changed, don't say the configuration didn't
+	 * change, as it clearly has.
+	 */
+	else if (log_config_changed == false)
 	{
-		log_debug(_("reload_config(): configuration has not changed\n"));
+		log_info(_("configuration has not changed\n"));
 	}
 
 	return config_changed;
@@ -755,11 +777,11 @@ reload_config(t_configuration_options *orig_options)
 
 
 void
-error_list_append(ErrorList *error_list, char *error_message)
+item_list_append(ItemList *item_list, char *error_message)
 {
-	ErrorListCell *cell;
+	ItemListCell *cell;
 
-	cell = (ErrorListCell *) pg_malloc0(sizeof(ErrorListCell));
+	cell = (ItemListCell *) pg_malloc0(sizeof(ItemListCell));
 
 	if (cell == NULL)
 	{
@@ -767,19 +789,19 @@ error_list_append(ErrorList *error_list, char *error_message)
 		exit(ERR_BAD_CONFIG);
 	}
 
-	cell->error_message = pg_malloc0(MAXLEN);
+	cell->string = pg_malloc0(MAXLEN);
-	strncpy(cell->error_message, error_message, MAXLEN);
+	strncpy(cell->string, error_message, MAXLEN);
 
-	if (error_list->tail)
+	if (item_list->tail)
 	{
-		error_list->tail->next = cell;
+		item_list->tail->next = cell;
 	}
 	else
 	{
-		error_list->head = cell;
+		item_list->head = cell;
 	}
 
-	error_list->tail = cell;
+	item_list->tail = cell;
 }
 
 
@@ -789,7 +811,7 @@ error_list_append(ErrorList *error_list, char *error_message)
  * otherwise exit
  */
 int
-repmgr_atoi(const char *value, const char *config_item, ErrorList *error_list, bool allow_negative)
+repmgr_atoi(const char *value, const char *config_item, ItemList *error_list, bool allow_negative)
 {
 	char	  *endptr;
 	long	   longval = 0;
@@ -838,7 +860,7 @@ repmgr_atoi(const char *value, const char *config_item, ErrorList *error_list, b
 			exit(ERR_BAD_CONFIG);
 		}
 
-		error_list_append(error_list, error_message_buf);
+		item_list_append(error_list, error_message_buf);
 	}
 
 	return (int32) longval;
@@ -921,7 +943,7 @@ static void
 parse_event_notifications_list(t_configuration_options *options, const char *arg)
 {
 	const char *arg_ptr;
-	char	    event_type_buf[MAXLEN] = "";
+	char		event_type_buf[MAXLEN] = "";
 	char	   *dst_ptr = event_type_buf;
 
 
@@ -980,15 +1002,15 @@ parse_event_notifications_list(t_configuration_options *options, const char *arg
 
 
 static void
-exit_with_errors(ErrorList *config_errors)
+exit_with_errors(ItemList *config_errors)
 {
-	ErrorListCell *cell;
+	ItemListCell *cell;
 
 	log_err(_("%s: following errors were found in the configuration file.\n"), progname());
 
 	for (cell = config_errors->head; cell; cell = cell->next)
 	{
-		log_err("%s\n", cell->error_message);
+		log_err("%s\n", cell->string);
 	}
 
 	exit(ERR_BAD_CONFIG);

config.h

@@ -1,6 +1,7 @@
 /*
  * config.h
- * Copyright (c) 2ndQuadrant, 2010-2016
+ *
+ * Copyright (c) 2ndQuadrant, 2010-2017
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -57,11 +58,20 @@ typedef struct
 	int			node;
 	int         upstream_node;
 	char		conninfo[MAXLEN];
+	char		barman_server[MAXLEN];
+	char		barman_config[MAXLEN];
 	int			failover;
 	int			priority;
 	char		node_name[MAXLEN];
+	/* commands executed by repmgrd */
 	char		promote_command[MAXLEN];
 	char		follow_command[MAXLEN];
+	/* Overrides for pg_ctl commands */
+	char		service_stop_command[MAXLEN];
+	char		service_start_command[MAXLEN];
+	char		service_restart_command[MAXLEN];
+	char		service_reload_command[MAXLEN];
+	char		service_promote_command[MAXLEN];
 	char		loglevel[MAXLEN];
 	char		logfacility[MAXLEN];
 	char		rsync_options[QUERY_STR_LEN];
@@ -72,6 +82,7 @@ typedef struct
 	char		pg_bindir[MAXLEN];
 	char		pg_ctl_options[MAXLEN];
 	char		pg_basebackup_options[MAXLEN];
+	char		restore_command[MAXLEN];
 	char		logfile[MAXLEN];
 	int			monitor_interval_secs;
 	int			retry_promote_interval_secs;
@@ -82,32 +93,55 @@ typedef struct
 	TablespaceList tablespace_mapping;
 }	t_configuration_options;
 
-#define T_CONFIGURATION_OPTIONS_INITIALIZER { "", -1, NO_UPSTREAM_NODE, "", MANUAL_FAILOVER, -1, "", "", "", "", "", "", "", -1, -1, -1, "", "", "", "", 0, 0, 0, 0, "", { NULL, NULL }, {NULL, NULL} }
+/*
+ * The following will initialize the structure with a minimal set of options;
+ * actual defaults are set in parse_config() before parsing the configuration file
+ */
+#define T_CONFIGURATION_OPTIONS_INITIALIZER { "", UNKNOWN_NODE_ID, NO_UPSTREAM_NODE, "", "", "", MANUAL_FAILOVER, -1, "", "", "", "", "", "", "", "", "", "", "", "", -1, -1, -1, "", "", "", "", "", 0, 0, 0, 0, "", { NULL, NULL }, { NULL, NULL } }
 
-typedef struct ErrorListCell
+typedef struct ItemListCell
 {
-	struct ErrorListCell *next;
+	struct ItemListCell *next;
-	char			     *error_message;
+	char			    *string;
-} ErrorListCell;
+} ItemListCell;
 
-typedef struct ErrorList
+typedef struct ItemList
 {
-	ErrorListCell *head;
+	ItemListCell *head;
-	ErrorListCell *tail;
+	ItemListCell *tail;
-} ErrorList;
+} ItemList;
+
+typedef struct TablespaceDataListCell
+{
+	struct TablespaceDataListCell *next;
+	char	   *name;
+	char	   *oid;
+	char	   *location;
+	/* optional payload */
+	FILE       *f;
+} TablespaceDataListCell;
+
+typedef struct TablespaceDataList
+{
+	TablespaceDataListCell *head;
+	TablespaceDataListCell *tail;
+} TablespaceDataList;
 
 void set_progname(const char *argv0);
 const char * progname(void);
 
 bool		load_config(const char *config_file, bool verbose, t_configuration_options *options, char *argv0);
-bool		reload_config(t_configuration_options *orig_options);
+
+void		_parse_config(t_configuration_options *options, ItemList *error_list);
 bool		parse_config(t_configuration_options *options);
+bool		reload_config(t_configuration_options *orig_options);
+
 void		parse_line(char *buff, char *name, char *value);
 char	   *trim(char *s);
-void		error_list_append(ErrorList *error_list, char *error_message);
+void		item_list_append(ItemList *item_list, char *error_message);
 int			repmgr_atoi(const char *s,
 						const char *config_item,
-						ErrorList *error_list,
+						ItemList *error_list,
 						bool allow_negative);
-
+extern bool		config_file_found;
 #endif

dbutils.c

@@ -1,6 +1,7 @@
 /*
  * dbutils.c - Database connection/management functions
- * Copyright (C) 2ndQuadrant, 2010-2016
+ *
+ * Copyright (c) 2ndQuadrant, 2010-2017
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -31,9 +32,19 @@
 char repmgr_schema[MAXLEN] = "";
 char repmgr_schema_quoted[MAXLEN] = "";
 
+static int _get_node_record(PGconn *conn, char *cluster, char *sqlquery, t_node_info *node_info);
+static bool _set_config(PGconn *conn, const char *config_param, const char *sqlquery);
+
+/*
+ * _establish_db_connection()
+ *
+ * Connect to a database using a conninfo string.
+ *
+ * NOTE: *do not* use this for replication connections; use establish_db_connection_by_params() instead.
+ */
 
 PGconn *
-_establish_db_connection(const char *conninfo, const bool exit_on_error, const bool log_notice)
+_establish_db_connection(const char *conninfo, const bool exit_on_error, const bool log_notice, const bool verbose_only)
 {
 	/* Make a connection to the database */
 	PGconn	   *conn = NULL;
@@ -49,15 +60,23 @@ _establish_db_connection(const char *conninfo, const bool exit_on_error, const b
 	/* Check to see that the backend connection was successfully made */
 	if ((PQstatus(conn) != CONNECTION_OK))
 	{
-		if (log_notice)
+		bool emit_log = true;
+
+		if (verbose_only == true && verbose_logging == false)
+			emit_log = false;
+
+		if (emit_log)
 		{
-			log_notice(_("connection to database failed: %s\n"),
+			if (log_notice)
-					PQerrorMessage(conn));
+			{
-		}
+				log_notice(_("connection to database failed: %s\n"),
-		else
+						   PQerrorMessage(conn));
-		{
+			}
-			log_err(_("connection to database failed: %s\n"),
+			else
-					PQerrorMessage(conn));
+			{
+				log_err(_("connection to database failed: %s\n"),
+						PQerrorMessage(conn));
+			}
 		}
 
 		if (exit_on_error)
@@ -67,19 +86,51 @@ _establish_db_connection(const char *conninfo, const bool exit_on_error, const b
 		}
 	}
 
+	/*
+	 * set "synchronous_commit" to "local" in case synchronous replication is in use
+	 */
+
+	else if (set_config(conn, "synchronous_commit", "local") == false)
+	{
+		if (exit_on_error)
+		{
+			PQfinish(conn);
+			exit(ERR_DB_CON);
+		}
+	}
+
 	return conn;
 }
 
+
+/*
+ * Establish a database connection, optionally exit on error
+ */
 PGconn *
 establish_db_connection(const char *conninfo, const bool exit_on_error)
 {
-	return _establish_db_connection(conninfo, exit_on_error, false);
+	return _establish_db_connection(conninfo, exit_on_error, false, false);
 }
 
+/*
+ * Attempt to establish a database connection, never exit on error, only
+ * output error messages if --verbose option used
+ */
 PGconn *
-test_db_connection(const char *conninfo, const bool exit_on_error)
+establish_db_connection_quiet(const char *conninfo)
 {
-	return _establish_db_connection(conninfo, exit_on_error, true);
+	return _establish_db_connection(conninfo, false, false, true);
+}
+
+/*
+ * Attempt to establish a database connection, never exit on error,
+ * output connection error messages as NOTICE (useful when connection
+ * failure is expected)
+ */
+PGconn *
+test_db_connection(const char *conninfo)
+{
+	return _establish_db_connection(conninfo, false, true, false);
 }
 
 
@@ -87,8 +138,12 @@ PGconn *
 establish_db_connection_by_params(const char *keywords[], const char *values[],
 								  const bool exit_on_error)
 {
-	/* Make a connection to the database */
+	PGconn	   *conn;
-	PGconn	   *conn = PQconnectdbParams(keywords, values, true);
+	bool	    replication_connection = false;
+	int	   	    i;
+
+	/* Connect to the database using the provided parameters */
+	conn = PQconnectdbParams(keywords, values, true);
 
 	/* Check to see that the backend connection was successfully made */
 	if ((PQstatus(conn) != CONNECTION_OK))
@@ -101,6 +156,28 @@ establish_db_connection_by_params(const char *keywords[], const char *values[],
 			exit(ERR_DB_CON);
 		}
 	}
+	else
+	{
+		/*
+		 * set "synchronous_commit" to "local" in case synchronous replication is in
+		 * use (provided this is not a replication connection)
+		 */
+
+		for (i = 0; keywords[i]; i++)
+		{
+			if (strcmp(keywords[i], "replication") == 0)
+				replication_connection = true;
+		}
+
+		if (replication_connection == false && set_config(conn, "synchronous_commit", "local") == false)
+		{
+			if (exit_on_error)
+			{
+				PQfinish(conn);
+				exit(ERR_DB_CON);
+			}
+		}
+	}
 
 	return conn;
 }
@@ -185,7 +262,7 @@ check_cluster_schema(PGconn *conn)
 	char		sqlquery[QUERY_STR_LEN];
 
 	sqlquery_snprintf(sqlquery,
-					  "SELECT 1 FROM pg_namespace WHERE nspname = '%s'",
+					  "SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname = '%s'",
 					  get_repmgr_schema());
 
 	log_verbose(LOG_DEBUG, "check_cluster_schema(): %s\n", sqlquery);
@@ -250,7 +327,6 @@ is_pgup(PGconn *conn, int timeout)
 	/* Check the connection status twice in case it changes after reset */
 	bool		twice = false;
 
-	/* Check the connection status twice in case it changes after reset */
 	for (;;)
 	{
 		if (PQstatus(conn) != CONNECTION_OK)
@@ -380,7 +456,7 @@ guc_set(PGconn *conn, const char *parameter, const char *op,
 	int			retval = 1;
 
 	sqlquery_snprintf(sqlquery,
-					  "SELECT true FROM pg_settings "
+					  "SELECT true FROM pg_catalog.pg_settings "
 					  " WHERE name = '%s' AND setting %s '%s'",
 					  parameter, op, value);
 
@@ -416,11 +492,11 @@ guc_set_typed(PGconn *conn, const char *parameter, const char *op,
 	int			retval = 1;
 
 	sqlquery_snprintf(sqlquery,
-					  "SELECT true FROM pg_settings "
+					  "SELECT true FROM pg_catalog.pg_settings "
 					  " WHERE name = '%s' AND setting::%s %s '%s'::%s",
 					  parameter, datatype, op, value, datatype);
 
-	log_verbose(LOG_DEBUG, "guc_set_typed():n%s\n", sqlquery);
+	log_verbose(LOG_DEBUG, "guc_set_typed():\n%s\n", sqlquery);
 
 	res = PQexec(conn, sqlquery);
 	if (PQresultStatus(res) != PGRES_TUPLES_OK)
@@ -448,7 +524,7 @@ get_cluster_size(PGconn *conn, char *size)
 
 	sqlquery_snprintf(sqlquery,
 					  "SELECT pg_catalog.pg_size_pretty(SUM(pg_catalog.pg_database_size(oid))::bigint) "
-					  "	 FROM pg_database ");
+					  "	 FROM pg_catalog.pg_database ");
 
 	log_verbose(LOG_DEBUG, "get_cluster_size():\n%s\n", sqlquery);
 
@@ -475,11 +551,11 @@ get_pg_setting(PGconn *conn, const char *setting, char *output)
 	char		sqlquery[QUERY_STR_LEN];
 	PGresult   *res;
 	int			i;
-	bool        success = true;
+	bool        success = false;
 
 	sqlquery_snprintf(sqlquery,
 					  "SELECT name, setting "
-					  " FROM pg_settings WHERE name = '%s'",
+					  "  FROM pg_catalog.pg_settings WHERE name = '%s'",
 					  setting);
 
 	log_verbose(LOG_DEBUG, "get_pg_setting(): %s\n", sqlquery);
@@ -538,7 +614,7 @@ get_conninfo_value(const char *conninfo, const char *keyword, char *output)
 
 	conninfo_options = PQconninfoParse(conninfo, NULL);
 
-	if (conninfo_options == false)
+	if (conninfo_options == NULL)
 	{
 		log_err(_("Unable to parse provided conninfo string \"%s\""), conninfo);
 		return false;
@@ -587,7 +663,7 @@ get_upstream_connection(PGconn *standby_conn, char *cluster, int node_id,
 		upstream_conninfo = upstream_conninfo_out;
 
 	sqlquery_snprintf(sqlquery,
-					  "    SELECT un.conninfo, un.name, un.id "
+					  "    SELECT un.conninfo, un.id "
 					  "      FROM %s.repl_nodes un "
 					  "INNER JOIN %s.repl_nodes n "
 					  "        ON (un.id = n.upstream_node_id AND un.cluster = n.cluster)"
@@ -604,7 +680,7 @@ get_upstream_connection(PGconn *standby_conn, char *cluster, int node_id,
 
 	if (PQresultStatus(res) != PGRES_TUPLES_OK)
 	{
-		log_err(_("unable to get conninfo for upstream server\n%s\n"),
+		log_err(_("error when attempting to find upstream server\n%s\n"),
 				PQerrorMessage(standby_conn));
 		PQclear(res);
 		return NULL;
@@ -612,9 +688,36 @@ get_upstream_connection(PGconn *standby_conn, char *cluster, int node_id,
 
 	if (!PQntuples(res))
 	{
-		log_notice(_("no record found for upstream server"));
 		PQclear(res);
-		return NULL;
+		log_debug("no record found for upstream server\n");
+
+		sqlquery_snprintf(sqlquery,
+						  "    SELECT un.conninfo, un.id "
+						  "      FROM %s.repl_nodes un "
+						  "     WHERE un.cluster = '%s' "
+						  "       AND un.type='master' "
+						  "       AND un.active IS TRUE",
+						  get_repmgr_schema_quoted(standby_conn),
+						  cluster);
+		res = PQexec(standby_conn, sqlquery);
+
+		if (PQresultStatus(res) != PGRES_TUPLES_OK)
+		{
+			log_err(_("error when attempting to find active master server\n%s\n"),
+					PQerrorMessage(standby_conn));
+			PQclear(res);
+			return NULL;
+		}
+
+		if (!PQntuples(res))
+		{
+			PQclear(res);
+			log_notice(_("no record found for active master server\n"));
+
+			return NULL;
+		}
+
+		log_debug("record found for active master server\n");
 	}
 
 	strncpy(upstream_conninfo, PQgetvalue(res, 0, 0), MAXCONNINFO);
@@ -889,7 +992,7 @@ get_repmgr_schema_quoted(PGconn *conn)
 
 
 bool
-create_replication_slot(PGconn *conn, char *slot_name, int server_version_num)
+create_replication_slot(PGconn *conn, char *slot_name, int server_version_num, PQExpBufferData *error_msg)
 {
 	char				sqlquery[QUERY_STR_LEN];
 	int					query_res;
@@ -908,8 +1011,9 @@ create_replication_slot(PGconn *conn, char *slot_name, int server_version_num)
 	{
 		if (strcmp(slot_info.slot_type, "physical") != 0)
 		{
-			log_err(_("Slot '%s' exists and is not a physical slot\n"),
+			appendPQExpBuffer(error_msg,
-					slot_name);
+							  _("Slot '%s' exists and is not a physical slot\n"),
+							  slot_name);
 			return false;
 		}
 
@@ -921,8 +1025,9 @@ create_replication_slot(PGconn *conn, char *slot_name, int server_version_num)
 			return true;
 		}
 
-		log_err(_("Slot '%s' already exists as an active slot\n"),
+		appendPQExpBuffer(error_msg,
-				slot_name);
+						  _("Slot '%s' already exists as an active slot\n"),
+						  slot_name);
 		return false;
 	}
 
@@ -930,25 +1035,26 @@ create_replication_slot(PGconn *conn, char *slot_name, int server_version_num)
 	if (server_version_num >= 90600)
 	{
 		sqlquery_snprintf(sqlquery,
-						  "SELECT * FROM pg_create_physical_replication_slot('%s', TRUE)",
+						  "SELECT * FROM pg_catalog.pg_create_physical_replication_slot('%s', TRUE)",
 						  slot_name);
 	}
 	else
 	{
 		sqlquery_snprintf(sqlquery,
-						  "SELECT * FROM pg_create_physical_replication_slot('%s')",
+						  "SELECT * FROM pg_catalog.pg_create_physical_replication_slot('%s')",
 						  slot_name);
 	}
 
-	log_debug(_("create_replication_slot(): Creating slot '%s' on primary\n"), slot_name);
+	log_debug(_("create_replication_slot(): Creating slot '%s' on master\n"), slot_name);
 	log_verbose(LOG_DEBUG, "create_replication_slot():\n%s\n", sqlquery);
 
 	res = PQexec(conn, sqlquery);
 	if (!res || PQresultStatus(res) != PGRES_TUPLES_OK)
 	{
-		log_err(_("unable to create slot '%s' on the primary node: %s\n"),
+		appendPQExpBuffer(error_msg,
-				slot_name,
+						  _("unable to create slot '%s' on the master node: %s\n"),
-				PQerrorMessage(conn));
+						  slot_name,
+						  PQerrorMessage(conn));
 		PQclear(res);
 		return false;
 	}
@@ -966,7 +1072,7 @@ get_slot_record(PGconn *conn, char *slot_name, t_replication_slot *record)
 
 	sqlquery_snprintf(sqlquery,
 					  "SELECT slot_name, slot_type, active "
-                      "  FROM pg_replication_slots "
+                      "  FROM pg_catalog.pg_replication_slots "
 					  " WHERE slot_name = '%s' ",
 					  slot_name);
 
@@ -1092,19 +1198,12 @@ stop_backup(PGconn *conn, char *last_wal_segment)
 }
 
 
+
 bool
-set_config_bool(PGconn *conn, const char *config_param, bool state)
+_set_config(PGconn *conn, const char *config_param, const char *sqlquery)
 {
-	char		sqlquery[QUERY_STR_LEN];
 	PGresult   *res;
 
-	sqlquery_snprintf(sqlquery,
-					  "SET %s TO %s",
-					  config_param,
-					  state ? "TRUE" : "FALSE");
-
-	log_verbose(LOG_DEBUG, "set_config_bool():\n%s\n", sqlquery);
-
 	res = PQexec(conn, sqlquery);
 
 	if (PQresultStatus(res) != PGRES_COMMAND_OK)
@@ -1119,6 +1218,36 @@ set_config_bool(PGconn *conn, const char *config_param, bool state)
 	return true;
 }
 
+bool
+set_config(PGconn *conn, const char *config_param,  const char *config_value)
+{
+	char		sqlquery[QUERY_STR_LEN];
+
+	sqlquery_snprintf(sqlquery,
+					  "SET %s TO '%s'",
+					  config_param,
+					  config_value);
+
+	log_verbose(LOG_DEBUG, "set_config():\n%s\n", sqlquery);
+
+	return _set_config(conn, config_param, sqlquery);
+}
+
+bool
+set_config_bool(PGconn *conn, const char *config_param, bool state)
+{
+	char		sqlquery[QUERY_STR_LEN];
+
+	sqlquery_snprintf(sqlquery,
+					  "SET %s TO %s",
+					  config_param,
+					  state ? "TRUE" : "FALSE");
+
+	log_verbose(LOG_DEBUG, "set_config_bool():\n%s\n", sqlquery);
+
+	return _set_config(conn, config_param, sqlquery);
+}
+
 
 /*
  * witness_copy_node_records()
@@ -1168,7 +1297,8 @@ witness_copy_node_records(PGconn *masterconn, PGconn *witnessconn, char *cluster
 
 	/* Get current records from primary */
 	sqlquery_snprintf(sqlquery,
-					  "SELECT id, type, upstream_node_id, name, conninfo, priority, slot_name, active FROM %s.repl_nodes",
+					  "SELECT id, type, upstream_node_id, name, conninfo, priority, slot_name, active "
+					  "  FROM %s.repl_nodes",
 					  get_repmgr_schema_quoted(masterconn));
 
 	log_verbose(LOG_DEBUG, "witness_copy_node_records():\n%s\n", sqlquery);
@@ -1282,7 +1412,8 @@ create_node_record(PGconn *conn, char *action, int node, char *type, int upstrea
 	sqlquery_snprintf(sqlquery,
 					  "INSERT INTO %s.repl_nodes "
 					  "       (id, type, upstream_node_id, cluster, "
-					  "        name, conninfo, slot_name, priority, active) "
+					  "        name, conninfo, slot_name, "
+					  "        priority, active) "
 					  "VALUES (%i, '%s', %s, '%s', '%s', '%s', %s, %i, %s) ",
 					  get_repmgr_schema_quoted(conn),
 					  node,
@@ -1376,10 +1507,11 @@ create_event_record(PGconn *conn, t_configuration_options *options, int node_id,
 	bool		success = true;
 	struct tm	ts;
 
-	/* Only attempt to write a record if a connection handle was provided.
+	/*
-	   Also check that the repmgr schema has been properly intialised - if
+	 * Only attempt to write a record if a connection handle was provided.
-	   not it means no configuration file was provided, which can happen with
+	 * Also check that the repmgr schema has been properly initialised - if
-	   e.g. `repmgr standby clone`, and we won't know which schema to write to.
+	 * not it means no configuration file was provided, which can happen with
+	 * e.g. `repmgr standby clone`, and we won't know which schema to write to.
 	 */
 	if (conn != NULL && strcmp(repmgr_schema, DEFAULT_REPMGR_SCHEMA_PREFIX) != 0)
 	{
@@ -1428,7 +1560,6 @@ create_event_record(PGconn *conn, t_configuration_options *options, int node_id,
 						PQerrorMessage(conn));
 
 			success = false;
-
 		}
 		else
 		{
@@ -1569,6 +1700,89 @@ create_event_record(PGconn *conn, t_configuration_options *options, int node_id,
 }
 
 
+bool
+update_node_record(PGconn *conn, char *action, int node, char *type, int upstream_node, char *cluster_name, char *node_name, char *conninfo, int priority, char *slot_name, bool active)
+{
+	char		sqlquery[QUERY_STR_LEN];
+	char		upstream_node_id[MAXLEN];
+	char		slot_name_buf[MAXLEN];
+	PGresult   *res;
+
+	/* XXX this segment copied from create_node_record() */
+	if (upstream_node == NO_UPSTREAM_NODE)
+	{
+		/*
+		 * No explicit upstream node id provided for standby - attempt to
+		 * get primary node id
+		 */
+		if (strcmp(type, "standby") == 0)
+		{
+			int primary_node_id = get_master_node_id(conn, cluster_name);
+			maxlen_snprintf(upstream_node_id, "%i", primary_node_id);
+		}
+		else
+		{
+			maxlen_snprintf(upstream_node_id, "%s", "NULL");
+		}
+	}
+	else
+	{
+		maxlen_snprintf(upstream_node_id, "%i", upstream_node);
+	}
+
+	if (slot_name != NULL && slot_name[0])
+	{
+		maxlen_snprintf(slot_name_buf, "'%s'", slot_name);
+	}
+	else
+	{
+		maxlen_snprintf(slot_name_buf, "%s", "NULL");
+	}
+
+	/* XXX convert to placeholder query */
+	sqlquery_snprintf(sqlquery,
+					  "UPDATE %s.repl_nodes SET "
+					  "       type = '%s', "
+					  "       upstream_node_id = %s, "
+					  "       cluster = '%s', "
+					  "       name = '%s', "
+					  "       conninfo = '%s', "
+					  "       slot_name = %s, "
+					  "       priority = %i, "
+					  "       active = %s "
+					  " WHERE id = %i ",
+					  get_repmgr_schema_quoted(conn),
+					  type,
+					  upstream_node_id,
+					  cluster_name,
+					  node_name,
+					  conninfo,
+					  slot_name_buf,
+					  priority,
+					  active == true ? "TRUE" : "FALSE",
+					  node);
+
+	log_verbose(LOG_DEBUG, "update_node_record(): %s\n", sqlquery);
+
+	if (action != NULL)
+	{
+		log_verbose(LOG_DEBUG, "update_node_record(): action is \"%s\"\n", action);
+	}
+
+	res = PQexec(conn, sqlquery);
+	if (!res || PQresultStatus(res) != PGRES_COMMAND_OK)
+	{
+		log_err(_("Unable to update node record\n%s\n"),
+				PQerrorMessage(conn));
+		PQclear(res);
+		return false;
+	}
+
+	PQclear(res);
+
+	return true;
+}
+
 /*
  * Update node record following change of status
  * (e.g. inactive primary converted to standby)
@@ -1654,12 +1868,12 @@ int
 get_node_record(PGconn *conn, char *cluster, int node_id, t_node_info *node_info)
 {
 	char		sqlquery[QUERY_STR_LEN];
-	PGresult   *res;
+	int		    result;
-	int         ntuples;
 
 	sqlquery_snprintf(
 		sqlquery,
-		"SELECT id, type, upstream_node_id, name, conninfo, slot_name, priority, active"
+		"SELECT id, type, upstream_node_id, name, conninfo, "
+		"       slot_name, priority, active"
 		"  FROM %s.repl_nodes "
 		" WHERE cluster = '%s' "
 		"   AND id = %i",
@@ -1669,6 +1883,51 @@ get_node_record(PGconn *conn, char *cluster, int node_id, t_node_info *node_info
 
 	log_verbose(LOG_DEBUG, "get_node_record():\n%s\n", sqlquery);
 
+	result = _get_node_record(conn, cluster, sqlquery, node_info);
+
+	if (result == 0)
+	{
+		log_verbose(LOG_DEBUG, "get_node_record(): no record found for node %i\n", node_id);
+	}
+
+	return result;
+}
+
+int
+get_node_record_by_name(PGconn *conn, char *cluster, const char *node_name, t_node_info *node_info)
+{
+	char		sqlquery[QUERY_STR_LEN];
+	int result;
+
+	sqlquery_snprintf(
+		sqlquery,
+		"SELECT id, type, upstream_node_id, name, conninfo, slot_name, priority, active"
+		"  FROM %s.repl_nodes "
+		" WHERE cluster = '%s' "
+		"   AND name = '%s'",
+		get_repmgr_schema_quoted(conn),
+		cluster,
+		node_name);
+
+	log_verbose(LOG_DEBUG, "get_node_record_by_name():\n%s\n", sqlquery);
+
+	result = _get_node_record(conn, cluster, sqlquery, node_info);
+
+	if (result == 0)
+	{
+		log_verbose(LOG_DEBUG, "get_node_record(): no record found for node %s\n", node_name);
+	}
+
+	return result;
+}
+
+
+static int
+_get_node_record(PGconn *conn, char *cluster, char *sqlquery, t_node_info *node_info)
+{
+	int         ntuples;
+	PGresult   *res;
+
 	res = PQexec(conn, sqlquery);
 	if (PQresultStatus(res) != PGRES_TUPLES_OK)
 	{
@@ -1679,13 +1938,21 @@ get_node_record(PGconn *conn, char *cluster, int node_id, t_node_info *node_info
 
 	if (ntuples == 0)
 	{
-		log_verbose(LOG_DEBUG, "get_node_record(): no record found for node %i\n", node_id);
 		return 0;
 	}
 
 	node_info->node_id = atoi(PQgetvalue(res, 0, 0));
 	node_info->type = parse_node_type(PQgetvalue(res, 0, 1));
-	node_info->upstream_node_id = atoi(PQgetvalue(res, 0, 2));
+
+	if (PQgetisnull(res, 0, 2))
+	{
+		node_info->upstream_node_id = NO_UPSTREAM_NODE;
+	}
+	else
+	{
+		node_info->upstream_node_id = atoi(PQgetvalue(res, 0, 2));
+	}
+
 	strncpy(node_info->name, PQgetvalue(res, 0, 3), MAXLEN);
 	strncpy(node_info->conninfo_str, PQgetvalue(res, 0, 4), MAXLEN);
 	strncpy(node_info->slot_name, PQgetvalue(res, 0, 5), MAXLEN);
@@ -1700,6 +1967,9 @@ get_node_record(PGconn *conn, char *cluster, int node_id, t_node_info *node_info
 }
 
 
+
+
+
 int
 get_node_replication_state(PGconn *conn, char *node_name, char *output)
 {

dbutils.h

@@ -1,6 +1,7 @@
 /*
  * dbutils.h
- * Copyright (c) 2ndQuadrant, 2010-2016
+ *
+ * Copyright (c) 2ndQuadrant, 2010-2017
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -21,6 +22,7 @@
 #define _REPMGR_DBUTILS_H_
 
 #include "access/xlogdefs.h"
+#include "pqexpbuffer.h"
 
 #include "config.h"
 #include "strutil.h"
@@ -52,18 +54,6 @@ typedef struct s_node_info
 }	t_node_info;
 
 
-/*
- * Struct to store replication slot information
- */
-
-typedef struct s_replication_slot
-{
-	char slot_name[MAXLEN];
-    char slot_type[MAXLEN];
-	bool active;
-}   t_replication_slot;
-
-
 #define T_NODE_INFO_INITIALIZER { \
   NODE_NOT_FOUND, \
   NO_UPSTREAM_NODE, \
@@ -78,13 +68,27 @@ typedef struct s_replication_slot
   InvalidXLogRecPtr \
 }
 
+/*
+ * Struct to store replication slot information
+ */
+
+typedef struct s_replication_slot
+{
+	char slot_name[MAXLEN];
+    char slot_type[MAXLEN];
+	bool active;
+}   t_replication_slot;
+
+extern char		repmgr_schema[MAXLEN];
+
 PGconn *_establish_db_connection(const char *conninfo,
 								 const bool exit_on_error,
-								 const bool log_notice);
+								 const bool log_notice,
+								 const bool verbose_only);
 PGconn *establish_db_connection(const char *conninfo,
 								const bool exit_on_error);
-PGconn *test_db_connection(const char *conninfo,
+PGconn *establish_db_connection_quiet(const char *conninfo);
-						   const bool exit_on_error);
+PGconn *test_db_connection(const char *conninfo);
 PGconn *establish_db_connection_by_params(const char *keywords[],
 								  const char *values[],
 								  const bool exit_on_error);
@@ -115,16 +119,19 @@ int			wait_connection_availability(PGconn *conn, long long timeout);
 bool		cancel_query(PGconn *conn, int timeout);
 char       *get_repmgr_schema(void);
 char       *get_repmgr_schema_quoted(PGconn *conn);
-bool		create_replication_slot(PGconn *conn, char *slot_name, int server_version_num);
+bool		create_replication_slot(PGconn *conn, char *slot_name, int server_version_num, PQExpBufferData *error_msg);
 int			get_slot_record(PGconn *conn, char *slot_name, t_replication_slot *record);
 bool		drop_replication_slot(PGconn *conn, char *slot_name);
 bool		start_backup(PGconn *conn, char *first_wal_segment, bool fast_checkpoint);
 bool		stop_backup(PGconn *conn, char *last_wal_segment);
+bool		set_config(PGconn *conn, const char *config_param,  const char *config_value);
 bool		set_config_bool(PGconn *conn, const char *config_param, bool state);
 bool		witness_copy_node_records(PGconn *masterconn, PGconn *witnessconn, char *cluster_name);
 bool		create_node_record(PGconn *conn, char *action, int node, char *type, int upstream_node, char *cluster_name, char *node_name, char *conninfo, int priority, char *slot_name, bool active);
 bool		delete_node_record(PGconn *conn, int node, char *action);
 int			get_node_record(PGconn *conn, char *cluster, int node_id, t_node_info *node_info);
+int			get_node_record_by_name(PGconn *conn, char *cluster, const char *node_name, t_node_info *node_info);
+bool        update_node_record(PGconn *conn, char *action, int node, char *type, int upstream_node, char *cluster_name, char *node_name, char *conninfo, int priority, char *slot_name, bool active);
 bool        update_node_record_status(PGconn *conn, char *cluster_name, int this_node_id, char *type, int upstream_node_id, bool active);
 bool        update_node_record_set_upstream(PGconn *conn, char *cluster_name, int this_node_id, int new_upstream_node_id);
 bool        create_event_record(PGconn *conn, t_configuration_options *options, int node_id, char *event, bool successful, char *details);

debian/DEBIAN/control

@@ -1,9 +1,9 @@
 Package: repmgr-auto
-Version: 3.0.1
+Version: 3.2dev
 Section: database
 Priority: optional
 Architecture: all
-Depends: rsync, postgresql-9.3 | postgresql-9.4
+Depends: rsync, postgresql-9.3 | postgresql-9.4 | postgresql-9.5
 Maintainer: Self built package <user@localhost>
 Description: PostgreSQL replication setup, magament and monitoring
  has two main executables

dirmod.c

@@ -0,0 +1,194 @@
+/*
+ *
+ * dirmod.c
+ *	  directory handling functions
+ *
+ * Copyright (c) 2ndQuadrant, 2010-2017
+ *
+ * Portions Copyright (c) 1996-2013, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "postgres_fe.h"
+
+/* Don't modify declarations in system headers */
+
+#include <unistd.h>
+#include <dirent.h>
+#include <sys/stat.h>
+
+/*
+ * pgfnames
+ *
+ * return a list of the names of objects in the argument directory.  Caller
+ * must call pgfnames_cleanup later to free the memory allocated by this
+ * function.
+ */
+char	  **
+pgfnames(const char *path)
+{
+	DIR		   *dir;
+	struct dirent *file;
+	char	  **filenames;
+	int			numnames = 0;
+	int			fnsize = 200;	/* enough for many small dbs */
+
+	dir = opendir(path);
+	if (dir == NULL)
+	{
+		return NULL;
+	}
+
+	filenames = (char **) palloc(fnsize * sizeof(char *));
+
+	while (errno = 0, (file = readdir(dir)) != NULL)
+	{
+		if (strcmp(file->d_name, ".") != 0 && strcmp(file->d_name, "..") != 0)
+		{
+			if (numnames + 1 >= fnsize)
+			{
+				fnsize *= 2;
+				filenames = (char **) repalloc(filenames,
+											   fnsize * sizeof(char *));
+			}
+			filenames[numnames++] = pstrdup(file->d_name);
+		}
+	}
+
+	if (errno)
+	{
+		fprintf(stderr, _("could not read directory \"%s\": %s\n"),
+				path, strerror(errno));
+	}
+
+	filenames[numnames] = NULL;
+
+	if (closedir(dir))
+	{
+		fprintf(stderr, _("could not close directory \"%s\": %s\n"),
+				path, strerror(errno));
+	}
+
+	return filenames;
+}
+
+
+/*
+ *	pgfnames_cleanup
+ *
+ *	deallocate memory used for filenames
+ */
+void
+pgfnames_cleanup(char **filenames)
+{
+	char	  **fn;
+
+	for (fn = filenames; *fn; fn++)
+		pfree(*fn);
+
+	pfree(filenames);
+}
+
+
+/*
+ *	rmtree
+ *
+ *	Delete a directory tree recursively.
+ *	Assumes path points to a valid directory.
+ *	Deletes everything under path.
+ *	If rmtopdir is true deletes the directory too.
+ *	Returns true if successful, false if there was any problem.
+ *	(The details of the problem are reported already, so caller
+ *	doesn't really have to say anything more, but most do.)
+ */
+bool
+rmtree(const char *path, bool rmtopdir)
+{
+	bool		result = true;
+	char		pathbuf[MAXPGPATH];
+	char	  **filenames;
+	char	  **filename;
+	struct stat statbuf;
+
+	/*
+	 * we copy all the names out of the directory before we start modifying
+	 * it.
+	 */
+	filenames = pgfnames(path);
+
+	if (filenames == NULL)
+		return false;
+
+	/* now we have the names we can start removing things */
+	for (filename = filenames; *filename; filename++)
+	{
+		snprintf(pathbuf, MAXPGPATH, "%s/%s", path, *filename);
+
+		/*
+		 * It's ok if the file is not there anymore; we were just about to
+		 * delete it anyway.
+		 *
+		 * This is not an academic possibility. One scenario where this
+		 * happens is when bgwriter has a pending unlink request for a file in
+		 * a database that's being dropped. In dropdb(), we call
+		 * ForgetDatabaseFsyncRequests() to flush out any such pending unlink
+		 * requests, but because that's asynchronous, it's not guaranteed that
+		 * the bgwriter receives the message in time.
+		 */
+		if (lstat(pathbuf, &statbuf) != 0)
+		{
+			if (errno != ENOENT)
+			{
+				result = false;
+			}
+			continue;
+		}
+
+		if (S_ISDIR(statbuf.st_mode))
+		{
+			/* call ourselves recursively for a directory */
+			if (!rmtree(pathbuf, true))
+			{
+				/* we already reported the error */
+				result = false;
+			}
+		}
+		else
+		{
+			if (unlink(pathbuf) != 0)
+			{
+				if (errno != ENOENT)
+				{
+					result = false;
+				}
+			}
+		}
+	}
+
+	if (rmtopdir)
+	{
+		if (rmdir(path) != 0)
+		{
+			result = false;
+		}
+	}
+
+	pgfnames_cleanup(filenames);
+
+	return result;
+}
+

dirmod.h

@@ -0,0 +1,23 @@
+/*
+ * dirmod.h
+ * Copyright (c) 2ndQuadrant, 2010-2017
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef _DIRMOD_H_
+#define _DIRMOD_H_
+
+#endif

docs/repmgrd-failover-mechanism.md

@@ -0,0 +1,75 @@
+repmgrd's failover algorithm
+============================
+
+When implementing automatic failover, there are two factors which are critical in
+ensuring the desired result is achieved:
+
+  - has the master node genuinely failed?
+  - which is the best node to promote to the new master?
+
+This document outlines repmgrd's decision-making process during automatic failover
+for standbys directly connected to the master node.
+
+
+Master node failure detection
+-----------------------------
+
+If a `repmgrd` instance running on a PostgreSQL standby node is unable to connect to
+the master node, this doesn't neccesarily mean that the master is down and a
+failover is required. Factors such as network connectivity issues could mean that
+even though the standby node is isolated, the replication cluster as a whole
+is functioning correctly, and promoting the standby without further verification
+could result in a "split-brain" situation.
+
+In the event that `repmgrd` is unable to connect to the master node, it will attempt
+to reconnect to the master server several times (as defined by the `reconnect_attempts`
+parameter in `repmgr.conf`), with reconnection attempts  occuring at the interval
+specified by `reconnect_interval`. This happens to verify that the master is definitively
+not accessible (e.g. that connection was not lost due to a brief network glitch).
+
+Appropriate values for these settings will depend very much on the replication
+cluster environment. There will necessarily be a trade-off between the time it
+takes to assume the master is not reachable, and the reliability of that conclusion.
+A standby in a different physical location to the master will probably need a longer
+check interval to rule out possible network issues, whereas one located in the same
+rack with a direct connection between servers could perform the check very quickly.
+
+Note that it's possible the master comes back online after this point is reached,
+but before a new master has been selected; in this case it will be noticed
+during the selection of a new master and no actual failover will take place.
+
+Promotion candidate selection
+-----------------------------
+
+Once `repmgrd` has decided the master is definitively unreachable, following checks
+will be carried out:
+
+* attempts to connect to all other nodes in the cluster (including the witness
+  node, if defined) to establish the state of the cluster, including their
+  current LSN
+
+* If less than half of the nodes are visible (from the viewpoint
+  of this node), `repmgrd` will not take any further action. This is to ensure that
+  e.g. if a replication cluster is spread over multiple data centres, a split-brain
+  situation does not occur if there is a network failure between datacentres. Note
+  that if nodes are split evenly between data centres, a witness server can be
+  used to establish the "majority" daat centre.
+
+* `repmgrd` polls all visible servers and waits for each node to return a valid LSN;
+  it updates the LSN previously  stored for this node if it has increased since
+  the initial check
+
+* once all LSNs have been retrieved, `repmgrd` will check for the highest LSN; if
+  its own node has the highest LSN, it will attempt to promote itself (using the
+  command defined in `promote_command` in `repmgr.conf`. Note that if using
+  `repmgr standby promote` as the promotion command, and the original master becomes available
+  before the promotion takes effect, `repmgr` will return an error and no promotion
+  will take place, and `repmgrd` will resume monitoring as usual.
+
+* if the node is not the promotion candidate, `repmgrd` will execute the
+  `follow_command` defined in `repmgr.conf`. If using `repmgr standby follow` here,
+  `repmgr` will attempt to detect the new master node and attach to that.
+
+
+
+

docs/repmgrd-node-fencing.md

@@ -0,0 +1,152 @@
+Fencing a failed master node with repmgrd and pgbouncer
+=======================================================
+
+With automatic failover, it's essential to ensure that a failed master
+remains inaccessible to your application, even if it comes back online
+again, to avoid a split-brain situation.
+
+By using `pgbouncer` together with `repmgrd`, it's possible to combine
+automatic failover with a process to isolate the failed master from
+your application and ensure that all connections which should go to
+the master are directed there smoothly without having to reconfigure
+your application. (Note that as a connection pooler, `pgbouncer` can
+benefit your application in other ways, but those are beyond the scope
+of this document).
+
+* * *
+
+> *WARNING*: automatic failover is tricky to get right. This document
+> demonstrates one possible implementation method, however you should
+> carefully configure and test any setup to suit the needs of your own
+> replication cluster/application.
+
+* * *
+
+In a failover situation, `repmgrd` promotes a standby to master by executing
+the command defined in `promote_command`. Normally this would be something like:
+
+    repmgr standby promote -f /etc/repmgr.conf
+
+By wrapping this in a custom script which adjusts the `pgbouncer` configuration
+on all nodes, it's possible to fence the failed master and redirect write
+connections to the new master.
+
+The script consists of three sections:
+
+* commands to pause `pgbouncer` on all nodes
+* the promotion command itself
+* commands to reconfigure and restart `pgbouncer` on all nodes
+
+Note that it requires password-less SSH access between all nodes to be able to
+update the `pgbouncer` configuration files.
+
+For the purposes of this demonstration, we'll assume there are 3 nodes (master
+and two standbys), with `pgbouncer` listening on port 6432 handling connections
+to a database called `appdb`.  The `postgres` system user must have write
+access to the `pgbouncer` configuration files on all nodes. We'll assume
+there's a main `pgbouncer` configuration file, `/etc/pgbouncer.ini`, which uses
+the `%include` directive (available from PgBouncer 1.6) to include a separate
+configuration file, `/etc/pgbouncer.database.ini`, which will be modified by
+`repmgr`.
+
+`/etc/pgbouncer.ini` should look something like this:
+
+    [pgbouncer]
+
+    logfile = /var/log/pgbouncer/pgbouncer.log
+    pidfile = /var/run/pgbouncer/pgbouncer.pid
+
+    listen_addr = *
+    listen_port = 6532
+    unix_socket_dir = /tmp
+
+    auth_type = trust
+    auth_file = /etc/pgbouncer.auth
+
+    admin_users = postgres
+    stats_users = postgres
+
+    pool_mode = transaction
+
+    max_client_conn = 100
+    default_pool_size = 20
+    min_pool_size = 5
+    reserve_pool_size = 5
+    reserve_pool_timeout = 3
+
+    log_connections = 1
+    log_disconnections = 1
+    log_pooler_errors = 1
+
+    %include /etc/pgbouncer.database.ini
+
+The actual script is as follows; adjust the configurable items as appropriate:
+
+`/var/lib/postgres/repmgr/promote.sh`
+
+
+    #!/usr/bin/env bash
+    set -u
+    set -e
+
+    # Configurable items
+    PGBOUNCER_HOSTS="node1 node2 node3"
+    PGBOUNCER_DATABASE_INI="/etc/pgbouncer.database.ini"
+    PGBOUNCER_DATABASE="appdb"
+    PGBOUNCER_PORT=6432
+
+    REPMGR_DB="repmgr"
+    REPMGR_USER="repmgr"
+    REPMGR_SCHEMA="repmgr_test"
+
+    # 1. Pause running pgbouncer instances
+    for HOST in $PGBOUNCER_HOSTS
+    do
+        psql -t -c "pause" -h $HOST -p $PGBOUNCER_PORT -U postgres pgbouncer
+    done
+
+    # 2. Promote this node from standby to master
+
+    repmgr standby promote -f /etc/repmgr.conf
+
+    # 3. Reconfigure pgbouncer instances
+
+    PGBOUNCER_DATABASE_INI_NEW="/tmp/pgbouncer.database.ini"
+
+    for HOST in $PGBOUNCER_HOSTS
+    do
+        # Recreate the pgbouncer config file
+        echo -e "[databases]\n" > $PGBOUNCER_DATABASE_INI_NEW
+
+        psql -d $REPMGR_DB -U $REPMGR_USER -t -A \
+          -c "SELECT '${PGBOUNCER_DATABASE}-rw= ' || conninfo || ' application_name=pgbouncer_${HOST}' \
+              FROM ${REPMGR_SCHEMA}.repl_nodes \
+              WHERE active = TRUE AND type='master'" >> $PGBOUNCER_DATABASE_INI_NEW
+
+        psql -d $REPMGR_DB -U $REPMGR_USER -t -A \
+          -c "SELECT '${PGBOUNCER_DATABASE}-ro= ' || conninfo || ' application_name=pgbouncer_${HOST}' \
+              FROM $REPMGR_SCHEMA.repl_nodes \
+              WHERE node_name='${HOST}'" >> $PGBOUNCER_DATABASE_INI_NEW
+
+        rsync $PGBOUNCER_DATABASE_INI_NEW $HOST:$PGBOUNCER_DATABASE_INI
+
+        psql -tc "reload" -h $HOST -p $PGBOUNCER_PORT -U postgres pgbouncer
+        psql -tc "resume" -h $HOST -p $PGBOUNCER_PORT -U postgres pgbouncer
+
+    done
+
+    # Clean up generated file
+    rm $PGBOUNCER_DATABASE_INI_NEW
+
+    echo "Reconfiguration of pgbouncer complete"
+
+Script and template file should be installed on each node where
+`repmgrd` is running.
+
+Finally, set `promote_command` in `repmgr.conf` on each node to
+point to the custom promote script:
+
+    promote_command=/var/lib/postgres/repmgr/promote.sh
+
+and reload/restart any running `repmgrd` instances for the changes to take
+effect.

errcode.h

@@ -1,6 +1,6 @@
 /*
  * errcode.h
- * Copyright (C) 2ndQuadrant, 2010-2016
+ * Copyright (c) 2ndQuadrant, 2010-2017
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -29,7 +29,6 @@
 #define ERR_DB_CON 6
 #define ERR_DB_QUERY 7
 #define ERR_PROMOTED 8
-#define ERR_BAD_PASSWORD 9
 #define ERR_STR_OVERFLOW 10
 #define ERR_FAILOVER_FAIL 11
 #define ERR_BAD_SSH 12
@@ -38,5 +37,9 @@
 #define ERR_INTERNAL 15
 #define ERR_MONITORING_FAIL 16
 #define ERR_BAD_BACKUP_LABEL 17
+#define ERR_SWITCHOVER_FAIL 18
+#define ERR_BARMAN 19
+#define ERR_REGISTRATION_SYNC 20
+
 
 #endif   /* _ERRCODE_H_ */

log.c

@@ -1,6 +1,6 @@
 /*
  * log.c - Logging methods
- * Copyright (C) 2ndQuadrant, 2010-2016
+ * Copyright (c) 2ndQuadrant, 2010-2017
  *
  * This module is a set of methods for logging (currently only syslog)
  *
@@ -40,15 +40,21 @@
 /* #define REPMGR_DEBUG */
 
 static int	detect_log_facility(const char *facility);
-static void _stderr_log_with_level(const char *level_name, int level, const char *fmt, va_list ap);
+static void _stderr_log_with_level(const char *level_name, int level, const char *fmt, va_list ap)
+__attribute__((format(PG_PRINTF_ATTRIBUTE, 3, 0)));
 
 int			log_type = REPMGR_STDERR;
 int			log_level = LOG_NOTICE;
 int			last_log_level = LOG_NOTICE;
 int			verbose_logging = false;
 int			terse_logging = false;
+/*
+ * Global variable to be set by the main application to ensure any log output
+ * emitted before logger_init is called, is output in the correct format
+ */
+int			logger_output_mode = OM_DAEMON;
 
-void
+extern void
 stderr_log_with_level(const char *level_name, int level, const char *fmt, ...)
 {
 	va_list		arglist;
@@ -61,9 +67,7 @@ stderr_log_with_level(const char *level_name, int level, const char *fmt, ...)
 static void
 _stderr_log_with_level(const char *level_name, int level, const char *fmt, va_list ap)
 {
-	time_t		t;
+	char		buf[100];
-	struct tm  *tm;
-	char		buff[100];
 
 	/*
 	 * Store the requested level so that if there's a subsequent
@@ -73,10 +77,21 @@ _stderr_log_with_level(const char *level_name, int level, const char *fmt, va_li
 
 	if (log_level >= level)
 	{
-		time(&t);
+
-		tm = localtime(&t);
+		/* Format log line prefix with timestamp if in daemon mode */
-		strftime(buff, 100, "[%Y-%m-%d %H:%M:%S]", tm);
+		if (logger_output_mode == OM_DAEMON)
-		fprintf(stderr, "%s [%s] ", buff, level_name);
+		{
+			time_t		t;
+			struct tm  *tm;
+			time(&t);
+			tm = localtime(&t);
+			strftime(buf, 100, "[%Y-%m-%d %H:%M:%S]", tm);
+			fprintf(stderr, "%s [%s] ", buf, level_name);
+		}
+		else
+		{
+			fprintf(stderr, "%s: ", level_name);
+		}
 
 		vfprintf(stderr, fmt, ap);
 
@@ -141,7 +156,7 @@ log_verbose(int level, const char *fmt, ...)
 
 
 bool
-logger_init(t_configuration_options * opts, const char *ident)
+logger_init(t_configuration_options *opts, const char *ident)
 {
 	char	   *level = opts->loglevel;
 	char	   *facility = opts->logfacility;
@@ -175,6 +190,13 @@ logger_init(t_configuration_options * opts, const char *ident)
 			stderr_log_warning(_("Invalid log level \"%s\" (available values: DEBUG, INFO, NOTICE, WARNING, ERR, ALERT, CRIT or EMERG)\n"), level);
 	}
 
+	/*
+	 * STDERR only logging requested - finish here without setting up any further
+	 * logging facility.
+	 */
+	if (logger_output_mode == OM_COMMAND_LINE)
+		return true;
+
 	if (facility && *facility)
 	{
 
@@ -235,9 +257,10 @@ logger_init(t_configuration_options * opts, const char *ident)
 		stderr_log_notice(_("Redirecting logging output to '%s'\n"), opts->logfile);
 		fd = freopen(opts->logfile, "a", stderr);
 
-		/* It's possible freopen() may still fail due to e.g. a race condition;
+		/*
-		   as it's not feasible to restore stderr after a failed freopen(),
+		 * It's possible freopen() may still fail due to e.g. a race condition;
-		   we'll write to stdout as a last resort.
+		 * as it's not feasible to restore stderr after a failed freopen(),
+		 * we'll write to stdout as a last resort.
 		 */
 		if (fd == NULL)
 		{

log.h

@@ -1,6 +1,6 @@
 /*
  * log.h
- * Copyright (c) 2ndQuadrant, 2010-2016
+ * Copyright (c) 2ndQuadrant, 2010-2017
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -25,7 +25,10 @@
 #define REPMGR_SYSLOG 1
 #define REPMGR_STDERR 2
 
-void
+#define OM_COMMAND_LINE 1
+#define OM_DAEMON       2
+
+extern void
 stderr_log_with_level(const char *level_name, int level, const char *fmt,...)
 __attribute__((format(PG_PRINTF_ATTRIBUTE, 3, 4)));
 
@@ -123,10 +126,15 @@ bool		logger_shutdown(void);
 void		logger_set_verbose(void);
 void		logger_set_terse(void);
 
-void		log_hint(const char *fmt, ...);
+void		log_hint(const char *fmt, ...)
-void		log_verbose(int level, const char *fmt, ...);
+__attribute__((format(PG_PRINTF_ATTRIBUTE, 1, 2)));
+void		log_verbose(int level, const char *fmt, ...)
+__attribute__((format(PG_PRINTF_ATTRIBUTE, 2, 3)));
 
 extern int	log_type;
 extern int	log_level;
+extern int	verbose_logging;
+extern int	terse_logging;
+extern int	logger_output_mode;
 
-#endif
+#endif /* _REPMGR_LOG_H_ */

repmgr.conf.sample

@@ -19,7 +19,7 @@
 
 # Node ID and name
 # (Note: we recommend to avoid naming nodes after their initial
-#  replication funcion, as this will cause confusion when e.g.
+#  replication function, as this will cause confusion when e.g.
 #  "standby2" is promoted to primary)
 #node=2           # a unique integer
 #node_name=node2  # an arbitrary (but unique) string; we recommend using
@@ -28,8 +28,16 @@
 
 # Database connection information as a conninfo string
 # This must be accessible to all servers in the cluster; for details see:
-#   http://www.postgresql.org/docs/current/static/libpq-connect.html#LIBPQ-CONNSTRING
+#
-#conninfo='host=192.168.204.104 dbname=repmgr_db user=repmgr_usr'
+#   https://www.postgresql.org/docs/current/static/libpq-connect.html#LIBPQ-CONNSTRING
+#
+#conninfo='host=192.168.204.104 dbname=repmgr user=repmgr'
+#
+# If repmgrd is in use, consider explicitly setting `connect_timeout` in the
+# conninfo string to determine the length of time which elapses before
+# a network connection attempt is abandoned; for details see:
+#
+#   https://www.postgresql.org/docs/current/static/libpq-connect.html#LIBPQ-CONNECT-CONNECT-TIMEOUT
 
 # Optional configuration items
 # ============================
@@ -58,6 +66,12 @@
 # (default: NOTICE)
 #loglevel=NOTICE
 
+# Note that logging facility settings will only apply to `repmgrd` by default;
+# `repmgr` will always write to STDERR unless the switch `--log-to-file` is
+# supplied, in which case it will log to the same destination as `repmgrd`.
+# This is mainly intended for those cases when `repmgr` is executed directly
+# by `repmgrd`.
+
 # Logging facility: possible values are STDERR or - for Syslog integration - one of LOCAL0, LOCAL1, ..., LOCAL7, USER
 # (default: STDERR)
 #logfacility=STDERR
@@ -92,6 +106,34 @@
 # path to PostgreSQL binary directory (location of pg_ctl, pg_basebackup etc.)
 # (if not provided, defaults to system $PATH)
 #pg_bindir=/usr/bin/
+#
+# Debian/Ubuntu users: you will probably need to set this to the directory
+# where `pg_ctl` is located, e.g. /usr/lib/postgresql/9.5/bin/
+
+# service control commands
+#
+# repmgr provides options to override the default pg_ctl commands
+# used to stop, start, restart, reload and promote the PostgreSQL cluster
+#
+# NOTE: These commands must be runnable on remote nodes as well for switchover
+# to function correctly.
+#
+# If you use sudo, the user repmgr runs as (usually 'postgres')  must have
+# passwordless sudo access to execute the command
+#
+# For example, to use systemd, you may use the following configuration:
+#
+#    # this is required when running sudo over ssh without -t:
+#    Defaults:postgres !requiretty
+#    postgres ALL = NOPASSWD: /usr/bin/systemctl stop postgresql-9.5, \
+#       /usr/bin/systemctl start postgresql-9.5, \
+#       /usr/bin/systemctl restart postgresql-9.5
+#
+# service_start_command = systemctl start postgresql-9.5
+# service_stop_command = systemctl stop postgresql-9.5
+# service_restart_command = systemctl restart postgresql-9.5
+# service_reload_command = pg_ctlcluster 9.5 main reload
+# service_promote_command = pg_ctlcluster 9.5 main promote
 
 # external command options
 
@@ -101,8 +143,15 @@
 # external command arguments. Values shown are examples.
 
 #pg_ctl_options='-s'
-#pg_basebackup_options='--xlog-method=s'
+#pg_basebackup_options='--label=repmgr_backup'
 
+# This is the host name of the barman server, which is used for connecting over
+# to the barman server (passwordless ssh keys should be in place)
+#barman_server='backup_server'
+# If you are placing the barman.conf file in a non-standard path, or using
+# a name other than barman.conf, use this parameter to specify the path and
+# name of the barman configuration file.
+#barman_config='/path/to/barman.conf'
 
 # Standby clone settings
 # ----------------------
@@ -113,6 +162,10 @@
 #
 # tablespace_mapping=/path/to/original/tablespace=/path/to/new/tablespace
 
+# You can specify a restore_command to be used in the recovery.conf that
+# will be placed in the cloned standby
+#
+# restore_command = cp /path/to/archived/wals/%f %p
 
 # Failover settings (repmgrd)
 # ---------------------------
@@ -120,9 +173,11 @@
 # These settings are only applied when repmgrd is running. Values shown
 # are defaults.
 
-# Number of seconds to wait for a response from the primary server before
+# monitoring interval in seconds; default is 2
-# deciding it has failed.
+#monitor_interval_secs=2
 
+# Maximum number of seconds to wait for a response from the primary server
+# before deciding it has failed.
 #master_response_timeout=60
 
 # Number of attempts at what interval (in seconds) to try and
@@ -132,16 +187,21 @@
 #reconnect_interval=10
 
 # Autofailover options
-#failover=manual    # one of 'automatic', 'manual'
+#failover=manual    # one of 'automatic', 'manual' (default: manual)
-                    # (default: manual)
+                    # defines the action to take in the event of upstream failure
-#priority=100       # a value of zero or less prevents the node being promoted to primary
+                    #
+                    # 'automatic': repmgrd will automatically attempt to promote the
+                    #    node or follow the new upstream node
+                    # 'manual': repmgrd will take no action and the mode will require
+                    #    manual attention to reattach it to replication
+
+#priority=100       # indicate a preferred priorty for promoting nodes
+                    # a value of zero or less prevents the node being promoted to primary
                     # (default: 100)
+
 #promote_command='repmgr standby promote -f /path/to/repmgr.conf'
 #follow_command='repmgr standby follow -f /path/to/repmgr.conf -W'
 
-# monitoring interval in seconds; default is 2
-#monitor_interval_secs=2
-
 # change wait time for primary; before we bail out and exit when the primary
 # disappears, we wait 'reconnect_attempts' * 'retry_promote_interval_secs'
 # seconds; by default this would be half an hour, as 'retry_promote_interval_secs'

repmgr.h

@@ -1,6 +1,6 @@
 /*
  * repmgr.h
- * Copyright (c) 2ndQuadrant, 2010-2016
+ * Copyright (c) 2ndQuadrant, 2010-2017
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -23,18 +23,20 @@
 #include <libpq-fe.h>
 #include <postgres_fe.h>
 #include <getopt_long.h>
+#include "pqexpbuffer.h"
 
 #include "strutil.h"
 #include "dbutils.h"
 #include "errcode.h"
 #include "config.h"
+#include "dirmod.h"
 
 #define MIN_SUPPORTED_VERSION		"9.3"
 #define MIN_SUPPORTED_VERSION_NUM	90300
 
 #define ERRBUFF_SIZE	512
 
-#define DEFAULT_WAL_KEEP_SEGMENTS	"5000"
+#define DEFAULT_WAL_KEEP_SEGMENTS	"0"
 #define DEFAULT_DEST_DIR		"."
 #define DEFAULT_REPMGR_SCHEMA_PREFIX	"repmgr_"
 #define DEFAULT_PRIORITY		100
@@ -46,55 +48,133 @@
 #define NO_UPSTREAM_NODE	-1
 #define UNKNOWN_NODE_ID     -1
 
+/* command line options without short versions */
+#define OPT_HELP                         1
+#define OPT_CHECK_UPSTREAM_CONFIG        2
+#define OPT_RECOVERY_MIN_APPLY_DELAY     3
+#define OPT_COPY_EXTERNAL_CONFIG_FILES   4
+#define OPT_CONFIG_ARCHIVE_DIR           5
+#define OPT_PG_REWIND                    6
+#define OPT_PWPROMPT                     7
+#define OPT_CSV                          8
+#define OPT_NODE                         9
+#define OPT_WITHOUT_BARMAN               10
+#define OPT_NO_UPSTREAM_CONNECTION       11
+#define OPT_REGISTER_WAIT                12
+#define OPT_CLUSTER                      13
+#define OPT_LOG_TO_FILE                  14
+#define OPT_UPSTREAM_CONNINFO            15
+#define OPT_NO_CONNINFO_PASSWORD         16
+#define OPT_REPLICATION_USER             17
+
+/* deprecated command line options */
+#define OPT_INITDB_NO_PWPROMPT           998
+#define OPT_IGNORE_EXTERNAL_CONFIG_FILES 999
+
+/* values for --copy-external-config-files */
+#define CONFIG_FILE_SAMEPATH 1
+#define CONFIG_FILE_PGDATA 2
 
 
 /* Run time options type */
 typedef struct
 {
+	/* general repmgr options */
+	char		config_file[MAXPGPATH];
+	bool		verbose;
+	bool		terse;
+	bool		force;
+	char		pg_bindir[MAXLEN]; /* overrides setting in repmgr.conf */
 
+	/* logging parameters */
+	char		loglevel[MAXLEN];  /* overrides setting in repmgr.conf */
+	bool		log_to_file;
+
+	/* connection parameters */
 	char		dbname[MAXLEN];
 	char		host[MAXLEN];
 	char		username[MAXLEN];
 	char		dest_dir[MAXPGPATH];
-	char		config_file[MAXPGPATH];
 	char		remote_user[MAXLEN];
 	char		superuser[MAXLEN];
+	char		masterport[MAXLEN];
+	bool		conninfo_provided;
+	bool		connection_param_provided;
+	bool		host_param_provided;
+
+	/* standby clone parameters */
+	bool		wal_keep_segments_used;
 	char		wal_keep_segments[MAXLEN];
-	bool		verbose;
-	bool		terse;
-	bool		force;
-	bool		wait_for_master;
 	bool		ignore_rsync_warn;
-	bool		witness_pwprompt;
 	bool		rsync_only;
 	bool		fast_checkpoint;
-	bool		ignore_external_config_files;
+	bool		without_barman;
-	char		pg_ctl_mode[MAXLEN];
+	bool		no_upstream_connection;
-	char		masterport[MAXLEN];
+	bool		no_conninfo_password;
-	/*
+	bool		copy_external_config_files;
-	 * configuration file parameters which can be overridden on the
+	int			copy_external_config_files_destination;
-	 * command line
+	char		upstream_conninfo[MAXLEN];
-	 */
+	char		replication_user[MAXLEN];
-	char		loglevel[MAXLEN];
-
-	/* parameter used by STANDBY SWITCHOVER */
-	char		remote_config_file[MAXLEN];
-	char		pg_rewind[MAXPGPATH];
-	/* parameter used by STANDBY {ARCHIVE_CONFIG | RESTORE_CONFIG} */
-	char		config_archive_dir[MAXLEN];
-	/* parameter used by CLUSTER CLEANUP */
-	int			keep_history;
-
-	char		pg_bindir[MAXLEN];
 
 	char		recovery_min_apply_delay[MAXLEN];
 
-	/* deprecated command line options */
+	/* standby register paarameters */
-	char		localport[MAXLEN];
+	bool		wait_register_sync;
-	bool		initdb_no_pwprompt;
+	int			wait_register_sync_seconds;
+
+	/* witness create parameters */
+	bool		witness_pwprompt;
+
+	/* standby follow parameters */
+	bool		wait_for_master;
+
+	/* cluster {show|matrix|crosscheck} parameters */
+	bool		csv_mode;
+
+	/* cluster cleanup parameters */
+	int			keep_history;
+
+	/* standby switchover parameters */
+	char		remote_config_file[MAXLEN];
+	bool		pg_rewind_supplied;
+	char		pg_rewind[MAXPGPATH];
+	char		pg_ctl_mode[MAXLEN];
+
+	/* standby {archive_config | restore_config} parameters  */
+	char		config_archive_dir[MAXLEN];
+
+	/* {standby|witness} unregister parameters */
+	int			node;
+
 }	t_runtime_options;
 
-#define T_RUNTIME_OPTIONS_INITIALIZER { "", "", "", "", "", "", "", DEFAULT_WAL_KEEP_SEGMENTS, false, false, false, false, false, false, false, false, false, "smart", "", "", "", "", "", 0, "", "", "", false }
+#define T_RUNTIME_OPTIONS_INITIALIZER { \
+		/* general repmgr options */	\
+		"", false, false, false, "",	\
+		/* logging parameters */ \
+		"", false,                      \
+		/* connection parameters */		\
+		"", "", "", "", "", "", "", 	\
+		false, false, false,		    \
+		/* standby clone parameters */  \
+		false, DEFAULT_WAL_KEEP_SEGMENTS, false, false, false, false, false, false, \
+		false, CONFIG_FILE_SAMEPATH, "", "", "", \
+		/* standby register paarameters */ \
+	    false, 0,							 \
+		/* witness create parameters */ \
+		false,                          \
+		/* standby follow parameters */ \
+		false,                          \
+		/* cluster {show|matrix|crosscheck} parameters */ \
+		false,                          \
+		/* cluster cleanup parameters */ \
+		0,                              \
+		/* standby switchover parameters */ \
+		"", false, "", "fast",          \
+		/* standby {archive_config | restore_config} parameters  */ \
+		"",                             \
+		/* {standby|witness} unregister parameters */ \
+		UNKNOWN_NODE_ID }
 
 struct BackupLabel
 {
@@ -108,7 +188,60 @@ struct BackupLabel
 	XLogRecPtr min_failover_slot_lsn;
 };
 
-extern char		repmgr_schema[MAXLEN];
+
-extern bool		config_file_found;
+typedef struct
+{
+	char		slot[MAXLEN];
+	char		xlog_method[MAXLEN];
+} t_basebackup_options;
+
+#define T_BASEBACKUP_OPTIONS_INITIALIZER { "", "" }
+
+typedef struct
+{
+	int    size;
+	char **keywords;
+	char **values;
+} t_conninfo_param_list;
+
+typedef struct
+{
+	char filepath[MAXPGPATH];
+	char filename[MAXPGPATH];
+	bool in_data_directory;
+} t_configfile_info;
+
+
+typedef struct
+{
+	int    size;
+	int    entries;
+	t_configfile_info **files;
+} t_configfile_list;
+
+#define T_CONFIGFILE_LIST_INITIALIZER { 0, 0, NULL }
+
+
+typedef struct
+{
+	int node_id;
+	int node_status;
+} t_node_status_rec;
+
+typedef struct
+{
+	int node_id;
+	char node_name[MAXLEN];
+	t_node_status_rec **node_status_list;
+} t_node_matrix_rec;
+
+typedef struct
+{
+	int node_id;
+	char node_name[MAXLEN];
+	t_node_matrix_rec **matrix_list_rec;
+} t_node_status_cube;
+
+
 
 #endif

repmgr.sql

@@ -1,7 +1,7 @@
 /*
  * repmgr.sql
  *
- * Copyright (C) 2ndQuadrant, 2010-2016
+ * Copyright (c) 2ndQuadrant, 2010-2017
  *
  */
 

sql/Makefile

@@ -1,7 +1,7 @@
 #
 # Makefile
 #
-# Copyright (c) 2ndQuadrant, 2010-2016
+# Copyright (c) 2ndQuadrant, 2010-2017
 #
 
 MODULE_big = repmgr_funcs

sql/repmgr2_repmgr3.sql

@@ -63,6 +63,15 @@ UPDATE repl_nodes SET type = 'master' WHERE id = $master_id;
 
 -- UPDATE repl_nodes SET active = FALSE WHERE id IN (...);
 
+/* There's also an event table which we need to create */
+CREATE TABLE repl_events (
+  node_id          INTEGER NOT NULL,
+  event            TEXT NOT NULL,
+  successful       BOOLEAN NOT NULL DEFAULT TRUE,
+  event_timestamp  TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  details          TEXT NULL
+);
+
 /* When you're sure of your changes, commit them */
 
 -- COMMIT;

sql/repmgr3.1.1_repmgr3.1.2.sql

@@ -27,5 +27,6 @@
 
 BEGIN;
 
-ALTER TABLE repl_nodes ALTER CONSTRAINT repl_nodes_upstream_node_id_fkey DEFERRABLE;
+ALTER TABLE repl_nodes DROP CONSTRAINT repl_nodes_upstream_node_id_fkey,
+      ADD CONSTRAINT repl_nodes_upstream_node_id_fkey FOREIGN KEY (upstream_node_id) REFERENCES repl_nodes(id) DEFERRABLE;
 COMMIT;

sql/repmgr_funcs.sql.in

@@ -1,6 +1,6 @@
 /*
  * repmgr_function.sql
- * Copyright (c) 2ndQuadrant, 2010-2016
+ * Copyright (c) 2ndQuadrant, 2010-2017
  *
  */
 

sql/uninstall_repmgr_funcs.sql

@@ -1,6 +1,6 @@
 /*
  * uninstall_repmgr_funcs.sql
- * Copyright (c) 2ndQuadrant, 2010-2016
+ * Copyright (c) 2ndQuadrant, 2010-2017
  *
  */
 

strutil.c

@@ -1,7 +1,7 @@
 /*
  * strutil.c
  *
- * Copyright (C) 2ndQuadrant, 2010-2016
+ * Copyright (c) 2ndQuadrant, 2010-2017
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -87,3 +87,21 @@ maxlen_snprintf(char *str, const char *format,...)
 
 	return retval;
 }
+
+
+/*
+ * Escape a string for use as a parameter in recovery.conf
+ * Caller must free returned value
+ */
+char *
+escape_recovery_conf_value(const char *src)
+{
+	char	   *result = escape_single_quotes_ascii(src);
+
+	if (!result)
+	{
+		fprintf(stderr, _("%s: out of memory\n"), progname());
+		exit(ERR_INTERNAL);
+	}
+	return result;
+}

strutil.h

@@ -1,6 +1,6 @@
 /*
  * strutil.h
- * Copyright (C) 2ndQuadrant, 2010-2016
+ * Copyright (c) 2ndQuadrant, 2010-2017
  *
  *
  * This program is free software: you can redistribute it and/or modify
@@ -22,6 +22,7 @@
 #define _STRUTIL_H_
 
 #include <stdlib.h>
+#include "pqexpbuffer.h"
 #include "errcode.h"
 
 
@@ -48,4 +49,6 @@ extern int
 maxlen_snprintf(char *str, const char *format,...)
 __attribute__((format(PG_PRINTF_ATTRIBUTE, 2, 3)));
 
+extern char *
+escape_recovery_conf_value(const char *src);
 #endif   /* _STRUTIL_H_ */

uninstall_repmgr.sql

@@ -1,7 +1,7 @@
 /*
  * uninstall_repmgr.sql
  *
- * Copyright (C) 2ndQuadrant, 2010-2016
+ * Copyright (c) 2ndQuadrant, 2010-2017
  *
  */
 

version.h

@@ -1,6 +1,6 @@
 #ifndef _VERSION_H_
 #define _VERSION_H_
 
-#define REPMGR_VERSION "3.2dev"
+#define REPMGR_VERSION "3.3.1"
 
 #endif