Fix oversight in the header of guc_setted_typed()

Patch by: Christian Kruse

CREDITS

@@ -10,3 +10,7 @@ Hannu Krosing <hannu@2ndQuadrant.com>
 Cédric Villemain <cedric@2ndquadrant.com>
 Charles Duffy <charles@dyfis.net>
 Daniel Farina <daniel@heroku.com>
+Shawn Ellis <shawn.ellis17@gmail.com>
+Jay Taylor <jay@jaytaylor.com>
+Christian Kruse <christian@2ndQuadrant.com>
+Krzysztof Gajdemski <songo@debian.org.pl>

HISTORY

@@ -1,4 +1,11 @@
-2.0beta 2012-07-27
+2.0beta2 2013-12-19
+        Improve autofailover logic and algorithms (Jaime, Andres)
+        Ignore pg_log when cloning (Jaime)
+        Add timestamps to log line in stderr (Christian)
+        Correctly check wal_keep_segments (Jay Taylor)
+        Add a ssh_options parameter (Jay Taylor)
+
+2.0beta1 2012-07-27
         Make CLONE command try to make an exact copy including $PGDATA location (Cedric) 
         Add detection of master failure (Jaime)
         Add the notion of a witness server (Jaime)

autofailover_quick_setup.rst

@@ -112,7 +112,7 @@ Log in node2.
 Clone the node1 (the current Master)::
 
   su - postgres
-  repmgr -d repmgr -U repmgr standby clone node1
+  repmgr -d repmgr -U repmgr -h node1 standby clone 
 
 Start the PostgreSQL server::
 
@@ -173,7 +173,7 @@ Log in witness.
 Initialize the witness server::
 
   su - postgres
-  repmgr -d repmgr -U repmgr -h 192.168.1.10 -D $WITNESS_PGDATA -f /etc/repmgr/repmgr.conf witness create node1
+  repmgr -d repmgr -U repmgr -h 192.168.1.10 -D $WITNESS_PGDATA -f /etc/repmgr/repmgr.conf witness create 
 
 It needs information to connect to the master to copy the configuration of the cluster, also it needs to know where it should initialize it's own $PGDATA.
 As part of the procees it also ask for the superuser password so it can connect when needed.

config.c

@@ -41,6 +41,7 @@ parse_config(const char *config_file, t_configuration_options *options)
 	memset(options->promote_command, 0, sizeof(options->promote_command));
 	memset(options->follow_command, 0, sizeof(options->follow_command));
 	memset(options->rsync_options, 0, sizeof(options->rsync_options));
+	memset(options->ssh_options, 0, sizeof(options->ssh_options));
 
 	/* if nothing has been provided defaults to 60 */
 	options->master_response_timeout = 60;
@@ -78,6 +79,8 @@ parse_config(const char *config_file, t_configuration_options *options)
 			strncpy (options->conninfo, value, MAXLEN);
 		else if (strcmp(name, "rsync_options") == 0)
 			strncpy (options->rsync_options, value, QUERY_STR_LEN);
+		else if (strcmp(name, "ssh_options") == 0)
+			strncpy (options->ssh_options, value, QUERY_STR_LEN);
 		else if (strcmp(name, "loglevel") == 0)
 			strncpy (options->loglevel, value, MAXLEN);
 		else if (strcmp(name, "logfacility") == 0)
@@ -283,6 +286,7 @@ reload_configuration(char *config_file, t_configuration_options *orig_options)
 	strcpy(orig_options->promote_command, new_options.promote_command);
 	strcpy(orig_options->follow_command, new_options.follow_command);
 	strcpy(orig_options->rsync_options, new_options.rsync_options);
+	strcpy(orig_options->ssh_options, new_options.ssh_options);
 	orig_options->master_response_timeout = new_options.master_response_timeout;
 	orig_options->reconnect_attempts = new_options.reconnect_attempts;
 	orig_options->reconnect_intvl = new_options.reconnect_intvl;

config.h

@@ -36,6 +36,7 @@ typedef struct
 	char loglevel[MAXLEN];
 	char logfacility[MAXLEN];
 	char rsync_options[QUERY_STR_LEN];
+	char ssh_options[QUERY_STR_LEN];
 	int  master_response_timeout;
 	int  reconnect_attempts;
 	int  reconnect_intvl;

dbutils.c

@@ -146,7 +146,8 @@ is_pgup(PGconn *conn, int timeout)
 			/*
 			* Send a SELECT 1 just to check if the connection is OK
 			*/
-			CancelQuery(conn, timeout);
+			if (!CancelQuery(conn, timeout))
+				goto failed;
 			if (wait_connection_availability(conn, timeout) != 1)
 				goto failed;
 
@@ -249,6 +250,40 @@ guc_setted(PGconn *conn, const char *parameter, const char *op,
 	return true;
 }
 
+/**
+ * Just like guc_setted except with an extra parameter containing the name of
+ * the pg datatype so that the comparison can be done properly.
+ */
+bool
+guc_setted_typed(PGconn *conn, const char *parameter, const char *op, 
+                 const char *value, const char *datatype)
+{
+	PGresult	*res;
+	char		sqlquery[QUERY_STR_LEN];
+
+	sqlquery_snprintf(sqlquery, "SELECT true FROM pg_settings "
+	                  " WHERE name = '%s' AND setting::%s %s '%s'::%s",
+	                  parameter, datatype, op, value, datatype);
+
+	res = PQexec(conn, sqlquery);
+	if (PQresultStatus(res) != PGRES_TUPLES_OK)
+	{
+		log_err(_("GUC setting check PQexec failed: %s"),
+		        PQerrorMessage(conn));
+		PQclear(res);
+		PQfinish(conn);
+		exit(ERR_DB_QUERY);
+	}
+	if (PQntuples(res) == 0)
+	{
+		PQclear(res);
+		return false;
+	}
+	PQclear(res);
+
+	return true;
+}
+
 
 const char *
 get_cluster_size(PGconn *conn)
@@ -408,7 +443,7 @@ wait_connection_availability(PGconn *conn, int timeout)
 	{
 		if (PQconsumeInput(conn) == 0)
 		{
-			log_warning(_("PQconsumeInput: Query could not be sent to primary. %s\n"), 
+			log_warning(_("wait_connection_availability: could not receive data from master. %s\n"),
 			            PQerrorMessage(conn));
 			return 0;
 		}
@@ -424,23 +459,40 @@ wait_connection_availability(PGconn *conn, int timeout)
 	}
 	if (timeout >= 0)
 		return 1;
-	else
+	else {
+		log_warning(_("wait_connection_availability: timeout reached"));
 		return -1;
 	}
+}
 
 
-void
+bool
 CancelQuery(PGconn *conn, int timeout)
 {
 	char errbuf[ERRBUFF_SIZE];
 	PGcancel *pgcancel;
 
-	wait_connection_availability(conn, timeout);
+	if (wait_connection_availability(conn, timeout) != 1)
+		return false;
 
 	pgcancel = PQgetCancel(conn);
 
-	if (!pgcancel || PQcancel(pgcancel, errbuf, ERRBUFF_SIZE) == 0)
+	if (pgcancel == NULL)
+		return false;
+
+	/*
+	 * PQcancel can only return 0 if socket()/connect()/send()
+ 	 * fails, in any of those cases we can assume something
+	 * bad happened to the connection
+	 */
+	if (PQcancel(pgcancel, errbuf, ERRBUFF_SIZE) == 0)
+	{
 		log_warning(_("Can't stop current query: %s\n"), errbuf);
+		PQfreeCancel(pgcancel);
+		return false;
+	}
 
 	PQfreeCancel(pgcancel);
+
+	return true;
 }

dbutils.h

@@ -32,10 +32,12 @@ bool	is_pgup(PGconn *conn, int timeout);
 char   *pg_version(PGconn *conn, char* major_version);
 bool	guc_setted(PGconn *conn, const char *parameter, const char *op,
                    const char *value);
+bool	guc_setted_typed(PGconn *conn, const char *parameter, const char *op,
+                   const char *value, const char *datatype);
 const char	 *get_cluster_size(PGconn *conn);
 PGconn *getMasterConnection(PGconn *standby_conn, char *schema, char *cluster,
                             int *master_id, char *master_conninfo_out);
 
 int wait_connection_availability(PGconn *conn, int timeout);
-void CancelQuery(PGconn *conn, int timeout);
+bool CancelQuery(PGconn *conn, int timeout);
 #endif

debian/DEBIAN/control

@@ -1,9 +1,9 @@
 Package: repmgr-auto
-Version: 1.0-1
+Version: 2.0beta2
 Section: database
 Priority: optional
 Architecture: all
-Depends: rsync, postgresql-9.0
+Depends: rsync, postgresql-9.0 | postgresql-9.1 | postgresql-9.2 | postgresql-9.3
-Maintainer: Greg Smith <greg@2ndQuadrant.com>
+Maintainer: Jaime Casanova <jaime@2ndQuadrant.com>
 Description: PostgreSQL replication setup, magament and monitoring
  has two main executables

debian/repmgr.repmgrd.default

@@ -0,0 +1,14 @@
+#!/bin/sh
+# default settings for repmgrd. This file is source by /bin/sh from
+# /etc/init.d/repmgrd
+
+# Options for repmgrd
+REPMGRD_OPTS=""
+
+# repmgrd binary
+REPMGR_BIN="/usr/bin/repmgr"
+
+# pid file
+REPMGR_PIDFILE="/var/run/repmgrd.pid"
+
+

debian/repmgr.repmgrd.init

@@ -0,0 +1,48 @@
+#!/bin/sh
+### BEGIN INIT INFO
+# Provides:          repmgrd
+# Required-Start:    $local_fs $remote_fs $network $syslog $postgresql
+# Required-Stop:     $local_fs $remote_fs $network $syslog $postgresql
+# Should-Start:      $syslog $postgresql
+# Should-Start:      $syslog $postgresql
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: Start/stop repmgrd
+### END INIT INFO
+
+set -e
+
+if test -f /etc/default/repmgrd; then
+    . /etc/default/repmgrd
+fi
+
+if [ -z "$REPMGRD_BIN" ]; then
+    REPMGRD_BIN="/usr/bin/repmgrd"
+fi
+
+if [ -z "$REPMGRD_PIDFILE" ]; then
+    REPMGRD_PIDFILE="/var/run/repmgrd.pid"
+fi
+
+test -x $REPMGRD_BIN || exit 0
+
+case "$1" in
+    start)
+        start-stop-daemon --start --quiet --make-pidfile --pidfile $REPMGRD_PIDFILE --exec $REPMGRD_BIN $REPMGRD_OPTS
+        ;;
+
+    stop)
+        start-stop-daemon --stop --oknodo --quiet --pidfile $REPMGRD_PIDFILE
+        ;;
+
+    restart)
+        $0 stop && $0 start || exit 1
+        ;;
+
+    *)
+        echo "Usage: $0 {start|stop|restart}"
+        exit 1
+        ;;
+esac
+
+exit 0

log.c

@@ -25,9 +25,11 @@
 
 #ifdef HAVE_SYSLOG
 #include <syslog.h>
-#include <stdarg.h>
 #endif
 
+#include <stdarg.h>
+#include <time.h>
+
 #include "log.h"
 
 #define DEFAULT_IDENT "repmgr"
@@ -37,6 +39,29 @@
 
 /* #define REPMGR_DEBUG */
 
+void stderr_log_with_level(const char *level_name, int level, const char *fmt, ...) {
+	size_t len = strlen(fmt);
+	char fmt1[len + 150];
+	time_t t;
+	struct tm *tm;
+	char buff[100];
+	va_list ap;
+
+	if(log_level >= level) {
+		time(&t);
+		tm = localtime(&t);
+
+		va_start(ap, fmt);
+
+		strftime(buff, 100, "[%Y-%m-%d %H:%M:%S]", tm);
+		snprintf(fmt1, len + 150, "%s [%s] %s", buff, level_name, fmt);
+		vfprintf(stderr, fmt1, ap);
+
+		va_end(ap);
+	}
+}
+
+
 static int detect_log_level(const char* level);
 static int detect_log_facility(const char* facility);
 

log.h

@@ -25,15 +25,17 @@
 #define REPMGR_SYSLOG 1
 #define REPMGR_STDERR 2
 
+void stderr_log_with_level(const char *level_name, int level, const char *fmt, ...);
+
 /* Standard error logging */
-#define stderr_log_debug(...) if (log_level >= LOG_DEBUG) fprintf(stderr, __VA_ARGS__)
+#define stderr_log_debug(...) stderr_log_with_level("DEBUG", LOG_DEBUG, __VA_ARGS__)
-#define stderr_log_info(...) if (log_level >= LOG_INFO) fprintf(stderr, __VA_ARGS__)
+#define stderr_log_info(...)  stderr_log_with_level("INFO", LOG_INFO, __VA_ARGS__)
-#define stderr_log_notice(...) if (log_level >= LOG_NOTICE) fprintf(stderr, __VA_ARGS__)
+#define stderr_log_notice(...) stderr_log_with_level("NOTICE", LOG_NOTICE, __VA_ARGS__)
-#define stderr_log_warning(...) if (log_level >= LOG_WARNING) fprintf(stderr, __VA_ARGS__)
+#define stderr_log_warning(...) stderr_log_with_level("WARNING", LOG_WARNING, __VA_ARGS__)
-#define stderr_log_err(...) if (log_level >= LOG_ERR) fprintf(stderr, __VA_ARGS__)
+#define stderr_log_err(...) stderr_log_with_level("ERROR", LOG_ERR, __VA_ARGS__)
-#define stderr_log_crit(...) if (log_level >= LOG_CRIT) fprintf(stderr, __VA_ARGS__)
+#define stderr_log_crit(...) stderr_log_with_level("CRITICAL", LOG_CRIT, __VA_ARGS__)
-#define stderr_log_alert(...) if (log_level >= LOG_ALERT) fprintf(stderr, __VA_ARGS__)
+#define stderr_log_alert(...) stderr_log_with_level("ALERT", LOG_ALERT, __VA_ARGS__)
-#define stderr_log_emerg(...) if (log_level >= LOG_EMERG) fprintf(stderr, __VA_ARGS__)
+#define stderr_log_emerg(...) stderr_log_with_level("EMERGENCY", LOG_EMERG, __VA_ARGS__)
 
 #ifdef HAVE_SYSLOG
 

repmgr.c

@@ -30,6 +30,7 @@
 
 #include <stdio.h>
 #include <stdlib.h>
+#include <sys/wait.h>
 #include <time.h>
 #include <unistd.h>
 
@@ -84,8 +85,8 @@ bool need_a_node = true;
 bool require_password = false;
 
 /* Initialization of runtime options */
-t_runtime_options runtime_options = { "", "", "", "", "", "", DEFAULT_WAL_KEEP_SEGMENTS, false, false, false, "", "", 0 };
+t_runtime_options runtime_options = { "", "", "", "", "", "", DEFAULT_WAL_KEEP_SEGMENTS, false, false, false, false, "", "", 0 };
-t_configuration_options options = { "", -1, "", MANUAL_FAILOVER, -1, "", "", "", "", "", "", -1 };
+t_configuration_options options = { "", -1, "", MANUAL_FAILOVER, -1, "", "", "", "", "", "", "", -1 };
 
 static char		*server_mode = NULL;
 static char		*server_cmd = NULL;
@@ -106,6 +107,7 @@ main(int argc, char **argv)
 		{"wal-keep-segments", required_argument, NULL, 'w'},
 		{"keep-history", required_argument, NULL, 'k'},
 		{"force", no_argument, NULL, 'F'},
+		{"wait", no_argument, NULL, 'W'},
 		{"ignore-rsync-warning", no_argument, NULL, 'I'},
 		{"verbose", no_argument, NULL, 'v'},
 		{NULL, 0, NULL, 0}
@@ -132,7 +134,7 @@ main(int argc, char **argv)
 	}
 
 
-	while ((c = getopt_long(argc, argv, "d:h:p:U:D:l:f:R:w:k:F:I:v", long_options,
+	while ((c = getopt_long(argc, argv, "d:h:p:U:D:l:f:R:w:k:FWIv", long_options,
 	                        &optindex)) != -1)
 	{
 		switch (c)
@@ -176,6 +178,9 @@ main(int argc, char **argv)
 		case 'F':
 			runtime_options.force = true;
 			break;
+		case 'W':
+			runtime_options.wait_for_master = true;
+			break;
 		case 'I':
 			runtime_options.ignore_rsync_warn = true;
 			break;
@@ -269,7 +274,7 @@ main(int argc, char **argv)
 		break;
 	default:
 		log_err(_("%s: too many command-line arguments (first extra is \"%s\")\n"),
-		        progname, argv[optind + 1]);
+		        progname, argv[optind]);
 		usage();
 		exit(ERR_BAD_CONFIG);
 	}
@@ -391,7 +396,7 @@ do_cluster_show(void)
 
 	if (PQresultStatus(res) != PGRES_TUPLES_OK)
 	{
-		log_err(_("Can't get nodes informations, have you regitered them?\n%s\n"), PQerrorMessage(conn));
+		log_err(_("Can't get nodes information, have you registered them?\n%s\n"), PQerrorMessage(conn));
 		PQclear(res);
 		PQfinish(conn);
 		exit(ERR_BAD_CONFIG);
@@ -837,7 +842,7 @@ do_standby_clone(void)
 		log_err(_("%s needs parameter 'wal_level' to be set to 'hot_standby'\n"), progname);
 		exit(ERR_BAD_CONFIG);
 	}
-	if (!guc_setted(conn, "wal_keep_segments", ">=", runtime_options.wal_keep_segments))
+	if (!guc_setted_typed(conn, "wal_keep_segments", ">=", runtime_options.wal_keep_segments, "integer"))
 	{
 		PQfinish(conn);
 		log_err(_("%s needs parameter 'wal_keep_segments' to be set to %s or greater (see the '-w' option or edit the postgresql.conf of the PostgreSQL master.)\n"), progname, runtime_options.wal_keep_segments);
@@ -947,7 +952,7 @@ do_standby_clone(void)
 	}
 	PQclear(res);
 
-	log_info(_("Succesfully connected to primary. Current installation size is %s\n"), get_cluster_size(conn));
+	log_info(_("Successfully connected to primary. Current installation size is %s\n"), get_cluster_size(conn));
 
 	/*
 	 * XXX  master_xlog_directory should be discovered from master configuration
@@ -1377,10 +1382,19 @@ do_standby_follow(void)
 		exit(ERR_BAD_CONFIG);
 	}
 
-	/* we also need to check if there is any master in the cluster */
+	/*
-	log_info(_("%s connecting to master database\n"), progname);
+	 * we also need to check if there is any master in the cluster
+	 * or wait for one to appear if we have set the wait option
+	 */
+	log_info(_("%s discovering new master...\n"), progname);
+
+	do
+	{
 		master_conn = getMasterConnection(conn, repmgr_schema,
 		                                  options.cluster_name, &master_id,(char *) &master_conninfo);
+	}
+	while (master_conn == NULL && runtime_options.wait_for_master);
+
 	if (master_conn == NULL)
 	{
 		log_err(_("There isn't a master to follow in this cluster\n"));
@@ -1521,7 +1535,7 @@ do_witness_create(void)
 		exit(ERR_BAD_CONFIG);
 	}
 
-	log_info(_("Succesfully connected to primary.\n"));
+	log_info(_("Successfully connected to primary.\n"));
 
 	r = test_ssh_connection(runtime_options.host, runtime_options.remote_user);
 	if (r != 0)
@@ -1658,7 +1672,7 @@ do_witness_create(void)
 	PQfinish(masterconn);
 	PQfinish(witnessconn);
 
-	log_notice(_("Configuration has been succesfully copied to the witness\n"));
+	log_notice(_("Configuration has been successfully copied to the witness\n"));
 }
 
 
@@ -1699,6 +1713,7 @@ help(const char *progname)
 	printf(_("	-I, --ignore-rsync-warning ignore rsync partial transfer warning\n"));
 	printf(_("  -k, --keep-history=VALUE   keeps indicated number of days of history\n"));
 	printf(_("	-F, --force				   force potentially dangerous operations to happen\n"));
+	printf(_("	-W, --wait				   wait for a master to appear"));
 
 	printf(_("\n%s performs some tasks like clone a node, promote it "), progname);
 	printf(_("or making follow another node and then exits.\n"));
@@ -1709,7 +1724,7 @@ help(const char *progname)
 	printf(_(" standby promote		 - allows manual promotion of a specific standby into a "));
 	printf(_("new master in the event of a failover\n"));
 	printf(_(" standby follow		 - allows the standby to re-point itself to a new master\n"));
-	printf(_(" cluster show            - print node informations\n"));
+	printf(_(" cluster show            - print node information\n"));
 	printf(_(" cluster cleanup         - cleans monitor's history\n"));
 }
 
@@ -1762,11 +1777,18 @@ test_ssh_connection(char *host, char *remote_user)
 	char script[MAXLEN];
 	int	 r;
 
+	/* On some OS, true is located in a different place than in Linux */
+#ifdef __FreeBSD__
+#define TRUEBIN_PATH "/usr/bin/true"
+#else
+#define TRUEBIN_PATH "/bin/true"
+#endif
+
 	/* Check if we have ssh connectivity to host before trying to rsync */
 	if (!remote_user[0])
-		maxlen_snprintf(script, "ssh -o Batchmode=yes %s /bin/true", host);
+		maxlen_snprintf(script, "ssh -o Batchmode=yes %s %s %s", options.ssh_options, host, TRUEBIN_PATH);
 	else
-		maxlen_snprintf(script, "ssh -o Batchmode=yes %s -l %s /bin/true", host, remote_user);
+		maxlen_snprintf(script, "ssh -o Batchmode=yes %s %s -l %s %s", options.ssh_options, host, remote_user, TRUEBIN_PATH);
 
 	log_debug(_("command is: %s"), script);
 	r = system(script);
@@ -1805,7 +1827,7 @@ copy_remote_files(char *host, char *remote_user, char *remote_path,
 
 	if (is_directory)
 	{
-		strcat(rsync_flags, " --exclude=pg_xlog* --exclude=pg_control --exclude=*.pid");
+		strcat(rsync_flags, " --exclude=pg_xlog* --exclude=pg_log* --exclude=pg_control --exclude=*.pid");
 		maxlen_snprintf(script, "rsync %s %s:%s/* %s",
 		                rsync_flags, host_string, remote_path, local_path);
 	}
@@ -2154,24 +2176,29 @@ write_primary_conninfo(char* line)
 
 	/* Environment variable for password (UGLY, please use .pgpass!) */
 	const char *password = getenv("PGPASSWORD");
-	if (password != NULL) {
+	if (password != NULL)
+	{
 		maxlen_snprintf(password_buf, " password=%s", password);
 	}
-	else if (require_password) {
+	else if (require_password)
+	{
 		log_err(_("%s: PGPASSWORD not set, but having one is required\n"),
 		        progname);
 		exit(ERR_BAD_PASSWORD);
 	}
 
-	if (runtime_options.host[0]) {
+	if (runtime_options.host[0])
+	{
 		maxlen_snprintf(host_buf, " host=%s", runtime_options.host);
 	}
 
-	if (runtime_options.username[0]) {
+	if (runtime_options.username[0])
+	{
 		maxlen_snprintf(user_buf, " user=%s", runtime_options.username);
 	}
 
-	if (options.node_name[0]) {
+	if (options.node_name[0])
+	{
 		maxlen_snprintf(appname_buf, " application_name=%s", options.node_name);
 	}
 

repmgr.conf.sample

@@ -11,7 +11,8 @@ node_name=standby2
 
 # Connection information
 conninfo='host=192.168.204.104'
-rsync_options=--archive --checksum --compress --progress --rsh=ssh
+rsync_options=--archive --checksum --compress --progress --rsh="ssh -o \"StrictHostKeyChecking no\""
+ssh_options=-o "StrictHostKeyChecking no"
 
 # How many seconds we wait for master response before declaring master failure
 master_response_timeout=60
@@ -23,8 +24,8 @@ reconnect_interval=10
 # Autofailover options
 failover=automatic
 priority=-1
-promote_command='repmgr promote'
+promote_command='repmgr standby promote -f /path/to/repmgr.conf'
-follow_command='repmgr follow'
+follow_command='repmgr standby follow -f /path/to/repmgr.conf -W'
 
 # Log level: possible values are DEBUG, INFO, NOTICE, WARNING, ERR, ALERT, CRIT or EMERG
 # Default: NOTICE

repmgr.h

@@ -59,6 +59,7 @@ typedef struct
 	char wal_keep_segments[MAXLEN];
 	bool verbose;
 	bool force;
+	bool wait_for_master;
 	bool ignore_rsync_warn;
 
 	char masterport[MAXLEN];

repmgrd.c

@@ -32,9 +32,39 @@
 #include "strutil.h"
 #include "version.h"
 
+/* PostgreSQL's headers needed to export some functionality */
 #include "access/xlogdefs.h"
 #include "libpq/pqsignal.h"
 
+/*
+ * we do not export InvalidXLogRecPtr so we need to define it
+ * but since 9.3 it will be defined in xlogdefs.h which we include
+ * so better to ask if it's defined to be future proof
+ */
+#ifndef InvalidXLogRecPtr
+const XLogRecPtr InvalidXLogRecPtr = {0, 0};
+#endif
+
+#if PG_VERSION_NUM >= 90300
+    #define XLAssign(a, b) \
+        a = b
+
+    #define XLAssignValue(a, xlogid, xrecoff) \
+        a = xrecoff
+
+    #define XLByteLT(a, b) \
+        (a < b)
+
+#else
+    #define XLAssign(a, b) \
+        a.xlogid  = b.xlogid; \
+        a.xrecoff = b.xrecoff
+
+    #define XLAssignValue(a, uxlogid, uxrecoff) \
+        a.xlogid = uxlogid; \
+        a.xrecoff = uxrecoff
+#endif
+
 /*
  * Struct to keep info about the nodes, used in the voting process in
  * do_failover()
@@ -42,8 +72,11 @@
 typedef struct nodeInfo
 {
 	int nodeId;
+	char conninfostr[MAXLEN];
 	XLogRecPtr xlog_location;
 	bool is_ready;
+	bool is_visible;
+	bool is_witness;
 } nodeInfo;
 
 
@@ -68,6 +101,8 @@ bool	verbose = false;
 bool	monitoring_history = false;
 char	repmgr_schema[MAXLEN];
 
+bool	failover_done = false;
+
 /*
  * should initialize with {0} to be ANSI complaint ? but this raises
  * error with gcc -Wall
@@ -100,7 +135,7 @@ static void setup_event_handlers(void);
 
 #define CloseConnections()	\
 	if (PQisBusy(primaryConn) == 1) \
-		CancelQuery(primaryConn, local_options.master_response_timeout); \
+		(void) CancelQuery(primaryConn, local_options.master_response_timeout); \
 	if (myLocalConn != NULL) \
 		PQfinish(myLocalConn);	\
 	if (primaryConn != NULL && primaryConn != myLocalConn) \
@@ -190,6 +225,16 @@ main(int argc, char **argv)
 		exit(ERR_BAD_CONFIG);
 	}
 
+
+	/*
+	 * MAIN LOOP
+	 * This loops cicles once per failover and at startup	
+	 * Requisites:
+	 *   - myLocalConn needs to be already setted with an active connection
+	 *   - no master connection
+ 	 */ 
+	do
+	{
 		/*
 		 * Set my server mode, establish a connection to primary
 		 * and start monitor
@@ -220,13 +265,15 @@ main(int argc, char **argv)
 				}
 	
 				log_info(_("%s Starting continuous primary connection check\n"), progname);
+	
 				/* Check that primary is still alive, and standbies are sending info */
+	
 				/*
 				 * Every SLEEP_MONITOR seconds, do master checks
 				 * XXX
 				 * Check that standbies are sending info
 				 */
-		for (;;)
+				do
 				{
 					if (CheckPrimaryConnection())
 					{
@@ -256,7 +303,7 @@ main(int argc, char **argv)
 						}
 						got_SIGHUP = false;
 					}
-		}
+				} while (!failover_done);
 				break;
 			case WITNESS_MODE:
 			case STANDBY_MODE:
@@ -294,7 +341,7 @@ main(int argc, char **argv)
 					log_info(_("%s Starting continuous standby node monitoring\n"), progname);
 				}
 	
-		for (;;)
+				do
 				{
 					if (myLocalMode == WITNESS_MODE)
 						WitnessMonitor();
@@ -313,12 +360,16 @@ main(int argc, char **argv)
 						}
 						got_SIGHUP = false;
 					}
-		}
+				} while (!failover_done);
 				break;
 			default:
 				log_err(_("%s: Unrecognized mode for node %d\n"), progname, local_options.node);
 		}
 
+		failover_done = false;
+
+	} while (true);
+
 	/* Prevent a double-free */
 	if (primaryConn == myLocalConn)
 		myLocalConn = NULL;
@@ -365,7 +416,8 @@ WitnessMonitor(void)
 	 * Cancel any query that is still being executed,
 	 * so i can insert the current record
 	 */
-	CancelQuery(primaryConn, local_options.master_response_timeout);
+	if (!CancelQuery(primaryConn, local_options.master_response_timeout))
+		return;
 	if (wait_connection_availability(primaryConn, local_options.master_response_timeout) != 1)
 		return;
 
@@ -467,6 +519,7 @@ StandbyMonitor(void)
 			 * a new primaryConn
 			 */
 			do_failover();
+			return;
 		}
 	}
 
@@ -486,7 +539,8 @@ StandbyMonitor(void)
 	 * Cancel any query that is still being executed,
 	 * so i can insert the current record
 	 */
-	CancelQuery(primaryConn, local_options.master_response_timeout);
+	if (!CancelQuery(primaryConn, local_options.master_response_timeout))
+		return;
 	if (wait_connection_availability(primaryConn, local_options.master_response_timeout) != 1)
 		return;
 
@@ -557,22 +611,22 @@ StandbyMonitor(void)
 static void
 do_failover(void)
 {
-	PGresult *res1;
+	PGresult *res;
-	PGresult *res2;
 	char 	sqlquery[8192];
 
 	int		total_nodes = 0;
 	int		visible_nodes = 0;
+	int     ready_nodes = 0;
+
 	bool	find_best = false;
 
 	int		i;
 	int		r;
 
-	int 	node;
+	uint32  uxlogid;
-	char	nodeConninfo[MAXLEN];
+	uint32  uxrecoff;
+	XLogRecPtr xlog_recptr;
 
-	unsigned int uxlogid;
-	unsigned int uxrecoff;
 	char last_wal_standby_applied[MAXLEN];
 
 	PGconn	*nodeConn = NULL;
@@ -582,91 +636,59 @@ do_failover(void)
 	 * which seems to be large enough for most scenarios
 	 */
 	nodeInfo nodes[50];
-	nodeInfo best_candidate;
 
-	/* first we get info about this node, and update shared memory */
+	/* initialize to keep compiler quiet */
-	sprintf(sqlquery, "SELECT pg_last_xlog_replay_location()");
+	nodeInfo best_candidate = {-1, "", InvalidXLogRecPtr, false, false, false};
-	res1 = PQexec(myLocalConn, sqlquery);
-	if (PQresultStatus(res1) != PGRES_TUPLES_OK)
-	{
-		log_err(_("PQexec failed: %s.\nReport an invalid value to not be considered as new primary and exit.\n"), PQerrorMessage(myLocalConn));
-		PQclear(res1);
-		sprintf(last_wal_standby_applied, "'%X/%X'", 0, 0);
-		update_shared_memory(last_wal_standby_applied);
-		exit(ERR_DB_QUERY);
-	}
-
-	/* write last location in shared memory */
-	update_shared_memory(PQgetvalue(res1, 0, 0));
-
-	/*
-	 * we sleep the monitor time + one second
-	 * we bet it should be enough for other repmgrd to update their own data
-	 */
-	sleep(SLEEP_MONITOR + 1);
 
 	/* get a list of standby nodes, including myself */
-	sprintf(sqlquery, "SELECT id, conninfo "
+	sprintf(sqlquery, "SELECT id, conninfo, witness "
 	        "  FROM %s.repl_nodes "
-	        " WHERE id <> %d "
+	        " WHERE cluster = '%s' "
-	        "   AND cluster = '%s' "
+	        " ORDER BY priority, id ",
-	        " ORDER BY priority ",
+	        repmgr_schema, local_options.cluster_name);
-	        repmgr_schema, primary_options.node, local_options.cluster_name);
 
-	res1 = PQexec(myLocalConn, sqlquery);
+	res = PQexec(myLocalConn, sqlquery);
-	if (PQresultStatus(res1) != PGRES_TUPLES_OK)
+	if (PQresultStatus(res) != PGRES_TUPLES_OK)
 	{
-		log_err(_("Can't get nodes info: %s\n"), PQerrorMessage(myLocalConn));
+		log_err(_("Can't get nodes' info: %s\n"), PQerrorMessage(myLocalConn));
-		PQclear(res1);
+		PQclear(res);
 		PQfinish(myLocalConn);
 		exit(ERR_DB_QUERY);
 	}
 
-	/* ask for the locations */
+	/*
-	for (i = 0; i < PQntuples(res1); i++)
+	 * total nodes that are registered
+	 */
+	total_nodes = PQntuples(res);
+	log_debug(_("%s: there are %d nodes registered\n"), progname, total_nodes);
+
+	/* Build an array with the nodes and indicate which ones are visible and ready */
+	for (i = 0; i < total_nodes; i++)
 	{
-		node = atoi(PQgetvalue(res1, i, 0));
+		nodes[i].nodeId = atoi(PQgetvalue(res, i, 0));
+		strncpy(nodes[i].conninfostr, PQgetvalue(res, i, 1), MAXLEN);
+		nodes[i].is_witness = (strcmp(PQgetvalue(res, i, 2), "t") == 0) ? true : false;
+
 		/* Initialize on false so if we can't reach this node we know that later */
+		nodes[i].is_visible = false;
 		nodes[i].is_ready = false;
-		strncpy(nodeConninfo, PQgetvalue(res1, i, 1), MAXLEN);
+		XLAssignValue(nodes[i].xlog_location, 0, 0);
-		nodeConn = establishDBConnection(nodeConninfo, false);
+
+		log_debug(_("%s: node=%d conninfo=\"%s\" witness=%s\n"), 
+					progname, nodes[i].nodeId, nodes[i].conninfostr, (nodes[i].is_witness) ? "true" : "false");
+
+		nodeConn = establishDBConnection(nodes[i].conninfostr, false);
 		/* if we can't see the node just skip it */
 		if (PQstatus(nodeConn) != CONNECTION_OK)
 			continue;
 
-		sqlquery_snprintf(sqlquery, "SELECT %s.repmgr_get_last_standby_location()", repmgr_schema);
-		res2 = PQexec(nodeConn, sqlquery);
-		if (PQresultStatus(res2) != PGRES_TUPLES_OK)
-		{
-			log_info(_("Can't get node's last standby location: %s\n"), PQerrorMessage(nodeConn));
-			log_info(_("Connection details: %s\n"), nodeConninfo);
-			PQclear(res2);
-			PQfinish(nodeConn);
-			continue;
-		}
-
 		visible_nodes++;
-
+		nodes[i].is_visible = true;
-		if (sscanf(PQgetvalue(res2, 0, 0), "%X/%X", &uxlogid, &uxrecoff) != 2)
-			log_info(_("could not parse transaction log location \"%s\"\n"), PQgetvalue(res2, 0, 0));
-
-		nodes[i].nodeId = node;
-		nodes[i].xlog_location.xlogid = uxlogid;
-		nodes[i].xlog_location.xrecoff = uxrecoff;
-		nodes[i].is_ready = true;
-
-		PQclear(res2);
 		PQfinish(nodeConn);
 	}
-	PQclear(res1);
+	PQclear(res);
-	/* Close the connection to this server */
-	PQfinish(myLocalConn);
 
-	/*
+	log_debug(_("Total nodes counted: registered=%d, visible=%d\n"), total_nodes, visible_nodes);
-	 * total nodes that are registered, include master which is a node but was
-	 * not counted because it's not a standby
-	 */
-	total_nodes = i + 1;
 
 	/*
 	 * am i on the group that should keep alive?
@@ -680,41 +702,201 @@ do_failover(void)
 		exit(ERR_FAILOVER_FAIL);
 	}
 
+	/* Query all the nodes to determine which ones are ready */ 
+	for (i = 0; i < total_nodes; i++)
+	{
+		/* if the node is not visible, skip it */
+		if (!nodes[i].is_visible)
+			continue;
+
+		if (nodes[i].is_witness)
+			continue;
+
+		nodeConn = establishDBConnection(nodes[i].conninfostr, false);
+		/* XXX 
+		 * This shouldn't happen, if this happens it means this is a major problem
+		 * maybe network outages? anyway, is better for a human to react 
+		 */
+		if (PQstatus(nodeConn) != CONNECTION_OK)
+		{
+			log_err(_("It seems new problems are arising, manual intervention is needed\n"));
+			exit(ERR_FAILOVER_FAIL);
+		}
+
+		sqlquery_snprintf(sqlquery, "SELECT pg_last_xlog_receive_location()");
+		res = PQexec(nodeConn, sqlquery);
+		if (PQresultStatus(res) != PGRES_TUPLES_OK)
+		{
+			log_info(_("Can't get node's last standby location: %s\n"), PQerrorMessage(nodeConn));
+			log_info(_("Connection details: %s\n"), nodes[i].conninfostr);
+			PQclear(res);
+			PQfinish(nodeConn);
+			exit(ERR_FAILOVER_FAIL);
+		}
+
+		if (sscanf(PQgetvalue(res, 0, 0), "%X/%X", &uxlogid, &uxrecoff) != 2)
+			log_info(_("could not parse transaction log location \"%s\"\n"), PQgetvalue(res, 0, 0));
+
+		log_debug("XLog position of node %d: log id=%u (%X), offset=%u (%X)\n", 
+					nodes[i].nodeId, uxlogid, uxlogid, uxrecoff, uxrecoff);
+
+		/* If position is 0/0, error */
+		if (uxlogid == 0 && uxrecoff == 0)
+		{
+			PQclear(res);
+			PQfinish(nodeConn);
+			log_info(_("InvalidXLogRecPtr detected in a standby\n"));
+			exit(ERR_FAILOVER_FAIL);
+		}
+
+		XLAssignValue(nodes[i].xlog_location, uxlogid, uxrecoff);
+
+		PQclear(res);
+		PQfinish(nodeConn);
+	}
+
+	/* last we get info about this node, and update shared memory */
+	sprintf(sqlquery, "SELECT pg_last_xlog_receive_location()");
+	res = PQexec(myLocalConn, sqlquery);
+	if (PQresultStatus(res) != PGRES_TUPLES_OK)
+	{
+		log_err(_("PQexec failed: %s.\nReport an invalid value to not be considered as new primary and exit.\n"), PQerrorMessage(myLocalConn));
+		PQfinish(myLocalConn);
+		PQclear(res);
+		sprintf(last_wal_standby_applied, "'%X/%X'", 0, 0);
+		update_shared_memory(last_wal_standby_applied);
+		exit(ERR_DB_QUERY);
+	}	
+
+	/* write last location in shared memory */
+	update_shared_memory(PQgetvalue(res, 0, 0));
+	PQclear(res);
+
+	for (i = 0; i < total_nodes; i++)
+	{
+		while (!nodes[i].is_ready)
+		{
+			/* 
+        	 * the witness will always be masked as ready if it's still 
+        	 * not marked that way and avoid a useless query
+        	 */
+			if (nodes[i].is_witness)
+			{
+				if (!nodes[i].is_ready)
+				{
+					nodes[i].is_ready = true;
+					ready_nodes++;	
+				}
+				break;
+			}
+
+			/* if the node is not visible, skip it */
+			if (!nodes[i].is_visible)
+				break;
+
+			/* if the node is ready there is nothing to check, skip it too */
+			if (nodes[i].is_ready)
+				break;
+
+            nodeConn = establishDBConnection(nodes[i].conninfostr, false);
+            /* XXX 
+             * This shouldn't happen, if this happens it means this is a major problem
+             * maybe network outages? anyway, is better for a human to react 
+             */
+            if (PQstatus(nodeConn) != CONNECTION_OK)
+            {
+				/* XXX */
+                log_info(_("At this point, it could be some race conditions that are acceptable, assume the node is restarting and starting failover procedure\n"));
+				break;
+            }
+
+			sqlquery_snprintf(sqlquery, "SELECT %s.repmgr_get_last_standby_location()", repmgr_schema);
+			res = PQexec(nodeConn, sqlquery);
+            if (PQresultStatus(res) != PGRES_TUPLES_OK)
+            {
+                log_err(_("PQexec failed: %s.\nReport an invalid value to not be considered as new primary and exit.\n"), PQerrorMessage(nodeConn));
+                PQclear(res);
+                PQfinish(nodeConn);
+                exit(ERR_DB_QUERY);
+			}
+
+            if (sscanf(PQgetvalue(res, 0, 0), "%X/%X", &uxlogid, &uxrecoff) != 2)
+                log_info(_("could not parse transaction log location \"%s\"\n"), PQgetvalue(res, 0, 0));
+
+            PQclear(res);
+            PQfinish(nodeConn);
+            /* If position is 0/0, keep checking */
+            if (uxlogid == 0 && uxrecoff == 0)
+                continue;
+
+			XLAssignValue(xlog_recptr, uxlogid, uxrecoff);
+
+			if (XLByteLT(nodes[i].xlog_location, xlog_recptr))
+			{
+				XLAssignValue(nodes[i].xlog_location, uxlogid, uxrecoff);
+			}
+
+            log_debug("Last XLog position of node %d: log id=%u (%X), offset=%u (%X)\n",
+                        nodes[i].nodeId, uxlogid,  uxlogid, 
+										 uxrecoff, uxrecoff);
+
+			ready_nodes++;	
+			nodes[i].is_ready = true;
+		}
+	}
+
+	/* Close the connection to this server */
+	PQfinish(myLocalConn);
+
 	/*
 	 * determine which one is the best candidate to promote to primary
 	 */
-	for (i = 0; i < total_nodes - 1; i++)
+	for (i = 0; i < total_nodes; i++)
 	{
-		if (!nodes[i].is_ready)
+		/* witness is never a good candidate */
+		if (nodes[i].is_witness)
 			continue;
-		else if (!find_best)
+
+		if (!nodes[i].is_ready || !nodes[i].is_visible)
+			continue;
+
+		if (!find_best)
 		{
 			/* start with the first ready node, and then move on to the next one */
 			best_candidate.nodeId                = nodes[i].nodeId;
-			best_candidate.xlog_location.xlogid  = nodes[i].xlog_location.xlogid;
+			XLAssign(best_candidate.xlog_location, nodes[i].xlog_location);
-			best_candidate.xlog_location.xrecoff = nodes[i].xlog_location.xrecoff;
 			best_candidate.is_ready              = nodes[i].is_ready;
+			best_candidate.is_witness            = nodes[i].is_witness;
 			find_best = true;
 		}
 
-		/* we use the macros provided by xlogdefs.h to compare XLogPtr */
+		/* we use the macros provided by xlogdefs.h to compare XLogRecPtr */
 		/*
 		 * Nodes are retrieved ordered by priority, so if the current
-		 * best candidate is lower or equal to the next node's wal location
+		 * best candidate is lower than the next node's wal location
 		 * then assign next node as the new best candidate.
 		 */
-		if (XLByteLE(best_candidate.xlog_location, nodes[i].xlog_location))
+		if (XLByteLT(best_candidate.xlog_location, nodes[i].xlog_location))
 		{
 			best_candidate.nodeId                = nodes[i].nodeId;
-			best_candidate.xlog_location.xlogid  = nodes[i].xlog_location.xlogid;
+			XLAssign(best_candidate.xlog_location, nodes[i].xlog_location);
-			best_candidate.xlog_location.xrecoff = nodes[i].xlog_location.xrecoff;
 			best_candidate.is_ready              = nodes[i].is_ready;
+			best_candidate.is_witness            = nodes[i].is_witness;
 		}
 	}
 
 	/* once we know who is the best candidate, promote it */
 	if (find_best && (best_candidate.nodeId == local_options.node))
 	{
+		if (best_candidate.is_witness)
+		{
+			log_err(_("%s: Node selected as new master is a witness. Can't be promoted.\n"), progname);
+			exit(ERR_FAILOVER_FAIL);
+		}
+
+		/* wait */
+		sleep(5);
+
 		if (verbose)
 			log_info(_("%s: This node is the best candidate to be the new primary, promoting...\n"),
 			         progname);
@@ -728,6 +910,9 @@ do_failover(void)
 	}
 	else if (find_best)
 	{
+		/* wait */
+		sleep(10);
+
 		if (verbose)
 			log_info(_("%s: Node %d is the best candidate to be the new primary, we should follow it...\n"),
 			         progname, best_candidate.nodeId);
@@ -749,6 +934,9 @@ do_failover(void)
 		exit(ERR_FAILOVER_FAIL);
 	}
 
+	/* to force it to re-calculate mode and master node */
+	failover_done = true;
+
 	/* and reconnect to the local database */
 	myLocalConn = establishDBConnection(local_options.conninfo, true);
 }
@@ -781,7 +969,7 @@ CheckPrimaryConnection(void)
 			{
 				log_info(_("%s: Connection to master has been restored.\n"), progname);
 			}
-			break;
+			return true;
 		}
 	}
 	if (!is_pgup(primaryConn, local_options.master_response_timeout))
@@ -937,6 +1125,7 @@ static void
 handle_sigint(SIGNAL_ARGS)
 {
 	CloseConnections();
+	logger_shutdown();
 	exit(1);
 }