Fix oversight in the header of guc_setted_typed()

Patch by: Christian Kruse

.gitignore

@@ -0,0 +1,9 @@
+*~
+*.o
+*.so
+repmgr
+repmgrd
+README.htm*
+README.pdf
+sql/repmgr_funcs.so
+sql/repmgr_funcs.sql

COPYRIGHT

@@ -1,4 +1,4 @@
-Copyright (c) 2009-2010, 2ndQuadrant Limited
+Copyright (c) 2010-2012, 2ndQuadrant Limited
 All rights reserved.
 
 This program is free software: you can redistribute it and/or modify

CREDITS

@@ -0,0 +1,16 @@
+Code and documentation contributors to repmgr include:
+
+Jaime Casanova <jaime@2ndQuadrant.com>
+Simon Riggs <simon@2ndQuadrant.com>
+Greg Smith <greg@2ndQuadrant.com>
+Robert J. Noles <rj@2ndQuadrant.com>
+Gabriele Bartolini <gabriele@2ndQuadrant.com>
+Bas van Oostveen <v.oostveen@gmail.com>
+Hannu Krosing <hannu@2ndQuadrant.com>
+Cédric Villemain <cedric@2ndquadrant.com>
+Charles Duffy <charles@dyfis.net>
+Daniel Farina <daniel@heroku.com>
+Shawn Ellis <shawn.ellis17@gmail.com>
+Jay Taylor <jay@jaytaylor.com>
+Christian Kruse <christian@2ndQuadrant.com>
+Krzysztof Gajdemski <songo@debian.org.pl>

HISTORY

@@ -0,0 +1,64 @@
+2.0beta2 2013-12-19
+        Improve autofailover logic and algorithms (Jaime, Andres)
+        Ignore pg_log when cloning (Jaime)
+        Add timestamps to log line in stderr (Christian)
+        Correctly check wal_keep_segments (Jay Taylor)
+        Add a ssh_options parameter (Jay Taylor)
+
+2.0beta1 2012-07-27
+        Make CLONE command try to make an exact copy including $PGDATA location (Cedric) 
+        Add detection of master failure (Jaime)
+        Add the notion of a witness server (Jaime)
+        Add autofailover capabilities (Jaime)
+        Add a configuration parameter to indicate the script to execute on failover or follow (Jaime)
+        Make the monitoring optional and turned off by default, it can be turned on with --monitoring-history switch (Jaime)
+        Add tunables to specify number of retries to reconnect to master and the time between them (Jaime)
+
+1.2.0	2012-07-27
+		Test ssh connection before trying to rsync (Cédric)
+        Add CLUSTER SHOW command (Carlo)
+        Add CLUSTER CLEANUP command (Jaime)
+        Add function write_primary_conninfo (Marco)
+        Teach repmgr how to get tablespace's location in different pg version (Jaime)
+		Improve version message (Carlo)
+
+1.1.1	2012-04-18
+        Add --ignore-rsync-warning (Cédric)
+        Add strnlen for compatibility with OS X (Greg)
+        Improve performance of the repl_status view (Jaime)
+        Remove last argument from log_err (Jaime, Reported by Jeroen Dekkers)
+        Complete documentation about possible error conditions (Jaime)
+        Document how to clean history (Jaime)
+
+1.1.0   2011-03-09
+        Make options -U, -R and -p not mandatory (Jaime)
+
+1.1.0b1 2011-02-24 
+        Fix missing "--force" option in help (Greg Smith)
+        Correct warning message for wal_keep_segments (Bas van Oostveen)
+        Add Debian build/usage docs (Bas, Hannu Krosing, Cedric Villemain)
+        Add Debian .deb packaging (Hannu)
+        Move configuration data into a structure (Bas, Gabriele Bartolini)
+        Make rsync options configurable (Bas)
+        Add syslog as alternate logging destination (Gabriele)
+        Change from using malloc to static memory allocations (Gabriele)
+        Add debugging messages after every query (Gabriele)
+        Parameterize schema name used for repmgr (Gabriele)
+        Avoid buffer overruns by using snprintf etc. (Gabriele)
+        Fix use of database query after close (Gabriele)
+        Add information about progress during "standby clone" (Gabriele)
+        Fix double free errors in repmgrd (Charles Duffy, Greg)
+        Make repmgr exit with an error code when encountering an error (Charles)
+        Standardize on error return codes, use in repmgrd too (Greg)
+        Add [un]install actions/SQL like most contrib modules (Daniel Farina)
+        Wrap all string construction and produce error on overflow (Daniel)
+        Correct freeing of memory from first_wal_segment (Daniel)
+        Allow creating recovery.conf file with a password (Daniel)
+        Inform when STANDBY CLONE sees an unused config file (Daniel)
+        Use 64-bit computation for WAL apply_lag (Greg)
+        Add info messages for database and general work done (Greg)
+        Map old verbose flag into a useful setting for the new logger (Greg)
+        Document repmgrd startup restrictions and log info about them (Greg)
+
+1.0.0   2010-12-05
+        First public release

Makefile

@@ -1,17 +1,21 @@
 #
 # Makefile
-# Copyright (c) 2ndQuadrant, 2010
+# Copyright (c) 2ndQuadrant, 2010-2012
 
-repmgrd_OBJS = dbutils.o config.o repmgrd.o
-repmgr_OBJS = dbutils.o check_dir.o config.o repmgr.o
+repmgrd_OBJS = dbutils.o config.o repmgrd.o log.o strutil.o
+repmgr_OBJS = dbutils.o check_dir.o config.o repmgr.o log.o strutil.o
+
+DATA = repmgr.sql uninstall_repmgr.sql
 
 PG_CPPFLAGS = -I$(libpq_srcdir)
 PG_LIBS = $(libpq_pgport)
 
 all:  repmgrd repmgr
+	$(MAKE) -C sql
 
 repmgrd: $(repmgrd_OBJS)
 	$(CC) $(CFLAGS) $(repmgrd_OBJS) $(PG_LIBS) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o repmgrd
+	$(MAKE) -C sql
 
 repmgr: $(repmgr_OBJS)
 	$(CC) $(CFLAGS) $(repmgr_OBJS) $(PG_LIBS) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o repmgr
@@ -26,11 +30,35 @@ include $(top_builddir)/src/Makefile.global
 include $(top_srcdir)/contrib/contrib-global.mk
 endif
 
+# XXX: Try to use PROGRAM construct (see pgxs.mk) someday. Right now
+# is overriding pgxs install.
 install:
 	$(INSTALL_PROGRAM) repmgrd$(X) '$(DESTDIR)$(bindir)'
 	$(INSTALL_PROGRAM) repmgr$(X) '$(DESTDIR)$(bindir)'
+	$(MAKE) -C sql install
+
+ifneq (,$(DATA)$(DATA_built))
+	@for file in $(addprefix $(srcdir)/, $(DATA)) $(DATA_built); do \
+	  echo "$(INSTALL_DATA) $$file '$(DESTDIR)$(datadir)/$(datamoduledir)'"; \
+	  $(INSTALL_DATA) $$file '$(DESTDIR)$(datadir)/$(datamoduledir)'; \
+	done
+endif
 
 clean:
 	rm -f *.o
 	rm -f repmgrd
 	rm -f repmgr
+	$(MAKE) -C sql clean
+
+deb: repmgrd repmgr
+	mkdir -p ./debian/usr/bin
+	cp repmgrd repmgr ./debian/usr/bin/
+	mkdir -p ./debian/usr/share/postgresql/9.0/contrib/
+	cp sql/repmgr_funcs.sql ./debian/usr/share/postgresql/9.0/contrib/
+	cp sql/uninstall_repmgr_funcs.sql ./debian/usr/share/postgresql/9.0/contrib/
+	mkdir -p ./debian/usr/lib/postgresql/9.0/lib/
+	cp sql/repmgr_funcs.so ./debian/usr/lib/postgresql/9.0/lib/
+	dpkg-deb --build debian
+	mv debian.deb ../postgresql-repmgr-9.0_1.0.0.deb
+	rm -rf ./debian/usr
+

TODO

@@ -0,0 +1,21 @@
+Known issues in repmgr
+======================
+
+* The check for whether ``wal_keep_segments`` is considered large enough
+  does a string comparison rather than an integer one.  It can give both
+  false positive (setting is large enough but flagged as too small) and
+  false negative (setting is too small but not noted as such) errors.
+
+* When running repmgr against a remote machine, operations that start
+  the database server using the ``pg_ctl`` command may accidentally
+  terminate after their associated ssh session ends.
+
+* After running repmgrd as a regular foreground application, hitting
+  control-C causes the program to crash.
+
+Planned feature improvements
+============================
+
+* Before running ``pg_start_backup()``, a sanity check that there is a
+  a working ssh connection to the destination would help find
+  configuration errors before disturbing the database.

autofailover_quick_setup.rst

@@ -0,0 +1,213 @@
+=====================================================
+ PostgreSQL Automatic Fail-Over - User Documentation
+=====================================================
+
+Automatic Failover
+==================
+
+repmgr allows setups for automatic failover when it detects the failure of the master node.
+Following is a quick setup for this.
+
+Installation
+============
+
+For convenience, we define:
+
+  * node1 is the hostname fully qualified of the Master server, IP 192.168.1.10
+  * node2 is the hostname fully qualified of the Standby server, IP 192.168.1.11
+  * witness is the hostname fully qualified of the server used for witness, IP 192.168.1.12
+
+:Note: It is not recommanded to use name defining status of a server like «masterserver»,
+       this is a name leading to confusion once a failover take place and the Master is
+       now on the «standbyserver».
+
+Summary
+-------
+
+2 PostgreSQL servers are involved in the replication.  Automatic fail-over need
+to vote to decide what server it should promote, thus an odd number is required
+and a witness-repmgrd is installed in a third server where it uses a PostgreSQL
+cluster to communicate with other repmgrd daemons.
+
+1. Install PostgreSQL in all the servers involved (including the server used for
+witness)
+2. Install repmgr in all the servers involved (including the server used for witness)
+3. Configure the Master PostreSQL
+4. Clone the Master to the Standby using "repmgr standby clone" command
+5. Configure repmgr in all the servers involved (including the server used for witness)
+6. Register Master and Standby nodes
+7. Initiate witness server
+8. Start the repmgrd daemons in all nodes
+
+:Note: A complete Hight-Availability design need at least 3 servers to still have
+       a backup node after a first failure.
+
+Install PostgreSQL
+------------------
+
+You can install PostgreSQL using any of the recommended methods. You should ensure
+it's 9.0 or superior.
+
+Install repmgr
+--------------
+
+Install repmgr following the steps in the README.
+
+Configure PostreSQL
+-------------------
+
+Log in node1.
+
+Edit the file postgresql.conf and modify the parameters::
+
+  listen_addresses='*'
+  wal_level = 'hot_standby'
+  archive_mode = on
+  archive_command = 'cd .'	 # we can also use exit 0, anything that 
+                             # just does nothing
+  max_wal_senders = 10
+  wal_keep_segments = 5000   # 80 GB required on pg_xlog
+  hot_standby = on
+  shared_preload_libraries = 'repmgr_funcs'
+
+Edit the file pg_hba.conf and add lines for the replication::
+
+  host     repmgr           repmgr      127.0.0.1/32            trust
+  host     repmgr           repmgr      192.168.1.10/30         trust
+  host     replication      all         192.168.1.10/30         trust
+
+:Note: It is also possible to use a password authentication (md5), .pgpass file
+       should be edited to allow connection between each node.
+
+Create the user and database to manage replication::
+
+  su - postgres
+  createuser -s repmgr
+  createdb -O repmgr repmgr
+  psql -f /usr/share/postgresql/9.0/contrib/repmgr_funcs.sql repmgr
+
+Restart the PostgreSQL server::
+
+  pg_ctl -D $PGDATA restart
+
+And check everything is fine in the server log.
+
+Create the ssh-key for the postgres user and copy it to other servers::
+
+  su - postgres
+  ssh-keygen             # /!\ do not use a passphrase /!\
+  cat ~/.ssh/id_rsa.pub > ~/.ssh/authorized_keys
+  chmod 600 ~/.ssh/authorized_keys
+  exit
+  rsync -avz ~postgres/.ssh/authorized_keys node2:~postgres/.ssh/
+  rsync -avz ~postgres/.ssh/authorized_keys witness:~postgres/.ssh/
+  rsync -avz ~postgres/.ssh/id_rsa* node2:~postgres/.ssh/
+  rsync -avz ~postgres/.ssh/id_rsa* witness:~postgres/.ssh/
+
+Clone Master
+------------
+
+Log in node2.
+
+Clone the node1 (the current Master)::
+
+  su - postgres
+  repmgr -d repmgr -U repmgr -h node1 standby clone 
+
+Start the PostgreSQL server::
+
+  pg_ctl -D $PGDATA start
+
+And check everything is fine in the server log.
+
+Configure repmgr
+----------------
+
+Log in each server and configure repmgr by editing the file
+/etc/repmgr/repmgr.conf::
+
+  cluster=my_cluster
+  node=1
+  node_name=earth
+  conninfo='host=192.168.1.10 dbname=repmgr user=repmgr'
+  master_response_timeout=60
+  reconnect_attempts=6
+  reconnect_interval=10
+  failover=automatic
+  promote_command='promote_command.sh'
+  follow_command='repmgr standby follow -f /etc/repmgr/repmgr.conf'
+
+* *cluster* is the name of the current replication.
+* *node* is the number of the current node (1, 2 or 3 in the current example).
+* *node_name* is an identifier for every node.
+* *conninfo* is used to connect to the local PostgreSQL server (where the configuration file is) from any node. In the witness server configuration it is needed to add a 'port=5499' to the conninfo.
+* *master_response_timeout* is the maximum amount of time we are going to wait before deciding the master has died and start failover procedure.
+* *reconnect_attempts* is the number of times we will try to reconnect to master after a failure has been detected and before start failover procedure.
+* *reconnect_interval* is the amount of time between retries to reconnect to master after a failure has been detected and before start failover procedure.
+* *failover* configure behavior : *manual* or *automatic*.
+* *promote_command* the command executed to do the failover (including the PostgreSQL failover itself). The command must return 0 on success.
+* *follow_command* the command executed to address the current standby to another Master. The command must return 0 on success.
+
+Register Master and Standby
+---------------------------
+
+Log in node1.
+
+Register the node as Master::
+
+  su - postgres
+  repmgr -f /etc/repmgr/repmgr.conf master register
+
+Log in node2.
+
+Register the node as Standby::
+
+  su - postgres
+  repmgr -f /etc/repmgr/repmgr.conf standby register
+
+Initialize witness server
+-------------------------
+
+Log in witness.
+
+Initialize the witness server::
+
+  su - postgres
+  repmgr -d repmgr -U repmgr -h 192.168.1.10 -D $WITNESS_PGDATA -f /etc/repmgr/repmgr.conf witness create 
+
+It needs information to connect to the master to copy the configuration of the cluster, also it needs to know where it should initialize it's own $PGDATA.
+As part of the procees it also ask for the superuser password so it can connect when needed.
+
+Start the repmgrd daemons
+-------------------------
+
+Log in node2 and witness.
+
+  su - postgres
+  repmgrd -f /etc/repmgr/repmgr.conf > /var/log/postgresql/repmgr.log 2>&1
+
+:Note: The Master does not need a repmgrd daemon.
+
+
+Suspend Automatic behavior
+==========================
+
+Edit the repmgr.conf of the node to remove from automatic processing and change::
+
+	failover=manual
+
+Then, signal repmgrd daemon::
+
+	su - postgres
+	kill -HUP `pidoff repmgrd`
+
+TODO : -HUP configuration update is not implemented and it should check its
+	   configuration file  against its configuration in DB, updating
+	   accordingly the SQL conf (especialy the failover manual or auto)
+	   this allow witness-standby and standby-not-promotable features
+	   and simpler usage of the tool ;)
+
+Usage
+=====
+
+The repmgr documentation is in the README file (how to build, options, etc.)

check_dir.c

@@ -1,8 +1,20 @@
 /*
- * check_dir.c
- * Copyright (c) 2ndQuadrant, 2010
+ * check_dir.c - Directories management functions
+ * Copyright (C) 2ndQuadrant, 2010-2012
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  *
- * Directories management functions
  */
 
 #include <sys/stat.h>
@@ -12,11 +24,12 @@
 #include <stdio.h>
 #include <string.h>
 
+/* NB: postgres_fe must be included BEFORE check_dir */
 #include "postgres_fe.h"
 #include "check_dir.h"
 
-
-static int mkdir_p(char *path, mode_t omode);
+#include "strutil.h"
+#include "log.h"
 
 /*
  * make sure the directory either doesn't exist or is empty
@@ -31,63 +44,63 @@ static int mkdir_p(char *path, mode_t omode);
 int
 check_dir(char *dir)
 {
-    DIR        *chkdir;
-    struct 		dirent *file;
-    int         result = 1;
+	DIR        *chkdir;
+	struct 		dirent *file;
+	int         result = 1;
 
-    errno = 0;
+	errno = 0;
 
-    chkdir = opendir(dir);
+	chkdir = opendir(dir);
 
-    if (!chkdir)
-        return (errno == ENOENT) ? 0 : -1;
+	if (!chkdir)
+		return (errno == ENOENT) ? 0 : -1;
 
-    while ((file = readdir(chkdir)) != NULL)
-    {
-        if (strcmp(".", file->d_name) == 0 ||
-            strcmp("..", file->d_name) == 0)
-        {
-            /* skip this and parent directory */
-            continue;
-        }
-        else
-        {
-            result = 2;         /* not empty */
-            break;
-        }
-    }
+	while ((file = readdir(chkdir)) != NULL)
+	{
+		if (strcmp(".", file->d_name) == 0 ||
+		        strcmp("..", file->d_name) == 0)
+		{
+			/* skip this and parent directory */
+			continue;
+		}
+		else
+		{
+			result = 2;			/* not empty */
+			break;
+		}
+	}
 
 #ifdef WIN32
-    /*
-     * This fix is in mingw cvs (runtime/mingwex/dirent.c rev 1.4), but not in
-     * released version
-     */
-    if (GetLastError() == ERROR_NO_MORE_FILES)
-        errno = 0;
+	/*
+	 * This fix is in mingw cvs (runtime/mingwex/dirent.c rev 1.4), but not in
+	 * released version
+	 */
+	if (GetLastError() == ERROR_NO_MORE_FILES)
+		errno = 0;
 #endif
 
-    closedir(chkdir);
+	closedir(chkdir);
 
-    if (errno != 0)
-        return -1;          /* some kind of I/O error? */
+	if (errno != 0)
+		return -1;          /* some kind of I/O error? */
 
 	return result;
 }
 
 
 /*
- * Create directory 
+ * Create directory
  */
 bool
 create_directory(char *dir)
 {
-    if (mkdir_p(dir, 0700) == 0)
-        return true;
+	if (mkdir_p(dir, 0700) == 0)
+		return true;
 
-    fprintf(stderr, _("Could not create directory \"%s\": %s\n"),
-            dir, strerror(errno));
+	log_err(_("Could not create directory \"%s\": %s\n"),
+	        dir, strerror(errno));
 
-    return false;
+	return false;
 }
 
 bool
@@ -99,7 +112,7 @@ set_directory_permissions(char *dir)
 
 
 /* function from initdb.c */
-/* source stolen from FreeBSD /src/bin/mkdir/mkdir.c and adapted */
+/* source adapted from FreeBSD /src/bin/mkdir/mkdir.c */
 
 /*
  * this tries to build all the elements of a path to a directory a la mkdir -p
@@ -109,15 +122,15 @@ set_directory_permissions(char *dir)
  * note that on failure, the path arg has been modified to show the particular
  * directory level we had problems with.
  */
-static int
+int
 mkdir_p(char *path, mode_t omode)
 {
 	struct stat sb;
 	mode_t		numask,
-				oumask;
+	            oumask;
 	int			first,
-				last,
-				retval;
+	            last,
+	            retval;
 	char	   *p;
 
 	p = path;
@@ -136,8 +149,8 @@ mkdir_p(char *path, mode_t omode)
 				return 1;
 		}
 		else if (p[1] == ':' &&
-				 ((p[0] >= 'a' && p[0] <= 'z') ||
-				  (p[0] >= 'A' && p[0] <= 'Z')))
+		         ((p[0] >= 'a' && p[0] <= 'z') ||
+		          (p[0] >= 'A' && p[0] <= 'Z')))
 		{
 			/* local drive */
 			p += 2;
@@ -207,10 +220,88 @@ mkdir_p(char *path, mode_t omode)
 bool
 is_pg_dir(char *dir)
 {
-	char	path[8192];
-	struct stat sb;
+	const size_t buf_sz = 8192;
+	char		 path[buf_sz];
+	struct stat	 sb;
+	int		r;
 
-	sprintf(path, "%s/PG_VERSION", dir);
+	// test pgdata
+	xsnprintf(path, buf_sz, "%s/PG_VERSION", dir);
+	if (stat(path, &sb) == 0)
+		return true;
 
-        return (stat(path, &sb) == 0) ? true : false;
+	// test tablespace dir
+	sprintf(path, "ls %s/PG_*/ -I*", dir);
+	r = system(path);
+	if (r == 0)
+		return true;
+
+	return false;
+}
+
+
+bool
+create_pgdir(char *dir, bool force)
+{
+	bool	pg_dir = false;
+
+	/* Check this directory could be used as a PGDATA dir */
+	switch (check_dir(dir))
+	{
+	case 0:
+		/* dir not there, must create it */
+		log_info(_("creating directory \"%s\"...\n"), dir);
+
+		if (!create_directory(dir))
+		{
+			log_err(_("couldn't create directory \"%s\"...\n"),
+			        dir);
+			exit(ERR_BAD_CONFIG);
+		}
+		break;
+	case 1:
+		/* Present but empty, fix permissions and use it */
+		log_info(_("checking and correcting permissions on existing directory %s ...\n"),
+		         dir);
+
+		if (!set_directory_permissions(dir))
+		{
+			log_err(_("could not change permissions of directory \"%s\": %s\n"),
+			        dir, strerror(errno));
+			exit(ERR_BAD_CONFIG);
+		}
+		break;
+	case 2:
+		/* Present and not empty */
+		log_warning(_("directory \"%s\" exists but is not empty\n"),
+		            dir);
+
+		pg_dir = is_pg_dir(dir);
+
+		/*
+		 * we use force to reduce the time needed to restore a node which
+		 * turn async after a failover or anything else
+		 */
+		if (pg_dir && force)
+		{
+			/* Let it continue */
+			break;
+		}
+		else if (pg_dir && !force)
+		{
+			log_warning(_("\nThis looks like a PostgreSQL directory.\n"
+			              "If you are sure you want to clone here, "
+			              "please check there is no PostgreSQL server "
+			              "running and use the --force option\n"));
+			exit(ERR_BAD_CONFIG);
+		}
+
+		return false;
+	default:
+		/* Trouble accessing directory */
+		log_err(_("could not access directory \"%s\": %s\n"),
+		        dir, strerror(errno));
+		exit(ERR_BAD_CONFIG);
+	}
+	return true;
 }

check_dir.h

@@ -1,10 +1,30 @@
 /*
  * check_dir.h
- * Copyright (c) 2ndQuadrant, 2010
+ * Copyright (c) 2ndQuadrant, 2010-2012
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  *
  */
 
+#ifndef _REPMGR_CHECK_DIR_H_
+#define _REPMGR_CHECK_DIR_H_
+
+int mkdir_p(char *path, mode_t omode);
 int  check_dir(char *dir);
 bool create_directory(char *dir);
 bool set_directory_permissions(char *dir);
 bool is_pg_dir(char *dir);
+bool create_pgdir(char *dir, bool force);
+
+#endif

config.c

@@ -1,80 +1,189 @@
 /*
- * config.c
- * Copyright (c) 2ndQuadrant, 2010
+ * config.c - Functions to parse the config file
+ * Copyright (C) 2ndQuadrant, 2010-2012
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  *
- * Functions to parse the config file
  */
 
+#include "config.h"
+#include "log.h"
+#include "strutil.h"
 #include "repmgr.h"
 
 void
-parse_config(const char *config_file, char *cluster_name, int *node, char *conninfo)
+parse_config(const char *config_file, t_configuration_options *options)
 {
-	char *s, buff[256];
+	char *s, buff[MAXLINELENGTH];
+	char name[MAXLEN];
+	char value[MAXLEN];
+
 	FILE *fp = fopen (config_file, "r");
 
+	/* Initialize */
+	memset(options->cluster_name, 0, sizeof(options->cluster_name));
+	options->node = -1;
+	memset(options->conninfo, 0, sizeof(options->conninfo));
+	options->failover = MANUAL_FAILOVER;
+	options->priority = 0;
+	memset(options->node_name, 0, sizeof(options->node_name));
+	memset(options->promote_command, 0, sizeof(options->promote_command));
+	memset(options->follow_command, 0, sizeof(options->follow_command));
+	memset(options->rsync_options, 0, sizeof(options->rsync_options));
+	memset(options->ssh_options, 0, sizeof(options->ssh_options));
+
+	/* if nothing has been provided defaults to 60 */
+	options->master_response_timeout = 60;
+
+	/* it defaults to 6 retries with a time between retries of 10s */
+	options->reconnect_attempts = 6;
+	options->reconnect_intvl = 10;
+
+	/*
+	 * Since some commands don't require a config file at all, not
+	 * having one isn't necessarily a problem.
+	 */
 	if (fp == NULL)
-    	return;
+	{
+		log_err(_("Did not find the configuration file '%s', continuing\n"), config_file);
+		return;
+	}
 
 	/* Read next line */
 	while ((s = fgets (buff, sizeof buff, fp)) != NULL)
 	{
-		char name[MAXLEN];
-		char value[MAXLEN];
+		/* Skip blank lines and comments */
+		if (buff[0] == '\n' || buff[0] == '#')
+			continue;
 
-    	/* Skip blank lines and comments */
-    	if (buff[0] == '\n' || buff[0] == '#')
-    		continue;
-
-    	/* Parse name/value pair from line */
+		/* Parse name/value pair from line */
 		parse_line(buff, name, value);
 
-    	/* Copy into correct entry in parameters struct */
-    	if (strcmp(name, "cluster") == 0)
-    		strncpy (cluster_name, value, MAXLEN);
-    	else if (strcmp(name, "node") == 0)
-    		*node = atoi(value);
-    	else if (strcmp(name, "conninfo") == 0)
-    		strncpy (conninfo, value, MAXLEN);
-    	else
-    		printf ("WARNING: %s/%s: Unknown name/value pair!\n", name, value);
-  	}
+		/* Copy into correct entry in parameters struct */
+		if (strcmp(name, "cluster") == 0)
+			strncpy (options->cluster_name, value, MAXLEN);
+		else if (strcmp(name, "node") == 0)
+			options->node = atoi(value);
+		else if (strcmp(name, "conninfo") == 0)
+			strncpy (options->conninfo, value, MAXLEN);
+		else if (strcmp(name, "rsync_options") == 0)
+			strncpy (options->rsync_options, value, QUERY_STR_LEN);
+		else if (strcmp(name, "ssh_options") == 0)
+			strncpy (options->ssh_options, value, QUERY_STR_LEN);
+		else if (strcmp(name, "loglevel") == 0)
+			strncpy (options->loglevel, value, MAXLEN);
+		else if (strcmp(name, "logfacility") == 0)
+			strncpy (options->logfacility, value, MAXLEN);
+		else if (strcmp(name, "failover") == 0)
+		{
+			char failoverstr[MAXLEN];
+			strncpy(failoverstr, value, MAXLEN);
 
-  	/* Close file */
-  	fclose (fp);
+			if (strcmp(failoverstr, "manual") == 0)
+				options->failover = MANUAL_FAILOVER;
+			else if (strcmp(failoverstr, "automatic") == 0)
+				options->failover = AUTOMATIC_FAILOVER;
+			else
+			{
+				log_warning(_("value for failover option is incorrect, it should be automatic or manual. Defaulting to manual.\n"));
+				options->failover = MANUAL_FAILOVER;
+			}
+		}
+		else if (strcmp(name, "priority") == 0)
+			options->priority = atoi(value);
+		else if (strcmp(name, "node_name") == 0)
+			strncpy(options->node_name, value, MAXLEN);
+		else if (strcmp(name, "promote_command") == 0)
+			strncpy(options->promote_command, value, MAXLEN);
+		else if (strcmp(name, "follow_command") == 0)
+			strncpy(options->follow_command, value, MAXLEN);
+		else if (strcmp(name, "master_response_timeout") == 0)
+			options->master_response_timeout = atoi(value);
+		else if (strcmp(name, "reconnect_attempts") == 0)
+			options->reconnect_attempts = atoi(value);
+		else if (strcmp(name, "reconnect_interval") == 0)
+			options->reconnect_intvl = atoi(value);
+		else
+			log_warning(_("%s/%s: Unknown name/value pair!\n"), name, value);
+	}
+
+	/* Close file */
+	fclose (fp);
+
+	/* Check config settings */
+	if (strnlen(options->cluster_name, MAXLEN)==0)
+	{
+		log_err(_("Cluster name is missing. Check the configuration file.\n"));
+		exit(ERR_BAD_CONFIG);
+	}
+
+	if (options->node == -1)
+	{
+		log_err(_("Node information is missing. Check the configuration file.\n"));
+		exit(ERR_BAD_CONFIG);
+	}
+
+	if (options->master_response_timeout <= 0)
+	{
+		log_err(_("Master response timeout must be greater than zero. Check the configuration file.\n"));
+		exit(ERR_BAD_CONFIG);
+	}
+
+	if (options->reconnect_attempts < 0)
+	{
+		log_err(_("Reconnect attempts must be zero or greater. Check the configuration file.\n"));
+		exit(ERR_BAD_CONFIG);
+	}
+
+	if (options->reconnect_intvl <= 0)
+	{
+		log_err(_("Reconnect intervals must be zero or greater. Check the configuration file.\n"));
+		exit(ERR_BAD_CONFIG);
+	}
 }
 
+
 char *
 trim (char *s)
 {
-  /* Initialize start, end pointers */
-  char *s1 = s, *s2 = &s[strlen (s) - 1];
+	/* Initialize start, end pointers */
+	char *s1 = s, *s2 = &s[strlen (s) - 1];
 
-  /* Trim and delimit right side */
-  while ( (isspace (*s2)) && (s2 >= s1) )
-    s2--;
-  *(s2+1) = '\0';
+	/* Trim and delimit right side */
+	while ( (isspace (*s2)) && (s2 >= s1) )
+		--s2;
+	*(s2+1) = '\0';
 
-  /* Trim left side */
-  while ( (isspace (*s1)) && (s1 < s2) )
-    s1++;
+	/* Trim left side */
+	while ( (isspace (*s1)) && (s1 < s2) )
+		++s1;
 
-  /* Copy finished string */
-  strcpy (s, s1);
-  return s;
+	/* Copy finished string */
+	strcpy (s, s1);
+	return s;
 }
 
 void
 parse_line(char *buff, char *name, char *value)
 {
-	int i;
-	int j;
+	int i = 0;
+	int j = 0;
 
 	/*
 	 * first we find the name of the parameter
 	 */
-	j = 0;
-	for (i = 0; i < MAXLEN; i++)
+	for ( ; i < MAXLEN; ++i)
 	{
 		if (buff[i] != '=')
 			name[j++] = buff[i];
@@ -83,12 +192,11 @@ parse_line(char *buff, char *name, char *value)
 	}
 	name[j] = '\0';
 
-	i++;
 	/*
 	 * Now the value
-	 */ 
+	 */
 	j = 0;
-	for ( ; i < MAXLEN; i++)
+	for ( ++i ; i < MAXLEN; ++i)
 		if (buff[i] == '\'')
 			continue;
 		else if (buff[i] != '\n')
@@ -96,5 +204,102 @@ parse_line(char *buff, char *name, char *value)
 		else
 			break;
 	value[j] = '\0';
-    trim(value);
+	trim(value);
+}
+
+bool
+reload_configuration(char *config_file, t_configuration_options *orig_options)
+{
+	PGconn	*conn;
+
+	t_configuration_options new_options;
+
+	/*
+	 * Re-read the configuration file: repmgr.conf
+	 */
+	log_info(_("Reloading configuration file and updating repmgr tables\n"));
+	parse_config(config_file, &new_options);
+	if (new_options.node == -1)
+	{
+		log_warning(_("\nCannot load new configuration, will keep current one.\n"));
+		return false;
+	}
+
+	if (strcmp(new_options.cluster_name, orig_options->cluster_name) != 0)
+	{
+		log_warning(_("\nCannot change cluster name, will keep current configuration.\n"));
+		return false;
+	}
+
+	if (new_options.node != orig_options->node)
+	{
+		log_warning(_("\nCannot change node number, will keep current configuration.\n"));
+		return false;
+	}
+
+	if (new_options.node_name != orig_options->node_name)
+	{
+		log_warning(_("\nCannot change standby name, will keep current configuration.\n"));
+		return false;
+	}
+
+	if (new_options.failover != MANUAL_FAILOVER && new_options.failover != AUTOMATIC_FAILOVER)
+	{
+		log_warning(_("\nNew value for failover is not valid. Should be MANUAL or AUTOMATIC.\n"));
+		return false;
+	}
+
+	if (new_options.master_response_timeout <= 0)
+	{
+		log_warning(_("\nNew value for master_response_timeout is not valid. Should be greater than zero.\n"));
+		return false;
+	}
+
+	if (new_options.reconnect_attempts < 0)
+	{
+		log_warning(_("\nNew value for reconnect_attempts is not valid. Should be greater or equal than zero.\n"));
+		return false;
+	}
+
+	if (new_options.reconnect_intvl < 0)
+	{
+		log_warning(_("\nNew value for reconnect_interval is not valid. Should be greater or equal than zero.\n"));
+		return false;
+	}
+
+	/* Test conninfo string */
+	conn = establishDBConnection(new_options.conninfo, false);
+	if (!conn || (PQstatus(conn) != CONNECTION_OK))
+	{
+		log_warning(_("\nconninfo string is not valid, will keep current configuration.\n"));
+		return false;
+	}
+	PQfinish(conn);
+
+	/* Configuration seems ok, will load new values */
+	strcpy(orig_options->cluster_name, new_options.cluster_name);
+	orig_options->node = new_options.node;
+	strcpy(orig_options->conninfo, new_options.conninfo);
+	orig_options->failover = new_options.failover;
+	orig_options->priority = new_options.priority;
+	strcpy(orig_options->node_name, new_options.node_name);
+	strcpy(orig_options->promote_command, new_options.promote_command);
+	strcpy(orig_options->follow_command, new_options.follow_command);
+	strcpy(orig_options->rsync_options, new_options.rsync_options);
+	strcpy(orig_options->ssh_options, new_options.ssh_options);
+	orig_options->master_response_timeout = new_options.master_response_timeout;
+	orig_options->reconnect_attempts = new_options.reconnect_attempts;
+	orig_options->reconnect_intvl = new_options.reconnect_intvl;
+	/*
+	 * XXX These ones can change with a simple SIGHUP?
+
+		strcpy (orig_options->loglevel, new_options.loglevel);
+		strcpy (orig_options->logfacility, new_options.logfacility);
+
+		logger_shutdown();
+		XXX do we have progname here ?
+		logger_init(progname, orig_options.loglevel, orig_options.logfacility);
+	*/
+
+	return true;
 }

config.h

@@ -1,9 +1,50 @@
 /*
  * config.h
- * Copyright (c) 2ndQuadrant, 2010
+ * Copyright (c) 2ndQuadrant, 2010-2012
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  *
  */
 
-void parse_config(const char *config_file, char *cluster_name, int *node, char *service);
+#ifndef _REPMGR_CONFIG_H_
+#define _REPMGR_CONFIG_H_
+
+#include "repmgr.h"
+#include "strutil.h"
+
+typedef struct
+{
+	char cluster_name[MAXLEN];
+	int node;
+	char conninfo[MAXLEN];
+	int failover;
+	int priority;
+	char node_name[MAXLEN];
+	char promote_command[MAXLEN];
+	char follow_command[MAXLEN];
+	char loglevel[MAXLEN];
+	char logfacility[MAXLEN];
+	char rsync_options[QUERY_STR_LEN];
+	char ssh_options[QUERY_STR_LEN];
+	int  master_response_timeout;
+	int  reconnect_attempts;
+	int  reconnect_intvl;
+} t_configuration_options;
+
+void parse_config(const char *config_file, t_configuration_options *options);
 void parse_line(char *buff, char *name, char *value);
 char *trim(char *s);
+bool reload_configuration(char *config_file, t_configuration_options *orig_options);
+
+#endif

dbutils.c

@@ -1,122 +1,283 @@
 /*
- * dbutils.c
- * Copyright (c) 2ndQuadrant, 2010
+ * dbutils.c - Database connection/management functions
+ * Copyright (C) 2ndQuadrant, 2010-2012
  *
- * Database connection/management functions
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  *
  */
 
+#include <unistd.h>
+
 #include "repmgr.h"
+#include "strutil.h"
+#include "log.h"
 
 PGconn *
 establishDBConnection(const char *conninfo, const bool exit_on_error)
 {
-	PGconn *conn;
-    /* Make a connection to the database */
-    conn = PQconnectdb(conninfo);
-    /* Check to see that the backend connection was successfully made */
-    if ((PQstatus(conn) != CONNECTION_OK))
-    {
-        fprintf(stderr, "Connection to database failed: %s", 
-					PQerrorMessage(conn));
+	/* Make a connection to the database */
+	PGconn *conn = NULL;
+	char    connection_string[MAXLEN];
+
+	strcpy(connection_string, conninfo);
+	strcat(connection_string, " fallback_application_name='repmgr'");
+	conn = PQconnectdb(connection_string);
+
+	/* Check to see that the backend connection was successfully made */
+	if ((PQstatus(conn) != CONNECTION_OK))
+	{
+		log_err(_("Connection to database failed: %s\n"),
+		        PQerrorMessage(conn));
+
 		if (exit_on_error)
 		{
-    	    PQfinish(conn);
-			exit(1);
+			PQfinish(conn);
+			exit(ERR_DB_CON);
 		}
-    }
+	}
 
 	return conn;
 }
 
+PGconn *
+establishDBConnectionByParams(const char *keywords[], const char *values[],const bool exit_on_error)
+{
+	/* Make a connection to the database */
+	PGconn *conn = PQconnectdbParams(keywords, values, true);
 
+	/* Check to see that the backend connection was successfully made */
+	if ((PQstatus(conn) != CONNECTION_OK))
+	{
+		log_err(_("Connection to database failed: %s\n"),
+		        PQerrorMessage(conn));
+		if (exit_on_error)
+		{
+			PQfinish(conn);
+			exit(ERR_DB_CON);
+		}
+	}
+
+	return conn;
+}
 
 bool
 is_standby(PGconn *conn)
 {
-    PGresult   *res;
-	bool		result;
+	PGresult   *res;
+	bool		result = false;
 
-    res = PQexec(conn, "SELECT pg_is_in_recovery()");
-    if (PQresultStatus(res) != PGRES_TUPLES_OK)
-    {
-        fprintf(stderr, "Can't query server mode: %s", PQerrorMessage(conn));
-        PQclear(res);
-        PQfinish(conn);
-		exit(1);
-    }
+	res = PQexec(conn, "SELECT pg_is_in_recovery()");
 
-	if (strcmp(PQgetvalue(res, 0, 0), "f") == 0)
-		result = false;
-	else
-		result = true;	
+	if (PQresultStatus(res) != PGRES_TUPLES_OK)
+	{
+		log_err(_("Can't query server mode: %s"),
+		        PQerrorMessage(conn));
+		PQclear(res);
+		PQfinish(conn);
+		exit(ERR_DB_QUERY);
+	}
+
+	if (PQntuples(res) == 1 && strcmp(PQgetvalue(res, 0, 0), "t") == 0)
+		result = true;
 
 	PQclear(res);
 	return result;
 }
 
 
-/* 
+
+bool
+is_witness(PGconn *conn, char *schema, char *cluster, int node_id)
+{
+	PGresult   *res;
+	bool		result = false;
+	char		sqlquery[QUERY_STR_LEN];
+
+	sqlquery_snprintf(sqlquery, "SELECT witness from %s.repl_nodes where cluster = '%s' and id = %d",
+	                  schema, cluster, node_id);
+	res = PQexec(conn, sqlquery);
+	if (PQresultStatus(res) != PGRES_TUPLES_OK)
+	{
+		log_err(_("Can't query server mode: %s"), PQerrorMessage(conn));
+		PQclear(res);
+		PQfinish(conn);
+		exit(ERR_DB_QUERY);
+	}
+
+	if (PQntuples(res) == 1 && strcmp(PQgetvalue(res, 0, 0), "t") == 0)
+		result = true;
+
+	PQclear(res);
+	return result;
+}
+
+
+/* check the PQStatus and try to 'select 1' to confirm good connection */
+bool
+is_pgup(PGconn *conn, int timeout)
+{
+	char		sqlquery[QUERY_STR_LEN];
+	/* Check the connection status twice in case it changes after reset */
+	bool		twice = false;
+
+	/* Check the connection status twice in case it changes after reset */
+	for (;;)
+	{
+		if (PQstatus(conn) != CONNECTION_OK)
+		{
+			if (twice)
+				return false;
+			PQreset(conn);  // reconnect
+			twice = true;
+		}
+		else
+		{
+			/*
+			* Send a SELECT 1 just to check if the connection is OK
+			*/
+			if (!CancelQuery(conn, timeout))
+				goto failed;
+			if (wait_connection_availability(conn, timeout) != 1)
+				goto failed;
+
+			sqlquery_snprintf(sqlquery, "SELECT 1");
+			if (PQsendQuery(conn, sqlquery) == 0)
+			{
+				log_warning(_("PQsendQuery: Query could not be sent to primary. %s\n"),
+				            PQerrorMessage(conn));
+				goto failed;
+			}
+			if (wait_connection_availability(conn, timeout) != 1)
+				goto failed;
+
+			break;
+
+failed:
+			// we need to retry, because we might just have loose the connection once
+			if (twice)
+				return false;
+			PQreset(conn);  // reconnect
+			twice = true;
+		}
+	}
+	return true;
+}
+
+
+/*
  * If postgreSQL version is 9 or superior returns the major version
  * if 8 or inferior returns an empty string
  */
 char *
-pg_version(PGconn *conn)
+pg_version(PGconn *conn, char* major_version)
 {
 	PGresult	*res;
-	char		*major_version;
 
-	int			major_version1;
-	char		*major_version2;
+	int					 major_version1;
+	char				*major_version2;
 
-    res = PQexec(conn, "WITH pg_version(ver) AS (SELECT split_part(version(), ' ', 2)) "
-                       "SELECT split_part(ver, '.', 1), split_part(ver, '.', 2) FROM pg_version");
-    if (PQresultStatus(res) != PGRES_TUPLES_OK)
-    {
-		fprintf(stderr, "PQexec failed: %s", PQerrorMessage(conn));
-        PQclear(res);
+	res = PQexec(conn,
+	             "WITH pg_version(ver) AS "
+	             "(SELECT split_part(version(), ' ', 2)) "
+	             "SELECT split_part(ver, '.', 1), split_part(ver, '.', 2) "
+	             "FROM pg_version");
+
+	if (PQresultStatus(res) != PGRES_TUPLES_OK)
+	{
+		log_err(_("Version check PQexec failed: %s"),
+		        PQerrorMessage(conn));
+		PQclear(res);
 		PQfinish(conn);
-		exit(1);
-    }
-    major_version1 = atoi(PQgetvalue(res, 0, 0));
-    major_version2 = PQgetvalue(res, 0, 1);
-    PQclear(res);
+		exit(ERR_DB_QUERY);
+	}
+
+	major_version1 = atoi(PQgetvalue(res, 0, 0));
+	major_version2 = PQgetvalue(res, 0, 1);
 
-	major_version = malloc(10);
 	if (major_version1 >= 9)
 	{
 		/* form a major version string */
-		sprintf(major_version, "%d.%s", major_version1, major_version2);
+		xsnprintf(major_version, MAXVERSIONSTR, "%d.%s", major_version1,
+		          major_version2);
 	}
 	else
 		strcpy(major_version, "");
 
+	PQclear(res);
+
 	return major_version;
 }
 
 
 bool
-guc_setted(PGconn *conn, const char *parameter, const char *op, const char *value)
+guc_setted(PGconn *conn, const char *parameter, const char *op,
+           const char *value)
 {
 	PGresult	*res;
-	char		sqlquery[8192];
+	char		sqlquery[QUERY_STR_LEN];
 
-	sprintf(sqlquery, "SELECT true FROM pg_settings "
-					  " WHERE name = '%s' AND setting %s '%s'",
-					  parameter, op, value);
+	sqlquery_snprintf(sqlquery, "SELECT true FROM pg_settings "
+	                  " WHERE name = '%s' AND setting %s '%s'",
+	                  parameter, op, value);
 
-    res = PQexec(conn, sqlquery);
-    if (PQresultStatus(res) != PGRES_TUPLES_OK)
-    {
-		fprintf(stderr, "PQexec failed: %s", PQerrorMessage(conn));
-        PQclear(res);
+	res = PQexec(conn, sqlquery);
+	if (PQresultStatus(res) != PGRES_TUPLES_OK)
+	{
+		log_err(_("GUC setting check PQexec failed: %s"),
+		        PQerrorMessage(conn));
+		PQclear(res);
 		PQfinish(conn);
-		exit(1);
-    }
+		exit(ERR_DB_QUERY);
+	}
 	if (PQntuples(res) == 0)
 	{
 		PQclear(res);
-	   	return false;
+		return false;
+	}
+	PQclear(res);
+
+	return true;
+}
+
+/**
+ * Just like guc_setted except with an extra parameter containing the name of
+ * the pg datatype so that the comparison can be done properly.
+ */
+bool
+guc_setted_typed(PGconn *conn, const char *parameter, const char *op, 
+                 const char *value, const char *datatype)
+{
+	PGresult	*res;
+	char		sqlquery[QUERY_STR_LEN];
+
+	sqlquery_snprintf(sqlquery, "SELECT true FROM pg_settings "
+	                  " WHERE name = '%s' AND setting::%s %s '%s'::%s",
+	                  parameter, datatype, op, value, datatype);
+
+	res = PQexec(conn, sqlquery);
+	if (PQresultStatus(res) != PGRES_TUPLES_OK)
+	{
+		log_err(_("GUC setting check PQexec failed: %s"),
+		        PQerrorMessage(conn));
+		PQclear(res);
+		PQfinish(conn);
+		exit(ERR_DB_QUERY);
+	}
+	if (PQntuples(res) == 0)
+	{
+		PQclear(res);
+		return false;
 	}
 	PQclear(res);
 
@@ -128,75 +289,115 @@ const char *
 get_cluster_size(PGconn *conn)
 {
 	PGresult	*res;
-	const char		*size;
-	char		sqlquery[8192];
+	const char	*size;
+	char		 sqlquery[QUERY_STR_LEN];
 
-	sprintf(sqlquery, "SELECT pg_size_pretty(SUM(pg_database_size(oid))::bigint) "
-					  "  FROM pg_database ");
+	sqlquery_snprintf(
+	    sqlquery,
+	    "SELECT pg_size_pretty(SUM(pg_database_size(oid))::bigint) "
+	    "	 FROM pg_database ");
 
-    res = PQexec(conn, sqlquery);
-    if (PQresultStatus(res) != PGRES_TUPLES_OK)
-    {
-		fprintf(stderr, "PQexec failed: %s", PQerrorMessage(conn));
-        PQclear(res);
+	res = PQexec(conn, sqlquery);
+	if (PQresultStatus(res) != PGRES_TUPLES_OK)
+	{
+		log_err(_("Get cluster size PQexec failed: %s"),
+		        PQerrorMessage(conn));
+		PQclear(res);
 		PQfinish(conn);
-		exit(1);
-    }
-   	size = PQgetvalue(res, 0, 0);
+		exit(ERR_DB_QUERY);
+	}
+	size = PQgetvalue(res, 0, 0);
 	PQclear(res);
 	return size;
 }
 
 /*
- * get a connection to master by reading repl_nodes, creating a connection 
+ * get a connection to master by reading repl_nodes, creating a connection
  * to each node (one at a time) and finding if it is a master or a standby
+ *
+ * NB: If master_conninfo_out may be NULL.  If it is non-null, it is assumed to
+ * point to allocated memory of MAXCONNINFO in length, and the master server
+ * connection string is placed there.
  */
 PGconn *
-getMasterConnection(PGconn *standby_conn, int id, char *cluster, int *master_id)
+getMasterConnection(PGconn *standby_conn, char *schema, char *cluster,
+                    int *master_id, char *master_conninfo_out)
 {
-	PGconn	 *master_conn = NULL;
-    PGresult *res1;
-    PGresult *res2;
-	char 	 sqlquery[8192];
-	char	 master_conninfo[8192];
+	PGconn		*master_conn	 = NULL;
+	PGresult	*res1;
+	PGresult	*res2;
+	char		 sqlquery[QUERY_STR_LEN];
+	char		 master_conninfo_stack[MAXCONNINFO];
+	char		*master_conninfo = &*master_conninfo_stack;
+	char		 schema_quoted[MAXLEN];
+
 	int		 i;
 
-	/* find all nodes belonging to this cluster */
-	sprintf(sqlquery, "SELECT * FROM repmgr_%s.repl_nodes "
- 					  " WHERE cluster = '%s' and id <> %d",
-					  cluster, cluster, id);
+	/*
+	 * If the caller wanted to get a copy of the connection info string, sub
+	 * out the local stack pointer for the pointer passed by the caller.
+	 */
+	if (master_conninfo_out != NULL)
+		master_conninfo = master_conninfo_out;
 
-    res1 = PQexec(standby_conn, sqlquery);
-    if (PQresultStatus(res1) != PGRES_TUPLES_OK)
-    {
-        fprintf(stderr, "Can't get nodes info: %s\n", PQerrorMessage(standby_conn));
-        PQclear(res1);
+	/*
+	 * XXX: This is copied in at least two other procedures
+	 *
+	 * Assemble the unquoted schema name
+	 */
+	{
+		char *identifier = PQescapeIdentifier(standby_conn, schema,
+		                                      strlen(schema));
+
+		maxlen_snprintf(schema_quoted, "%s", identifier);
+		PQfreemem(identifier);
+	}
+
+	/* find all nodes belonging to this cluster */
+	log_info(_("finding node list for cluster '%s'\n"),
+	         cluster);
+
+	sqlquery_snprintf(sqlquery, "SELECT id, conninfo FROM %s.repl_nodes "
+	                  " WHERE cluster = '%s' and not witness",
+	                  schema_quoted, cluster);
+
+	res1 = PQexec(standby_conn, sqlquery);
+	if (PQresultStatus(res1) != PGRES_TUPLES_OK)
+	{
+		log_err(_("Can't get nodes info: %s\n"),
+		        PQerrorMessage(standby_conn));
+		PQclear(res1);
 		PQfinish(standby_conn);
-		exit(1);
-    }
+		exit(ERR_DB_QUERY);
+	}
 
 	for (i = 0; i < PQntuples(res1); i++)
-    {
+	{
 		/* initialize with the values of the current node being processed */
 		*master_id = atoi(PQgetvalue(res1, i, 0));
-		strcpy(master_conninfo, PQgetvalue(res1, i, 2));
+		strncpy(master_conninfo, PQgetvalue(res1, i, 1), MAXCONNINFO);
+		log_info(_("checking role of cluster node '%s'\n"),
+		         master_conninfo);
 		master_conn = establishDBConnection(master_conninfo, false);
+
 		if (PQstatus(master_conn) != CONNECTION_OK)
 			continue;
 
-		/* 
-		 * I can't use the is_standby() function here because on error that 
-  		 * function closes the connection i pass and exit, but i still need to close
-		 * standby_conn
+		/*
+		 * Can't use the is_standby() function here because on error that
+		 * function closes the connection passed and exits.  This still
+		 * needs to close master_conn first.
 		 */
-    	res2 = PQexec(master_conn, "SELECT pg_is_in_recovery()");
-    	if (PQresultStatus(res2) != PGRES_TUPLES_OK)
-    	{
-    	    fprintf(stderr, "Can't get recovery state from this node: %s\n", PQerrorMessage(master_conn));
-    	    PQclear(res2);
+		res2 = PQexec(master_conn, "SELECT pg_is_in_recovery()");
+
+		if (PQresultStatus(res2) != PGRES_TUPLES_OK)
+		{
+			log_err(_("Can't get recovery state from this node: %s\n"),
+			        PQerrorMessage(master_conn));
+			PQclear(res2);
 			PQfinish(master_conn);
 			continue;
-    	}
+		}
 
 		/* if false, this is the master */
 		if (strcmp(PQgetvalue(res2, 0, 0), "f") == 0)
@@ -212,17 +413,86 @@ getMasterConnection(PGconn *standby_conn, int id, char *cluster, int *master_id)
 			PQfinish(master_conn);
 			*master_id = -1;
 		}
-    }
+	}
 
 	/* If we finish this loop without finding a master then
-     * we doesn't have the info or the master has failed (or we 
-     * reached max_connections or superuser_reserved_connections, 
-     * anything else i'm missing?),
-	 * Probably we will need to check the error to know if we need 
-     * to start failover procedure or just fix some situation on the
-     * standby.
-     */
+	 * we doesn't have the info or the master has failed (or we
+	 * reached max_connections or superuser_reserved_connections,
+	 * anything else I'm missing?).
+	 *
+	 * Probably we will need to check the error to know if we need
+	 * to start failover procedure or just fix some situation on the
+	 * standby.
+	 */
 	PQclear(res1);
 	return NULL;
 }
 
+
+/*
+ * wait until current query finishes ignoring any results, this could be an async command
+ * or a cancelation of a query
+ * return 1 if Ok; 0 if any error ocurred; -1 if timeout reached
+ */
+int
+wait_connection_availability(PGconn *conn, int timeout)
+{
+	PGresult   *res;
+
+	while(timeout-- >= 0)
+	{
+		if (PQconsumeInput(conn) == 0)
+		{
+			log_warning(_("wait_connection_availability: could not receive data from master. %s\n"),
+			            PQerrorMessage(conn));
+			return 0;
+		}
+
+		if (PQisBusy(conn) == 0)
+		{
+			res = PQgetResult(conn);
+			if (res == NULL)
+				break;
+			PQclear(res);
+		}
+		sleep(1);
+	}
+	if (timeout >= 0)
+		return 1;
+	else {
+		log_warning(_("wait_connection_availability: timeout reached"));
+		return -1;
+	}
+}
+
+
+bool
+CancelQuery(PGconn *conn, int timeout)
+{
+	char errbuf[ERRBUFF_SIZE];
+	PGcancel *pgcancel;
+
+	if (wait_connection_availability(conn, timeout) != 1)
+		return false;
+
+	pgcancel = PQgetCancel(conn);
+
+	if (pgcancel == NULL)
+		return false;
+
+	/*
+	 * PQcancel can only return 0 if socket()/connect()/send()
+ 	 * fails, in any of those cases we can assume something
+	 * bad happened to the connection
+	 */
+	if (PQcancel(pgcancel, errbuf, ERRBUFF_SIZE) == 0)
+	{
+		log_warning(_("Can't stop current query: %s\n"), errbuf);
+		PQfreeCancel(pgcancel);
+		return false;
+	}
+
+	PQfreeCancel(pgcancel);
+
+	return true;
+}

dbutils.h

@@ -1,12 +1,43 @@
 /*
  * dbutils.h
- * Copyright (c) 2ndQuadrant, 2010
+ * Copyright (c) 2ndQuadrant, 2010-2012
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  *
  */
 
+#ifndef _REPMGR_DBUTILS_H_
+#define _REPMGR_DBUTILS_H_
+
+#include "strutil.h"
+
 PGconn *establishDBConnection(const char *conninfo, const bool exit_on_error);
+PGconn *establishDBConnectionByParams(const char *keywords[],
+                                      const char *values[],
+                                      const bool exit_on_error);
 bool    is_standby(PGconn *conn);
-char   *pg_version(PGconn *conn);
-bool 	guc_setted(PGconn *conn, const char *parameter, const char *op, const char *value);
-const char   *get_cluster_size(PGconn *conn);
-PGconn * getMasterConnection(PGconn *standby_conn, int id, char *cluster, int *master_id);
+bool    is_witness(PGconn *conn, char *schema, char *cluster, int node_id);
+bool	is_pgup(PGconn *conn, int timeout);
+char   *pg_version(PGconn *conn, char* major_version);
+bool	guc_setted(PGconn *conn, const char *parameter, const char *op,
+                   const char *value);
+bool	guc_setted_typed(PGconn *conn, const char *parameter, const char *op,
+                   const char *value, const char *datatype);
+const char	 *get_cluster_size(PGconn *conn);
+PGconn *getMasterConnection(PGconn *standby_conn, char *schema, char *cluster,
+                            int *master_id, char *master_conninfo_out);
+
+int wait_connection_availability(PGconn *conn, int timeout);
+bool CancelQuery(PGconn *conn, int timeout);
+#endif

debian/DEBIAN/control

@@ -0,0 +1,9 @@
+Package: repmgr-auto
+Version: 2.0beta2
+Section: database
+Priority: optional
+Architecture: all
+Depends: rsync, postgresql-9.0 | postgresql-9.1 | postgresql-9.2 | postgresql-9.3
+Maintainer: Jaime Casanova <jaime@2ndQuadrant.com>
+Description: PostgreSQL replication setup, magament and monitoring
+ has two main executables

debian/repmgr.repmgrd.default

@@ -0,0 +1,14 @@
+#!/bin/sh
+# default settings for repmgrd. This file is source by /bin/sh from
+# /etc/init.d/repmgrd
+
+# Options for repmgrd
+REPMGRD_OPTS=""
+
+# repmgrd binary
+REPMGR_BIN="/usr/bin/repmgr"
+
+# pid file
+REPMGR_PIDFILE="/var/run/repmgrd.pid"
+
+

debian/repmgr.repmgrd.init

@@ -0,0 +1,48 @@
+#!/bin/sh
+### BEGIN INIT INFO
+# Provides:          repmgrd
+# Required-Start:    $local_fs $remote_fs $network $syslog $postgresql
+# Required-Stop:     $local_fs $remote_fs $network $syslog $postgresql
+# Should-Start:      $syslog $postgresql
+# Should-Start:      $syslog $postgresql
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: Start/stop repmgrd
+### END INIT INFO
+
+set -e
+
+if test -f /etc/default/repmgrd; then
+    . /etc/default/repmgrd
+fi
+
+if [ -z "$REPMGRD_BIN" ]; then
+    REPMGRD_BIN="/usr/bin/repmgrd"
+fi
+
+if [ -z "$REPMGRD_PIDFILE" ]; then
+    REPMGRD_PIDFILE="/var/run/repmgrd.pid"
+fi
+
+test -x $REPMGRD_BIN || exit 0
+
+case "$1" in
+    start)
+        start-stop-daemon --start --quiet --make-pidfile --pidfile $REPMGRD_PIDFILE --exec $REPMGRD_BIN $REPMGRD_OPTS
+        ;;
+
+    stop)
+        start-stop-daemon --stop --oknodo --quiet --pidfile $REPMGRD_PIDFILE
+        ;;
+
+    restart)
+        $0 stop && $0 start || exit 1
+        ;;
+
+    *)
+        echo "Usage: $0 {start|stop|restart}"
+        exit 1
+        ;;
+esac
+
+exit 0

errcode.h

@@ -0,0 +1,39 @@
+/*
+ * errcode.h
+ * Copyright (C) 2ndQuadrant, 2010-2012
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef _ERRCODE_H_
+#define _ERRCODE_H_
+
+/* Exit return code */
+
+#define SUCCESS 0
+#define ERR_BAD_CONFIG 1
+#define ERR_BAD_RSYNC 2
+#define ERR_STOP_BACKUP 3
+#define ERR_NO_RESTART 4
+#define ERR_NEEDS_XLOG 5
+#define ERR_DB_CON 6
+#define ERR_DB_QUERY 7
+#define ERR_PROMOTED 8
+#define ERR_BAD_PASSWORD 9
+#define ERR_STR_OVERFLOW 10
+#define ERR_FAILOVER_FAIL 11
+#define ERR_BAD_SSH	12
+
+#endif	/* _ERRCODE_H_ */

log.c

@@ -0,0 +1,238 @@
+/*
+ * log.c - Logging methods
+ * Copyright (C) 2ndQuadrant, 2010-2012
+ *
+ * This module is a set of methods for logging (currently only syslog)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "repmgr.h"
+
+#include <stdlib.h>
+
+#ifdef HAVE_SYSLOG
+#include <syslog.h>
+#endif
+
+#include <stdarg.h>
+#include <time.h>
+
+#include "log.h"
+
+#define DEFAULT_IDENT "repmgr"
+#ifdef HAVE_SYSLOG
+#define DEFAULT_SYSLOG_FACILITY LOG_LOCAL0
+#endif
+
+/* #define REPMGR_DEBUG */
+
+void stderr_log_with_level(const char *level_name, int level, const char *fmt, ...) {
+	size_t len = strlen(fmt);
+	char fmt1[len + 150];
+	time_t t;
+	struct tm *tm;
+	char buff[100];
+	va_list ap;
+
+	if(log_level >= level) {
+		time(&t);
+		tm = localtime(&t);
+
+		va_start(ap, fmt);
+
+		strftime(buff, 100, "[%Y-%m-%d %H:%M:%S]", tm);
+		snprintf(fmt1, len + 150, "%s [%s] %s", buff, level_name, fmt);
+		vfprintf(stderr, fmt1, ap);
+
+		va_end(ap);
+	}
+}
+
+
+static int detect_log_level(const char* level);
+static int detect_log_facility(const char* facility);
+
+int log_type = REPMGR_STDERR;
+int log_level = LOG_NOTICE;
+
+bool logger_init(const char* ident, const char* level, const char* facility)
+{
+
+	int l;
+	int f;
+
+#ifdef HAVE_SYSLOG
+	int syslog_facility = DEFAULT_SYSLOG_FACILITY;
+#endif
+
+#ifdef REPMGR_DEBUG
+	printf("Logger initialisation (Level: %s, Facility: %s)\n", level, facility);
+#endif
+
+	if (!ident)
+	{
+		ident = DEFAULT_IDENT;
+	}
+
+	if (level && *level)
+	{
+		l = detect_log_level(level);
+#ifdef REPMGR_DEBUG
+		printf("Assigned level for logger: %d\n", l);
+#endif
+
+		if (l > 0)
+			log_level = l;
+		else
+			stderr_log_warning(_("Cannot detect log level %s (use any of DEBUG, INFO, NOTICE, WARNING, ERR, ALERT, CRIT or EMERG)\n"), level);
+	}
+
+	if (facility && *facility)
+	{
+
+		f = detect_log_facility(facility);
+#ifdef REPMGR_DEBUG
+		printf("Assigned facility for logger: %d\n", f);
+#endif
+
+		if (f == 0)
+		{
+			/* No syslog requested, just stderr */
+#ifdef REPMGR_DEBUG
+			printf(_("Use stderr for logging\n"));
+#endif
+		}
+		else if (f == -1)
+		{
+			stderr_log_warning(_("Cannot detect log facility %s (use any of LOCAL0, LOCAL1, ..., LOCAL7, USER or STDERR)\n"), facility);
+		}
+#ifdef HAVE_SYSLOG
+		else
+		{
+			syslog_facility = f;
+			log_type = REPMGR_SYSLOG;
+		}
+#endif
+	}
+
+#ifdef HAVE_SYSLOG
+
+	if (log_type == REPMGR_SYSLOG)
+	{
+		setlogmask (LOG_UPTO (log_level));
+		openlog (ident, LOG_CONS | LOG_PID | LOG_NDELAY, syslog_facility);
+
+		stderr_log_notice(_("Setup syslog (level: %s, facility: %s)\n"), level, facility);
+	}
+
+#endif
+
+	return true;
+
+}
+
+bool logger_shutdown(void)
+{
+
+#ifdef HAVE_SYSLOG
+	if (log_type == REPMGR_SYSLOG)
+		closelog();
+#endif
+
+	return true;
+}
+
+/*
+ * Set a minimum logging level.  Intended for command line verbosity
+ * options, which might increase requested logging over what's specified
+ * in the regular configuration file.
+ */
+void logger_min_verbose(int minimum)
+{
+	if (log_level < minimum)
+		log_level = minimum;
+}
+
+int detect_log_level(const char* level)
+{
+	if (!strcmp(level, "DEBUG"))
+		return LOG_DEBUG;
+	if (!strcmp(level, "INFO"))
+		return LOG_INFO;
+	if (!strcmp(level, "NOTICE"))
+		return LOG_NOTICE;
+	if (!strcmp(level, "WARNING"))
+		return LOG_WARNING;
+	if (!strcmp(level, "ERR"))
+		return LOG_ERR;
+	if (!strcmp(level, "ALERT"))
+		return LOG_ALERT;
+	if (!strcmp(level, "CRIT"))
+		return LOG_CRIT;
+	if (!strcmp(level, "EMERG"))
+		return LOG_EMERG;
+
+	return 0;
+}
+
+int detect_log_facility(const char* facility)
+{
+	int local = 0;
+	if (!strncmp(facility, "LOCAL", 5) && strlen(facility) == 6)
+	{
+
+		local = atoi (&facility[5]);
+
+		switch (local)
+		{
+		case 0:
+			return LOG_LOCAL0;
+			break;
+		case 1:
+			return LOG_LOCAL1;
+			break;
+		case 2:
+			return LOG_LOCAL2;
+			break;
+		case 3:
+			return LOG_LOCAL3;
+			break;
+		case 4:
+			return LOG_LOCAL4;
+			break;
+		case 5:
+			return LOG_LOCAL5;
+			break;
+		case 6:
+			return LOG_LOCAL6;
+			break;
+		case 7:
+			return LOG_LOCAL7;
+			break;
+		}
+
+	}
+	else if (!strcmp(facility, "USER"))
+	{
+		return LOG_USER;
+	}
+	else if (!strcmp(facility, "STDERR"))
+	{
+		return 0;
+	}
+
+	return -1;
+}

log.h

@@ -0,0 +1,123 @@
+/*
+ * log.h
+ * Copyright (c) 2ndQuadrant, 2010-2012
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef _REPMGR_LOG_H_
+#define _REPMGR_LOG_H_
+
+#include "repmgr.h"
+
+#define REPMGR_SYSLOG 1
+#define REPMGR_STDERR 2
+
+void stderr_log_with_level(const char *level_name, int level, const char *fmt, ...);
+
+/* Standard error logging */
+#define stderr_log_debug(...) stderr_log_with_level("DEBUG", LOG_DEBUG, __VA_ARGS__)
+#define stderr_log_info(...)  stderr_log_with_level("INFO", LOG_INFO, __VA_ARGS__)
+#define stderr_log_notice(...) stderr_log_with_level("NOTICE", LOG_NOTICE, __VA_ARGS__)
+#define stderr_log_warning(...) stderr_log_with_level("WARNING", LOG_WARNING, __VA_ARGS__)
+#define stderr_log_err(...) stderr_log_with_level("ERROR", LOG_ERR, __VA_ARGS__)
+#define stderr_log_crit(...) stderr_log_with_level("CRITICAL", LOG_CRIT, __VA_ARGS__)
+#define stderr_log_alert(...) stderr_log_with_level("ALERT", LOG_ALERT, __VA_ARGS__)
+#define stderr_log_emerg(...) stderr_log_with_level("EMERGENCY", LOG_EMERG, __VA_ARGS__)
+
+#ifdef HAVE_SYSLOG
+
+#include <syslog.h>
+
+#define log_debug(...) \
+	if (log_type == REPMGR_SYSLOG) \
+		syslog(LOG_DEBUG, __VA_ARGS__); \
+	else \
+		stderr_log_debug(__VA_ARGS__);
+
+#define log_info(...) \
+	{ \
+		if (log_type == REPMGR_SYSLOG) syslog(LOG_INFO, __VA_ARGS__); \
+		else stderr_log_info(__VA_ARGS__); \
+	}
+
+#define log_notice(...) \
+	{ \
+		if (log_type == REPMGR_SYSLOG) syslog(LOG_NOTICE, __VA_ARGS__); \
+		else stderr_log_notice(__VA_ARGS__); \
+	}
+
+#define log_warning(...) \
+	{ \
+		if (log_type == REPMGR_SYSLOG) syslog(LOG_WARNING, __VA_ARGS__); \
+		else stderr_log_warning(__VA_ARGS__); \
+	}
+
+#define log_err(...) \
+	{ \
+		if (log_type == REPMGR_SYSLOG) syslog(LOG_ERR, __VA_ARGS__); \
+		else stderr_log_err(__VA_ARGS__); \
+	}
+
+#define log_crit(...) \
+	{ \
+		if (log_type == REPMGR_SYSLOG) syslog(LOG_CRIT, __VA_ARGS__); \
+		else stderr_log_crit(__VA_ARGS__); \
+	}
+
+#define log_alert(...) \
+	{ \
+		if (log_type == REPMGR_SYSLOG) syslog(LOG_ALERT, __VA_ARGS__); \
+		else stderr_log_alert(__VA_ARGS__); \
+	}
+
+#define log_emerg(...) \
+	{ \
+		if (log_type == REPMGR_SYSLOG) syslog(LOG_ALERT, __VA_ARGS__); \
+		else stderr_log_alert(__VA_ARGS__); \
+	}
+
+#else
+
+#define	LOG_EMERG	0	/* system is unusable */
+#define	LOG_ALERT	1	/* action must be taken immediately */
+#define	LOG_CRIT	2	/* critical conditions */
+#define	LOG_ERR		3	/* error conditions */
+#define	LOG_WARNING	4	/* warning conditions */
+#define	LOG_NOTICE	5	/* normal but significant condition */
+#define	LOG_INFO	6	/* informational */
+#define	LOG_DEBUG	7	/* debug-level messages */
+
+#define log_debug(...) stderr_log_debug(__VA_ARGS__)
+#define log_info(...) stderr_log_info(__VA_ARGS__)
+#define log_notice(...) stderr_log_notice(__VA_ARGS__)
+#define log_warning(...) stderr_log_warning(__VA_ARGS__)
+#define log_err(...) stderr_log_err(__VA_ARGS__)
+#define log_crit(...) stderr_log_crit(__VA_ARGS__)
+#define log_alert(...) stderr_log_alert(__VA_ARGS__)
+#define log_emerg(...) stderr_log_emerg(__VA_ARGS__)
+
+#endif
+
+
+/* Logger initialisation and shutdown */
+bool logger_shutdown(void);
+bool logger_init(const char* ident, const char* level, const char* facility);
+void logger_min_verbose(int minimum);
+
+extern int log_type;
+extern int log_level;
+
+#endif

repmgr.conf

@@ -1,3 +0,0 @@
-cluster=test
-node=2
-conninfo='host=192.168.204.104'

repmgr.conf.sample

@@ -0,0 +1,36 @@
+###################################################
+# Replication Manager configuration file
+###################################################
+
+# Cluster name
+cluster=test
+
+# Node ID
+node=2
+node_name=standby2
+
+# Connection information
+conninfo='host=192.168.204.104'
+rsync_options=--archive --checksum --compress --progress --rsh="ssh -o \"StrictHostKeyChecking no\""
+ssh_options=-o "StrictHostKeyChecking no"
+
+# How many seconds we wait for master response before declaring master failure
+master_response_timeout=60
+
+# How many time we try to reconnect to master before starting failover procedure
+reconnect_attempts=6
+reconnect_interval=10
+
+# Autofailover options
+failover=automatic
+priority=-1
+promote_command='repmgr standby promote -f /path/to/repmgr.conf'
+follow_command='repmgr standby follow -f /path/to/repmgr.conf -W'
+
+# Log level: possible values are DEBUG, INFO, NOTICE, WARNING, ERR, ALERT, CRIT or EMERG
+# Default: NOTICE
+loglevel=NOTICE
+
+# Logging facility: possible values are STDERR or - for Syslog integration - one of LOCAL0, LOCAL1, ..., LOCAL7, USER
+# Default: STDERR
+logfacility=STDERR

repmgr.h

@@ -1,6 +1,19 @@
 /*
- * dbutils.h
- * Copyright (c) 2ndQuadrant, 2010
+ * repmgr.h
+ * Copyright (c) 2ndQuadrant, 2010-2012
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  *
  */
 
@@ -11,14 +24,51 @@
 #include "getopt_long.h"
 #include "libpq-fe.h"
 
+#include "strutil.h"
 #include "dbutils.h"
-#include "config.h"
-
+#include "errcode.h"
 
 #define PRIMARY_MODE		0
 #define STANDBY_MODE		1
+#define WITNESS_MODE		2
 
-#define MAXLEN				80
-#define CONFIG_FILE			"repmgr.conf"
+#include "config.h"
+#define MAXFILENAME		1024
+#define ERRBUFF_SIZE		512
+
+#define DEFAULT_CONFIG_FILE		"./repmgr.conf"
+#define DEFAULT_WAL_KEEP_SEGMENTS	"5000"
+#define DEFAULT_DEST_DIR		"."
+#define DEFAULT_MASTER_PORT		"5432"
+#define DEFAULT_DBNAME			"postgres"
+#define DEFAULT_REPMGR_SCHEMA_PREFIX	"repmgr_"
+
+#define MANUAL_FAILOVER		0
+#define AUTOMATIC_FAILOVER	1
+
+/* Run time options type */
+typedef struct
+{
+
+	char dbname[MAXLEN];
+	char host[MAXLEN];
+	char username[MAXLEN];
+	char dest_dir[MAXFILENAME];
+	char config_file[MAXFILENAME];
+	char remote_user[MAXLEN];
+	char wal_keep_segments[MAXLEN];
+	bool verbose;
+	bool force;
+	bool wait_for_master;
+	bool ignore_rsync_warn;
+
+	char masterport[MAXLEN];
+	char localport[MAXLEN];
+
+	/* parameter used by CLUSTER CLEANUP */
+	int keep_history;
+} t_runtime_options;
+
+#define SLEEP_MONITOR		2
 
 #endif

repmgr.sql

@@ -1,3 +1,10 @@
+/*
+ * repmgr.sql
+ *
+ * Copyright (C) 2ndQuadrant, 2010-2012
+ *
+ */
+
 CREATE USER repmgr;
 CREATE SCHEMA repmgr;
 
@@ -5,51 +12,50 @@ CREATE SCHEMA repmgr;
  * The table repl_nodes keeps information about all machines in
  * a cluster
  */
-drop table if exists repl_nodes cascade;
 CREATE TABLE repl_nodes (
-  id 		integer	primary key,
-  cluster   text	not null,	-- Name to identify the cluster
-  conninfo	text	not null 	
+  id            integer primary key,
+  cluster   	text    not null,       -- Name to identify the cluster
+  name			text	not null,
+  conninfo      text    not null,
+  priority  	integer not null,
+  witness   	boolean not null default false
 );
 ALTER TABLE repl_nodes OWNER TO repmgr;
 
 /*
- * Keeps monitor info about every node and their relative "position" 
+ * Keeps monitor info about every node and their relative "position"
  * to primary
  */
-drop table if exists repl_monitor cascade;
 CREATE TABLE repl_monitor (
   primary_node                   INTEGER NOT NULL,
   standby_node                   INTEGER NOT NULL,
-  last_monitor_time		    	 TIMESTAMP WITH TIME ZONE NOT NULL,
-  last_wal_primary_location      TEXT NOT NULL,		
-  last_wal_standby_location      TEXT NOT NULL,
-  replication_lag                BIGINT NOT NULL, 
-  apply_lag                      BIGINT NOT NULL  
+  last_monitor_time                      TIMESTAMP WITH TIME ZONE NOT NULL,
+  last_wal_primary_location      TEXT NOT NULL,
+  last_wal_standby_location      TEXT,		-- In case of a witness server this will be NULL
+  replication_lag                BIGINT NOT NULL,
+  apply_lag                      BIGINT NOT NULL
 );
 ALTER TABLE repl_monitor OWNER TO repmgr;
 
-
 /*
  * This view shows the latest monitor info about every node.
  * Interesting thing to see:
- * replication_lag: in bytes (this is how far the latest xlog record 
+ * replication_lag: in bytes (this is how far the latest xlog record
  *                            we have received is from master)
  * apply_lag: in bytes (this is how far the latest xlog record
- *                      we have applied is from the latest record we 
+ *                      we have applied is from the latest record we
  *                      have received)
  * time_lag: how many seconds are we from being up-to-date with master
  */
-drop view if exists repl_status;
 CREATE VIEW repl_status AS
-WITH monitor_info AS (SELECT *, ROW_NUMBER() OVER (PARTITION BY primary_node, standby_node
-                                                       ORDER BY last_monitor_time desc)
-                        FROM repl_monitor)
-SELECT primary_node, standby_node, last_monitor_time, last_wal_primary_location, 
-       last_wal_standby_location, pg_size_pretty(replication_lag) replication_lag, 
-       pg_size_pretty(apply_lag) apply_lag, 
+SELECT primary_node, standby_node, name AS standby_name, last_monitor_time, last_wal_primary_location,
+       last_wal_standby_location, pg_size_pretty(replication_lag) replication_lag,
+       pg_size_pretty(apply_lag) apply_lag,
        age(now(), last_monitor_time) AS time_lag
-  FROM monitor_info a
- WHERE row_number = 1;
+ FROM repl_monitor JOIN repl_nodes ON standby_node = id
+WHERE (standby_node, last_monitor_time) IN (SELECT standby_node, MAX(last_monitor_time)
+                                              FROM repl_monitor GROUP BY 1);
 
 ALTER VIEW repl_status OWNER TO repmgr;
+
+CREATE INDEX idx_repl_status_sort ON repl_monitor(last_monitor_time, standby_node);

sql/Makefile

@@ -0,0 +1,19 @@
+#
+# Makefile
+# Copyright (c) 2ndQuadrant, 2010
+#
+
+MODULE_big = repmgr_funcs
+DATA_built=repmgr_funcs.sql
+DATA=uninstall_repmgr_funcs.sql
+OBJS=repmgr_funcs.o
+
+ifdef USE_PGXS
+PGXS := $(shell pg_config --pgxs)
+include $(PGXS)
+else
+subdir = contrib/repmgr/sql
+top_builddir = ../../..
+include $(top_builddir)/src/Makefile.global
+include $(top_srcdir)/contrib/contrib-global.mk
+endif

sql/repmgr_funcs.c

@@ -0,0 +1,189 @@
+/*
+ * repmgr_funcs.c
+ * Copyright (c) 2ndQuadrant, 2010
+ *
+ * Shared memory state management and some backend functions in SQL
+ */
+
+#include "postgres.h"
+#include "fmgr.h"
+#include "access/xlog.h"
+#include "miscadmin.h"
+#include "storage/ipc.h"
+#include "storage/lwlock.h"
+#include "storage/procarray.h"
+#include "storage/shmem.h"
+#include "storage/spin.h"
+#include "utils/builtins.h"
+
+/* same definition as the one in xlog_internal.h */
+#define MAXFNAMELEN 	64
+
+PG_MODULE_MAGIC;
+
+/*
+ * Global shared state
+ */
+typedef struct repmgrSharedState
+{
+    LWLockId    lock;           		/* protects search/modification */
+    char		location[MAXFNAMELEN];	/* last known xlog location */
+} repmgrSharedState;
+
+/* Links to shared memory state */
+static repmgrSharedState *shared_state = NULL;
+
+static shmem_startup_hook_type prev_shmem_startup_hook = NULL;
+
+void        _PG_init(void);
+void        _PG_fini(void);
+
+static void repmgr_shmem_startup(void);
+static Size repmgr_memsize(void);
+
+static bool repmgr_set_standby_location(char *locationstr);
+
+Datum repmgr_update_standby_location(PG_FUNCTION_ARGS);
+Datum repmgr_get_last_standby_location(PG_FUNCTION_ARGS);
+
+PG_FUNCTION_INFO_V1(repmgr_update_standby_location);
+PG_FUNCTION_INFO_V1(repmgr_get_last_standby_location);
+
+
+/*
+ * Module load callback
+ */
+void
+_PG_init(void)
+{
+	/*
+	 * In order to create our shared memory area, we have to be loaded via
+	 * shared_preload_libraries.  If not, fall out without hooking into any of
+	 * the main system.  (We don't throw error here because it seems useful to
+	 * allow the repmgr functions to be created even when the
+	 * module isn't active.  The functions must protect themselves against
+	 * being called then, however.)
+	 */
+	if (!process_shared_preload_libraries_in_progress)
+		return;
+
+	/*
+	 * Request additional shared resources.  (These are no-ops if we're not in
+	 * the postmaster process.)  We'll allocate or attach to the shared
+	 * resources in repmgr_shmem_startup().
+	 */
+	RequestAddinShmemSpace(repmgr_memsize());
+	RequestAddinLWLocks(1);
+
+	/*
+	 * Install hooks.
+	 */
+	prev_shmem_startup_hook = shmem_startup_hook;
+	shmem_startup_hook = repmgr_shmem_startup;
+}
+
+/*
+ * Module unload callback
+ */
+void
+_PG_fini(void)
+{
+	/* Uninstall hooks. */
+	shmem_startup_hook = prev_shmem_startup_hook;
+}
+
+/*
+ * shmem_startup hook: allocate or attach to shared memory,
+ */
+static void
+repmgr_shmem_startup(void)
+{
+	bool		found;
+
+	if (prev_shmem_startup_hook)
+		prev_shmem_startup_hook();
+
+	/* reset in case this is a restart within the postmaster */
+	shared_state = NULL;
+
+	/*
+	 * Create or attach to the shared memory state, including hash table
+	 */
+	LWLockAcquire(AddinShmemInitLock, LW_EXCLUSIVE);
+
+	shared_state = ShmemInitStruct("repmgr shared state",
+						   sizeof(repmgrSharedState),
+						   &found);
+
+	if (!found)
+	{
+		/* First time through ... */
+		shared_state->lock = LWLockAssign();
+		snprintf(shared_state->location,
+				sizeof(shared_state->location), "%X/%X", 0, 0);
+	}
+
+	LWLockRelease(AddinShmemInitLock);
+}
+
+
+/*
+ * Estimate shared memory space needed.
+ */
+static Size
+repmgr_memsize(void)
+{
+    return MAXALIGN(sizeof(repmgrSharedState));
+}
+
+
+static bool
+repmgr_set_standby_location(char *locationstr)
+{
+    /* Safety check... */
+    if (!shared_state)
+        return false;
+
+    LWLockAcquire(shared_state->lock, LW_EXCLUSIVE);
+	strncpy(shared_state->location, locationstr, MAXFNAMELEN);
+    LWLockRelease(shared_state->lock);
+
+	return true;
+}
+
+
+/* SQL Functions */
+
+/* Read last xlog location reported by this standby from shared memory */
+Datum
+repmgr_get_last_standby_location(PG_FUNCTION_ARGS)
+{
+	char location[MAXFNAMELEN];
+
+	/* Safety check... */
+	if (!shared_state)
+		PG_RETURN_NULL();
+
+	LWLockAcquire(shared_state->lock, LW_SHARED);
+	strncpy(location, shared_state->location, MAXFNAMELEN);
+	LWLockRelease(shared_state->lock);
+
+	PG_RETURN_TEXT_P(cstring_to_text(location));
+}
+
+
+/* Set update last xlog location reported by this standby to shared memory */
+Datum
+repmgr_update_standby_location(PG_FUNCTION_ARGS)
+{
+	text       *location = PG_GETARG_TEXT_P(0);
+	char       *locationstr;
+
+    /* Safety check... */
+    if (!shared_state)
+        PG_RETURN_BOOL(false);
+
+    locationstr = text_to_cstring(location);
+
+	PG_RETURN_BOOL(repmgr_set_standby_location(locationstr));
+}

sql/repmgr_funcs.sql.in

@@ -0,0 +1,15 @@
+/*
+ * repmgr_function.sql
+ * Copyright (c) 2ndQuadrant, 2010
+ *
+ */
+
+-- SET SEARCH_PATH TO 'repmgr';
+
+CREATE FUNCTION repmgr_update_standby_location(text) RETURNS boolean
+AS 'MODULE_PATHNAME', 'repmgr_update_standby_location'
+LANGUAGE C STRICT;
+
+CREATE FUNCTION repmgr_get_last_standby_location() RETURNS text
+AS 'MODULE_PATHNAME', 'repmgr_get_last_standby_location'
+LANGUAGE C STRICT;

sql/uninstall_repmgr_funcs.sql

@@ -0,0 +1,2 @@
+DROP FUNCTION repmgr_update_standby_location(text);
+DROP FUNCTION repmgr_get_last_standby_location();

strutil.c

@@ -0,0 +1,96 @@
+/*
+ * strutil.c
+ *
+ * Copyright (C) 2ndQuadrant, 2010-2012
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "log.h"
+#include "strutil.h"
+
+static int xvsnprintf(char *str, size_t size, const char *format, va_list ap);
+
+/* Add strnlen on platforms that don't have it, like OS X */
+#ifndef strnlen
+size_t
+strnlen(const char *s, size_t n)
+{
+	const char *end = (const char *) memchr(s, '\0', n);
+	return(end ? end - s : n);
+}
+#endif
+
+static int
+xvsnprintf(char *str, size_t size, const char *format, va_list ap)
+{
+	int retval;
+
+	retval = vsnprintf(str, size, format, ap);
+
+	if (retval >= size)
+	{
+		log_err(_("Buffer of size not large enough to format entire string '%s'\n"),
+		        str);
+		exit(ERR_STR_OVERFLOW);
+	}
+
+	return retval;
+}
+
+
+int
+xsnprintf(char *str, size_t size, const char *format, ...)
+{
+	va_list arglist;
+	int retval;
+
+	va_start(arglist, format);
+	retval = xvsnprintf(str, size, format, arglist);
+	va_end(arglist);
+
+	return retval;
+}
+
+
+int
+sqlquery_snprintf(char *str, const char *format, ...)
+{
+	va_list		arglist;
+	int			retval;
+
+	va_start(arglist, format);
+	retval = xvsnprintf(str, QUERY_STR_LEN, format, arglist);
+	va_end(arglist);
+
+	return retval;
+}
+
+
+int maxlen_snprintf(char *str, const char *format, ...)
+{
+	va_list		arglist;
+	int			retval;
+
+	va_start(arglist, format);
+	retval = xvsnprintf(str, MAXLEN, format, arglist);
+	va_end(arglist);
+
+	return retval;
+}

strutil.h

@@ -0,0 +1,43 @@
+/*
+ * strutil.h
+ * Copyright (C) 2ndQuadrant, 2010-2012
+ *
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef _STRUTIL_H_
+#define _STRUTIL_H_
+
+#include <stdlib.h>
+#include <errcode.h>
+
+#define QUERY_STR_LEN	8192
+#define MAXLEN			1024
+#define MAXLINELENGTH	4096
+#define MAXVERSIONSTR	16
+#define MAXCONNINFO		1024
+
+
+extern int xsnprintf(char *str, size_t size, const char *format, ...);
+extern int sqlquery_snprintf(char *str, const char *format, ...);
+extern int maxlen_snprintf(char *str, const char *format, ...);
+
+/* Add strnlen on platforms that don't have it, like OS X */
+#ifndef strnlen
+extern size_t strnlen(const char *s, size_t n);
+#endif
+
+#endif	/* _STRUTIL_H_ */

uninstall_repmgr.sql

@@ -0,0 +1,13 @@
+/*
+ * uninstall_repmgr.sql
+ *
+ * Copyright (C) 2ndQuadrant, 2010-2012
+ *
+ */
+
+DROP TABLE IF EXISTS repl_nodes;
+DROP TABLE IF EXISTS repl_monitor;
+DROP VIEW IF EXISTS repl_status;
+
+DROP SCHEMA repmgr;
+DROP USER repmgr;

version.h

@@ -0,0 +1,4 @@
+#ifndef _VERSION_H_
+#define _VERSION_H_
+#define REPMGR_VERSION "2.0beta1"
+#endif