Add missing line break in logging output

Something else we don't have to worry about in repmgr4.

CONTRIBUTING.md

@@ -2,7 +2,7 @@ License and Contributions
 =========================
 
 `repmgr` is licensed under the GPL v3.  All of its code and documentation is
-Copyright 2010-2016, 2ndQuadrant Limited.  See the files COPYRIGHT and LICENSE for
+Copyright 2010-2017, 2ndQuadrant Limited.  See the files COPYRIGHT and LICENSE for
 details.
 
 The development of repmgr has primarily been sponsored by 2ndQuadrant customers.
@@ -21,9 +21,11 @@ copy of the relevant Copyright Assignment Form.
 Code style
 ----------
 
-Code in repmgr is formatted to a consistent style using the following command:
+Code in repmgr should be formatted to the same standards as the main PostgreSQL
+project. For more details see:
 
-    astyle --style=ansi --indent=tab --suffix=none *.c *.h
+    https://www.postgresql.org/docs/current/static/source-format.html
 
 Contributors should reformat their code similarly before submitting code to
-the project, in order to minimize merge conflicts with other work.
+the project, in order to minimize merge conflicts with other work.
+

COPYRIGHT

@@ -1,4 +1,4 @@
-Copyright (c) 2010-2016, 2ndQuadrant Limited
+Copyright (c) 2010-2017, 2ndQuadrant Limited
 All rights reserved.
 
 This program is free software: you can redistribute it and/or modify

HISTORY

@@ -1,3 +1,33 @@
+3.4.0   2019-02-
+		default log level is now INFO (Ian)
+        repmgr: fix `standby register --force` when updating existing node record (Ian)
+		repmgrd: set LSN shared memory value at standby startup (Ian)
+		repmgrd: improve logging during failover (Ian)
+
+3.3.2   2017-06-01
+        Add support for PostgreSQL 10 (Ian)
+        repmgr: ensure --replication-user option is honoured when passing database
+          connection parameters as a conninfo string (Ian)
+        repmgr: improve detection of pg_rewind on remote server (Ian)
+        repmgr: add DETAIL log output for additional clarification of error messages (Ian)
+        repmgr: suppress various spurious error messages in `standby follow` and
+          `standby switchover` (Ian)
+        repmgr: add missing `-P` option (Ian)
+        repmgrd: monitoring statistic reporting fixes (Ian)
+
+3.3.1   2017-03-13
+        repmgrd: prevent invalid apply lag value being written to the
+          monitoring table (Ian)
+        repmgrd: fix error in XLogRecPtr conversion when calculating
+          monitoring statistics (Ian)
+        repmgr: if replication slots in use, where possible delete slot on old
+          upstream node after following new upstream (Ian)
+        repmgr: improve logging of rsync actions (Ian)
+        repmgr: improve `standby clone` when synchronous replication in use (Ian)
+        repmgr: stricter checking of allowed node id values
+        repmgr: enable `master register --force` when there is a foreign key
+          dependency from a standby node (Ian)
+
 3.3     2016-12-27
         repmgr: always log to STDERR even if log facility defined (Ian)
         repmgr: add --log-to-file to log repmgr output to the defined

Makefile

@@ -1,6 +1,6 @@
 #
 # Makefile
-# Copyright (c) 2ndQuadrant, 2010-2016
+# Copyright (c) 2ndQuadrant, 2010-2017
 
 HEADERS = $(wildcard *.h)
 
@@ -8,8 +8,9 @@ repmgrd_OBJS = dbutils.o config.o repmgrd.o log.o strutil.o
 repmgr_OBJS = dbutils.o check_dir.o config.o repmgr.o log.o strutil.o dirmod.o compat.o
 
 DATA = repmgr.sql uninstall_repmgr.sql
+REGRESS = repmgr_funcs repmgr_test
 
-PG_CPPFLAGS = -I$(libpq_srcdir)
+PG_CPPFLAGS = -I$(includedir_internal) -I$(libpq_srcdir)
 PG_LIBS     = $(libpq_pgport)
 
 
@@ -17,11 +18,11 @@ all: repmgrd repmgr
 	$(MAKE) -C sql
 
 repmgrd: $(repmgrd_OBJS)
-	$(CC) -o repmgrd $(CFLAGS) $(repmgrd_OBJS) $(PG_LIBS) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS)
+	$(CC) -o repmgrd $(CFLAGS) $(repmgrd_OBJS) $(PG_LIBS) $(LDFLAGS) $(LDFLAGS_EX)
 	$(MAKE) -C sql
 
 repmgr: $(repmgr_OBJS)
-	$(CC) -o repmgr $(CFLAGS) $(repmgr_OBJS) $(PG_LIBS) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS)
+	$(CC) -o repmgr $(CFLAGS) $(repmgr_OBJS) $(PG_LIBS) $(LDFLAGS) $(LDFLAGS_EX)
 
 # Make all objects depend on all include files. This is a bit of a
 # shotgun approach, but the codebase is small enough that a complete rebuild

README.md

@@ -7,9 +7,13 @@ replication capabilities with utilities to set up standby servers, monitor
 replication, and perform administrative tasks such as failover or switchover
 operations.
 
-The current `repmgr` version, 3.2, supports all PostgreSQL versions from
+This `repmgr` version (3.4) supports PostgreSQL versions from
 9.3 to 9.6.
 
+*NOTE*: we strongly recommend using the repmgr 4.x series, which contains
+many new features and usability enhancements and is being actively developed
+and maintained.
+
 Overview
 --------
 
@@ -189,6 +193,14 @@ system.
   Instructions can be found in the APT section of the PostgreSQL Wiki
   ( https://wiki.postgresql.org/wiki/Apt ).
 
+  *NOTE*: repmgr 3.3 packages are now only available via a 2ndQuadrant-hosted
+  repository which can be installed like this:
+
+    apt-key adv --fetch-keys http://packages.2ndquadrant.com/repmgr3/apt/0xD3FA41F6.asc
+
+    echo deb http://packages.2ndquadrant.com/repmgr3/apt/ $(lsb_release -cs)-2ndquadrant main > /etc/apt/sources.list.d/repmgr3.list
+
+
 See `PACKAGES.md` for details on building .deb and .rpm packages from the
 `repmgr` source code.
 
@@ -202,7 +214,7 @@ See `PACKAGES.md` for details on building .deb and .rpm packages from the
 Release tarballs are also available:
 
     https://github.com/2ndQuadrant/repmgr/releases
-    http://repmgr.org/downloads.php
+    https://repmgr.org/
 
 `repmgr` is compiled in the same way as a PostgreSQL extension using the PGXS
 infrastructure, e.g.:
@@ -314,7 +326,13 @@ The following replication settings may need to be adjusted:
     max_wal_senders = 10
 
     # Ensure WAL files contain enough information to enable read-only queries
-    # on the standby
+    # on the standby.
+    #
+    #  PostgreSQL 9.5 and earlier: one of 'hot_standby' or 'logical'
+    #  PostgreSQL 9.6 and later: one of 'replica' or 'logical'
+    #    ('hot_standby' will still be accepted as an alias for 'replica')
+    #
+    # See: https://www.postgresql.org/docs/current/static/runtime-config-wal.html#GUC-WAL-LEVEL
 
     wal_level = 'hot_standby'
 
@@ -333,10 +351,11 @@ The following replication settings may need to be adjusted:
     archive_command = '/bin/true'
 
     # If cloning using rsync, or you have configured `pg_basebackup_options`
-    # in `repmgr.conf` to include the setting `--xlog-method=fetch`, *and*
-    # you have not set `restore_command` in `repmgr.conf`to fetch WAL files
-    # from another source such as Barman, you'll need to set `wal_keep_segments`
-    # to a high enough value to ensure that all WAL files generated while
+    # in `repmgr.conf` to include the setting `--xlog-method=fetch` (from
+    # PostgreSQL 10 `--wal-method=fetch`), *and* you have not set
+    # `restore_command` in `repmgr.conf`to fetch WAL files from another
+    # source such as Barman, you'll need to set `wal_keep_segments` to a
+    # high enough value to ensure that all WAL files generated while
     # the standby is being cloned are retained until the standby starts up.
 
     # wal_keep_segments = 5000
@@ -390,7 +409,8 @@ least the following parameters:
 
 - `cluster`: an arbitrary name for the replication cluster; this must be identical
      on all nodes
-- `node`: a unique integer identifying the node
+- `node`: a unique integer identifying the node; note this must be a positive
+     32 bit signed integer between 1 and 2147483647
 - `node_name`: a unique string identifying the node; we recommend a name
      specific to the server (e.g. 'server_1'); avoid names indicating the
      current replication role like 'master' or 'standby' as the server's
@@ -398,7 +418,8 @@ least the following parameters:
 - `conninfo`: a valid connection string for the `repmgr` database on the
      *current* server. (On the standby, the database will not yet exist, but
      `repmgr` needs to know the connection details to complete the setup
-     process).
+     process). *NOTE* this must be a keyword/value string, not a connection
+     URI; this limitation will be removed in a future `repmgr` version.
 
 `repmgr.conf` should not be stored inside the PostgreSQL data directory,
 as it could be overwritten when setting up or reinitialising the PostgreSQL
@@ -423,7 +444,7 @@ to include this schema name, e.g.
 ### Initialise the master server
 
 To enable `repmgr` to support a replication cluster, the master node must
-be registered with `repmgr`, which creates the `repmgr` database and adds
+be registered with `repmgr`, which creates the `repmgr` metadatabase and adds
 a metadata record for the server:
 
     $ repmgr -f repmgr.conf master register
@@ -501,7 +522,8 @@ place. To ensure this happens when using the default `pg_basebackup` method,
 `repmgr` will set `pg_basebackup`'s `--xlog-method` parameter to `stream`,
 which will ensure all WAL files generated during the cloning process are
 streamed in parallel with the main backup. Note that this requires two
-replication connections to be available.
+replication connections to be available (`repmgr` will verify sufficient
+connections are available before attempting to clone).
 
 To override this behaviour, in `repmgr.conf` set `pg_basebackup`'s
 `--xlog-method` parameter to `fetch`:
@@ -513,6 +535,9 @@ See the `pg_basebackup` documentation for details:
 
     https://www.postgresql.org/docs/current/static/app-pgbasebackup.html
 
+> *NOTE*: From PostgreSQL 10, `pg_basebackup`'s `--xlog-method` parameter
+> has been renamed to `--wal-method`.
+
 Make any adjustments to the standby's PostgreSQL configuration files now,
 then start the server.
 
@@ -625,7 +650,7 @@ In order to enable Barman support for `repmgr standby clone`, you must
 ensure that:
 
 - the name of the server configured in Barman is equal to the
-  `cluster_name` setting in `repmgr.conf`;
+  `cluster` setting in `repmgr.conf`;
 - the `barman_server` setting in `repmgr.conf` is set to the SSH
   hostname of the Barman server;
 - the `restore_command` setting in `repmgr.conf` is configured to
@@ -990,6 +1015,13 @@ both passwordless SSH access and the path of `repmgr.conf` on that server.
 > careful preparation and with adequate attention. In particular you should
 > be confident that your network environment is stable and reliable.
 >
+> Additionally you should be sure that the current master can be shut down
+> quickly and cleanly. In particular, access from applications should be
+> minimalized or preferably blocked completely. Also check that there is
+> no backlog of files waiting to be archived, as PostgreSQL will not shut
+> down until archiving completes, and that any standbys attached to the
+> current primary don't have a significant amount of replication lag.
+>
 > We recommend running `repmgr standby switchover` at the most verbose
 > logging level (`--log-level DEBUG --verbose`) and capturing all output
 > to assist troubleshooting any problems.
@@ -1056,7 +1088,7 @@ should have been updated to reflect this:
 
 ### Caveats
 
-- The functionality provided `repmgr standby switchover` is primarily aimed
+- The functionality provided by `repmgr standby switchover` is primarily aimed
   at a two-server master/standby replication cluster and currently does
   not support additional standbys.
 - `repmgr standby switchover` is designed to use the `pg_rewind` utility,
@@ -1070,11 +1102,6 @@ should have been updated to reflect this:
   the `repmgrd` may try and promote a standby by itself.
 - Any other standbys attached to the old master will need to be manually
   instructed to point to the new master (e.g. with `repmgr standby follow`).
-- You must ensure that following a server start using `pg_ctl`, log output
-  is not send to STDERR (the default behaviour). If logging is not configured,
-  we recommend setting `logging_collector=on` in `postgresql.conf` and
-  providing an explicit `-l/--log` setting in `repmgr.conf`'s `pg_ctl_options`
-  parameter.
 
 We hope to remove some of these restrictions in future versions of `repmgr`.
 
@@ -1153,11 +1180,10 @@ Additionally the following `repmgrd` options must be set in `repmgr.conf`:
     promote_command='repmgr standby promote -f /etc/repmgr.conf --log-to-file'
     follow_command='repmgr standby follow -f /etc/repmgr.conf --log-to-file'
 
-Note that the `--log-to-file` option will cause `repmgr` output to be logged to
-the destination configured to receive  log output `repmgrd`.
+Note that the `--log-to-file` option will cause `repmgr`'s output to be logged to
+the destination configured to receive log output for `repmgrd`.
 See `repmgr.conf.sample` for further `repmgrd`-specific settings
 
-
 When `failover` is set to `automatic`, upon detecting failure of the current
 master, `repmgrd` will execute one of `promote_command` or `follow_command`,
 depending on whether the current server is becoming the new master or
@@ -1461,7 +1487,7 @@ In general `repmgr` can be upgraded as-is without any further action required,
 however feature releases may require the `repmgr` database to be upgraded.
 An SQL script will be provided - please check the release notes for details:
 
-* http://repmgr.org/release-notes-3.2.1.html#UPGRADING
+* http://repmgr.org/release-notes-3.3.html#UPGRADING
 
 
 Distribution-specific configuration
@@ -1605,7 +1631,7 @@ which contains connection details for the local database.
 
     Creates a witness server as a separate PostgreSQL instance. This instance
     can be on a separate server or a server running an existing node. The
-    witness server contain a copy of the repmgr metadata tables but will not
+    witness server contains a copy of the repmgr metadata tables but will not
     be set up as a standby; instead it will update its metadata copy each
     time a failover occurs.
 

check_dir.c

@@ -1,6 +1,6 @@
 /*
  * check_dir.c - Directories management functions
- * Copyright (C) 2ndQuadrant, 2010-2016
+ * Copyright (c) 2ndQuadrant, 2010-2017
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by

check_dir.h

@@ -1,6 +1,6 @@
 /*
  * check_dir.h
- * Copyright (c) 2ndQuadrant, 2010-2016
+ * Copyright (c) 2ndQuadrant, 2010-2017
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by

compat.c

@@ -1,10 +1,12 @@
 /*
  *
  * compat.c
- *	  Provide backports of various functions not publicly
- *    exposed before PostgreSQL 9.6
+ *	  Provides a couple of useful string utility functions adapted
+ *	  from the backend code, which are not publicly exposed. They're
+ *	  unlikely to change but it would be worth keeping an eye on them
+ *	  for any fixes/improvements
  *
- * Copyright (C) 2ndQuadrant, 2010-2016
+ * Copyright (c) 2ndQuadrant, 2010-2017
  *
  * Portions Copyright (c) 1996-2013, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
@@ -24,8 +26,6 @@
  *
  */
 
-#if (PG_VERSION_NUM < 90600)
-
 #include "repmgr.h"
 #include "compat.h"
 
@@ -34,8 +34,8 @@
  * the string as a value, in a keyword/pair value in a libpq connection
  * string
  *
- * This function is copied from src/bin/pg_dump/dumputils.c
- * as it is only publicly exposed from 9.6
+ * This function is adapted from src/fe_utils/string_utils.c (before 9.6
+ * located in: src/bin/pg_dump/dumputils.c)
  */
 void
 appendConnStrVal(PQExpBuffer buf, const char *str)
@@ -79,8 +79,6 @@ appendConnStrVal(PQExpBuffer buf, const char *str)
 
 /*
  * Adapted from: src/fe_utils/string_utils.c
- *
- * Function not publicly available before PostgreSQL 9.6.
  */
 void
 appendShellString(PQExpBuffer buf, const char *str)
@@ -107,5 +105,3 @@ appendShellString(PQExpBuffer buf, const char *str)
 	appendPQExpBufferChar(buf, '\'');
 }
 
-
-#endif

compat.h

@@ -1,6 +1,6 @@
 /*
  * compat.h
- * Copyright (c) 2ndQuadrant, 2010-2016
+ * Copyright (c) 2ndQuadrant, 2010-2017
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by

config.c

@@ -1,7 +1,7 @@
 /*
  * config.c - Functions to parse the config file
  *
- * Copyright (C) 2ndQuadrant, 2010-2016
+ * Copyright (c) 2ndQuadrant, 2010-2017
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -30,7 +30,7 @@ static void tablespace_list_append(t_configuration_options *options, const char
 static void exit_with_errors(ItemList *config_errors);
 
 const static char *_progname = NULL;
-static char config_file_path[MAXPGPATH];
+static char config_file_path[MAXPGPATH] = "";
 static bool config_file_provided = false;
 bool config_file_found = false;
 
@@ -59,7 +59,7 @@ progname(void)
  * added/changed in reload_config()
  *
  * NOTE: this function is called before the logger is set up, so we need
- * to handle the verbose option ourselves; also the default log level is NOTICE,
+ * to handle the verbose option ourselves; also the default log level is INFO,
  * so we can't use DEBUG.
  */
 bool
@@ -448,6 +448,10 @@ _parse_config(t_configuration_options *options, ItemList *error_list)
 	{
 		item_list_append(error_list, _("\"node\": must be greater than zero"));
 	}
+	else if (options->node < 0)
+	{
+		item_list_append(error_list, _("\"node\": must be a positive signed 32 bit integer, i.e. 2147483647 or less"));
+	}
 
 	if (strlen(options->conninfo))
 	{

config.h

@@ -1,7 +1,7 @@
 /*
  * config.h
  *
- * Copyright (c) 2ndQuadrant, 2010-2016
+ * Copyright (c) 2ndQuadrant, 2010-2017
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by

dbutils.c

@@ -1,7 +1,7 @@
 /*
  * dbutils.c - Database connection/management functions
  *
- * Copyright (C) 2ndQuadrant, 2010-2016
+ * Copyright (c) 2ndQuadrant, 2010-2017
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -33,6 +33,15 @@ char repmgr_schema[MAXLEN] = "";
 char repmgr_schema_quoted[MAXLEN] = "";
 
 static int _get_node_record(PGconn *conn, char *cluster, char *sqlquery, t_node_info *node_info);
+static bool _set_config(PGconn *conn, const char *config_param, const char *sqlquery);
+
+/*
+ * _establish_db_connection()
+ *
+ * Connect to a database using a conninfo string.
+ *
+ * NOTE: *do not* use this for replication connections; use establish_db_connection_by_params() instead.
+ */
 
 PGconn *
 _establish_db_connection(const char *conninfo, const bool exit_on_error, const bool log_notice, const bool verbose_only)
@@ -77,6 +86,19 @@ _establish_db_connection(const char *conninfo, const bool exit_on_error, const b
 		}
 	}
 
+	/*
+	 * set "synchronous_commit" to "local" in case synchronous replication is in use
+	 */
+
+	else if (set_config(conn, "synchronous_commit", "local") == false)
+	{
+		if (exit_on_error)
+		{
+			PQfinish(conn);
+			exit(ERR_DB_CON);
+		}
+	}
+
 	return conn;
 }
 
@@ -116,8 +138,12 @@ PGconn *
 establish_db_connection_by_params(const char *keywords[], const char *values[],
 								  const bool exit_on_error)
 {
-	/* Make a connection to the database */
-	PGconn	   *conn = PQconnectdbParams(keywords, values, true);
+	PGconn	   *conn;
+	bool	    replication_connection = false;
+	int	   	    i;
+
+	/* Connect to the database using the provided parameters */
+	conn = PQconnectdbParams(keywords, values, true);
 
 	/* Check to see that the backend connection was successfully made */
 	if ((PQstatus(conn) != CONNECTION_OK))
@@ -130,6 +156,28 @@ establish_db_connection_by_params(const char *keywords[], const char *values[],
 			exit(ERR_DB_CON);
 		}
 	}
+	else
+	{
+		/*
+		 * set "synchronous_commit" to "local" in case synchronous replication is in
+		 * use (provided this is not a replication connection)
+		 */
+
+		for (i = 0; keywords[i]; i++)
+		{
+			if (strcmp(keywords[i], "replication") == 0)
+				replication_connection = true;
+		}
+
+		if (replication_connection == false && set_config(conn, "synchronous_commit", "local") == false)
+		{
+			if (exit_on_error)
+			{
+				PQfinish(conn);
+				exit(ERR_DB_CON);
+			}
+		}
+	}
 
 	return conn;
 }
@@ -274,8 +322,6 @@ is_standby(PGconn *conn)
 bool
 is_pgup(PGconn *conn, int timeout)
 {
-	char		sqlquery[QUERY_STR_LEN];
-
 	/* Check the connection status twice in case it changes after reset */
 	bool		twice = false;
 
@@ -298,8 +344,7 @@ is_pgup(PGconn *conn, int timeout)
 			if (wait_connection_availability(conn, timeout) != 1)
 				goto failed;
 
-			sqlquery_snprintf(sqlquery, "SELECT 1");
-			if (PQsendQuery(conn, sqlquery) == 0)
+			if (PQsendQuery(conn, "SELECT 1") == 0)
 			{
 				log_warning(_("PQsendQuery: Query could not be sent to primary. %s\n"),
 							PQerrorMessage(conn));
@@ -380,6 +425,8 @@ int
 get_server_version(PGconn *conn, char *server_version)
 {
 	PGresult   *res;
+	int         server_version_num;
+
 	res = PQexec(conn,
 				 "SELECT current_setting('server_version_num'), "
 				 "       current_setting('server_version')");
@@ -393,9 +440,12 @@ get_server_version(PGconn *conn, char *server_version)
 	}
 
 	if (server_version != NULL)
-		strcpy(server_version, PQgetvalue(res, 0, 0));
+		strcpy(server_version, PQgetvalue(res, 0, 1));
 
-	return atoi(PQgetvalue(res, 0, 0));
+	server_version_num = atoi(PQgetvalue(res, 0, 0));
+
+	PQclear(res);
+	return server_version_num;
 }
 
 
@@ -1084,15 +1134,25 @@ drop_replication_slot(PGconn *conn, char *slot_name)
 
 
 bool
-start_backup(PGconn *conn, char *first_wal_segment, bool fast_checkpoint)
+start_backup(PGconn *conn, char *first_wal_segment, bool fast_checkpoint, int server_version_num)
 {
 	char		sqlquery[QUERY_STR_LEN];
 	PGresult   *res;
 
-	sqlquery_snprintf(sqlquery,
-					  "SELECT pg_catalog.pg_xlogfile_name(pg_catalog.pg_start_backup('repmgr_standby_clone_%ld', %s))",
-					  time(NULL),
-					  fast_checkpoint ? "TRUE" : "FALSE");
+	if (server_version_num >= 100000)
+	{
+		sqlquery_snprintf(sqlquery,
+						  "SELECT pg_catalog.pg_walfile_name(pg_catalog.pg_start_backup('repmgr_standby_clone_%ld', %s))",
+						  time(NULL),
+						  fast_checkpoint ? "TRUE" : "FALSE");
+	}
+	else
+	{
+		sqlquery_snprintf(sqlquery,
+						  "SELECT pg_catalog.pg_xlogfile_name(pg_catalog.pg_start_backup('repmgr_standby_clone_%ld', %s))",
+						  time(NULL),
+						  fast_checkpoint ? "TRUE" : "FALSE");
+	}
 
 	log_verbose(LOG_DEBUG, "start_backup():\n%s\n", sqlquery);
 
@@ -1120,12 +1180,19 @@ start_backup(PGconn *conn, char *first_wal_segment, bool fast_checkpoint)
 
 
 bool
-stop_backup(PGconn *conn, char *last_wal_segment)
+stop_backup(PGconn *conn, char *last_wal_segment, int server_version_num)
 {
 	char		sqlquery[QUERY_STR_LEN];
 	PGresult   *res;
 
-	sqlquery_snprintf(sqlquery, "SELECT pg_catalog.pg_xlogfile_name(pg_catalog.pg_stop_backup())");
+	if (server_version_num >= 100000)
+	{
+		sqlquery_snprintf(sqlquery, "SELECT pg_catalog.pg_walfile_name(pg_catalog.pg_stop_backup())");
+	}
+	else
+	{
+		sqlquery_snprintf(sqlquery, "SELECT pg_catalog.pg_xlogfile_name(pg_catalog.pg_stop_backup())");
+	}
 
 	res = PQexec(conn, sqlquery);
 	if (PQresultStatus(res) != PGRES_TUPLES_OK)
@@ -1150,19 +1217,12 @@ stop_backup(PGconn *conn, char *last_wal_segment)
 }
 
 
+
 bool
-set_config_bool(PGconn *conn, const char *config_param, bool state)
+_set_config(PGconn *conn, const char *config_param, const char *sqlquery)
 {
-	char		sqlquery[QUERY_STR_LEN];
 	PGresult   *res;
 
-	sqlquery_snprintf(sqlquery,
-					  "SET %s TO %s",
-					  config_param,
-					  state ? "TRUE" : "FALSE");
-
-	log_verbose(LOG_DEBUG, "set_config_bool():\n%s\n", sqlquery);
-
 	res = PQexec(conn, sqlquery);
 
 	if (PQresultStatus(res) != PGRES_COMMAND_OK)
@@ -1177,6 +1237,36 @@ set_config_bool(PGconn *conn, const char *config_param, bool state)
 	return true;
 }
 
+bool
+set_config(PGconn *conn, const char *config_param,  const char *config_value)
+{
+	char		sqlquery[QUERY_STR_LEN];
+
+	sqlquery_snprintf(sqlquery,
+					  "SET %s TO '%s'",
+					  config_param,
+					  config_value);
+
+	log_verbose(LOG_DEBUG, "set_config():\n%s\n", sqlquery);
+
+	return _set_config(conn, config_param, sqlquery);
+}
+
+bool
+set_config_bool(PGconn *conn, const char *config_param, bool state)
+{
+	char		sqlquery[QUERY_STR_LEN];
+
+	sqlquery_snprintf(sqlquery,
+					  "SET %s TO %s",
+					  config_param,
+					  state ? "TRUE" : "FALSE");
+
+	log_verbose(LOG_DEBUG, "set_config_bool():\n%s\n", sqlquery);
+
+	return _set_config(conn, config_param, sqlquery);
+}
+
 
 /*
  * witness_copy_node_records()
@@ -1628,6 +1718,27 @@ create_event_record(PGconn *conn, t_configuration_options *options, int node_id,
 	return success;
 }
 
+void
+create_checkpoint(PGconn *conn)
+{
+	char		sqlquery[MAXLEN];
+	PGresult   *res;
+
+	sqlquery_snprintf(sqlquery, "CHECKPOINT");
+	log_verbose(LOG_DEBUG, "checkpoint:\n%s\n", sqlquery);
+
+	res = PQexec(conn, sqlquery);
+	if (!res || PQresultStatus(res) != PGRES_COMMAND_OK)
+	{
+		log_err(_("Unable to create CHECKPOINT:\n%s\n"),
+				PQerrorMessage(conn));
+		PQfinish(conn);
+		exit(ERR_DB_QUERY);
+	}
+
+	log_notice(_("CHECKPOINT created\n"));
+}
+
 
 bool
 update_node_record(PGconn *conn, char *action, int node, char *type, int upstream_node, char *cluster_name, char *node_name, char *conninfo, int priority, char *slot_name, bool active)
@@ -1872,7 +1983,16 @@ _get_node_record(PGconn *conn, char *cluster, char *sqlquery, t_node_info *node_
 
 	node_info->node_id = atoi(PQgetvalue(res, 0, 0));
 	node_info->type = parse_node_type(PQgetvalue(res, 0, 1));
-	node_info->upstream_node_id = atoi(PQgetvalue(res, 0, 2));
+
+	if (PQgetisnull(res, 0, 2))
+	{
+		node_info->upstream_node_id = NO_UPSTREAM_NODE;
+	}
+	else
+	{
+		node_info->upstream_node_id = atoi(PQgetvalue(res, 0, 2));
+	}
+
 	strncpy(node_info->name, PQgetvalue(res, 0, 3), MAXLEN);
 	strncpy(node_info->conninfo_str, PQgetvalue(res, 0, 4), MAXLEN);
 	strncpy(node_info->slot_name, PQgetvalue(res, 0, 5), MAXLEN);
@@ -1972,3 +2092,64 @@ get_data_checksum_version(const char *data_directory)
 
 	return (int)control_file.data_checksum_version;
 }
+
+
+
+/* ========================== */
+/* backported from repmgr 4.x */
+/* ========================== */
+
+XLogRecPtr
+parse_lsn(const char *str)
+{
+	XLogRecPtr		ptr = InvalidXLogRecPtr;
+	uint32			high,
+					low;
+
+	if (sscanf(str, "%x/%x", &high, &low) == 2)
+		ptr = (((XLogRecPtr) high) << 32) + (XLogRecPtr) low;
+
+	return ptr;
+}
+
+
+XLogRecPtr
+get_last_wal_receive_location(PGconn *conn)
+{
+	PGresult   *res = NULL;
+	XLogRecPtr	ptr = InvalidXLogRecPtr;
+
+	if (PQserverVersion(conn) >= 100000)
+	{
+		res = PQexec(conn, "SELECT pg_catalog.pg_last_wal_receive_lsn()");
+	}
+	else
+	{
+		res = PQexec(conn, "SELECT pg_catalog.pg_last_xlog_receive_location()");
+	}
+
+	if (PQresultStatus(res) == PGRES_TUPLES_OK)
+	{
+		ptr = parse_lsn(PQgetvalue(res, 0, 0));
+	}
+
+	PQclear(res);
+
+	return ptr;
+}
+
+
+bool
+is_server_available(const char *conninfo)
+{
+	PGPing		status = PQping(conninfo);
+
+	log_verbose(LOG_DEBUG, "is_server_available(): ping status for \"%s\" is %i\n", conninfo, (int)status);
+
+	if (status == PQPING_OK)
+		return true;
+
+	log_warning("is_server_available(): ping status for \"%s\" is %i\n", conninfo, (int)status);
+
+	return false;
+}

dbutils.h

@@ -1,7 +1,7 @@
 /*
  * dbutils.h
  *
- * Copyright (c) 2ndQuadrant, 2010-2016
+ * Copyright (c) 2ndQuadrant, 2010-2017
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -28,6 +28,8 @@
 #include "strutil.h"
 
 
+#define format_lsn(x) (uint32) (x >> 32), (uint32) x
+
 typedef enum {
 	UNKNOWN = 0,
 	MASTER,
@@ -122,8 +124,9 @@ char       *get_repmgr_schema_quoted(PGconn *conn);
 bool		create_replication_slot(PGconn *conn, char *slot_name, int server_version_num, PQExpBufferData *error_msg);
 int			get_slot_record(PGconn *conn, char *slot_name, t_replication_slot *record);
 bool		drop_replication_slot(PGconn *conn, char *slot_name);
-bool		start_backup(PGconn *conn, char *first_wal_segment, bool fast_checkpoint);
-bool		stop_backup(PGconn *conn, char *last_wal_segment);
+bool		start_backup(PGconn *conn, char *first_wal_segment, bool fast_checkpoint, int server_version_num);
+bool		stop_backup(PGconn *conn, char *last_wal_segment, int server_version_num);
+bool		set_config(PGconn *conn, const char *config_param,  const char *config_value);
 bool		set_config_bool(PGconn *conn, const char *config_param, bool state);
 bool		witness_copy_node_records(PGconn *masterconn, PGconn *witnessconn, char *cluster_name);
 bool		create_node_record(PGconn *conn, char *action, int node, char *type, int upstream_node, char *cluster_name, char *node_name, char *conninfo, int priority, char *slot_name, bool active);
@@ -134,9 +137,15 @@ bool        update_node_record(PGconn *conn, char *action, int node, char *type,
 bool        update_node_record_status(PGconn *conn, char *cluster_name, int this_node_id, char *type, int upstream_node_id, bool active);
 bool        update_node_record_set_upstream(PGconn *conn, char *cluster_name, int this_node_id, int new_upstream_node_id);
 bool        create_event_record(PGconn *conn, t_configuration_options *options, int node_id, char *event, bool successful, char *details);
+void        create_checkpoint(PGconn *conn);
 
 int		    get_node_replication_state(PGconn *conn, char *node_name, char *output);
 t_server_type parse_node_type(const char *type);
 int			get_data_checksum_version(const char *data_directory);
-#endif
 
+/* backported from repmgr 4.x */
+XLogRecPtr	parse_lsn(const char *str);
+XLogRecPtr	get_last_wal_receive_location(PGconn *conn);
+bool		is_server_available(const char *conninfo);
+
+#endif

dirmod.c

@@ -3,7 +3,7 @@
  * dirmod.c
  *	  directory handling functions
  *
- * Copyright (C) 2ndQuadrant, 2010-2016
+ * Copyright (c) 2ndQuadrant, 2010-2017
  *
  * Portions Copyright (c) 1996-2013, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California

dirmod.h

@@ -1,6 +1,6 @@
 /*
  * dirmod.h
- * Copyright (c) 2ndQuadrant, 2010-2016
+ * Copyright (c) 2ndQuadrant, 2010-2017
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by

docs/repmgrd-failover-mechanism.md

@@ -53,7 +53,7 @@ will be carried out:
   e.g. if a replication cluster is spread over multiple data centres, a split-brain
   situation does not occur if there is a network failure between datacentres. Note
   that if nodes are split evenly between data centres, a witness server can be
-  used to establish the "majority" daat centre.
+  used to establish the "majority" data centre.
 
 * `repmgrd` polls all visible servers and waits for each node to return a valid LSN;
   it updates the LSN previously  stored for this node if it has increased since

docs/repmgrd-node-fencing.md

@@ -49,6 +49,14 @@ the `%include` directive (available from PgBouncer 1.6) to include a separate
 configuration file, `/etc/pgbouncer.database.ini`, which will be modified by
 `repmgr`.
 
+* * *
+
+> *NOTE*: in this self-contained demonstration, `pgbouncer` is running on the
+> database servers, however in a production environment it will make more
+> sense to run `pgbouncer` on either separate nodes or the application server.
+
+* * *
+
 `/etc/pgbouncer.ini` should look something like this:
 
     [pgbouncer]
@@ -125,7 +133,7 @@ The actual script is as follows; adjust the configurable items as appropriate:
 
         psql -d $REPMGR_DB -U $REPMGR_USER -t -A \
           -c "SELECT '${PGBOUNCER_DATABASE}-ro= ' || conninfo || ' application_name=pgbouncer_${HOST}' \
-              FROM $REPMGR_SCHEMA.repl_nodes \
+              FROM ${REPMGR_SCHEMA}.repl_nodes \
               WHERE node_name='${HOST}'" >> $PGBOUNCER_DATABASE_INI_NEW
 
         rsync $PGBOUNCER_DATABASE_INI_NEW $HOST:$PGBOUNCER_DATABASE_INI

errcode.h

@@ -1,6 +1,6 @@
 /*
  * errcode.h
- * Copyright (C) 2ndQuadrant, 2010-2016
+ * Copyright (c) 2ndQuadrant, 2010-2017
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by

expected/repmgr_funcs.out

@@ -0,0 +1,18 @@
+/*
+ * repmgr_function.sql
+ * Copyright (c) 2ndQuadrant, 2010-2017
+ *
+ */
+-- SET SEARCH_PATH TO 'repmgr';
+CREATE FUNCTION repmgr_update_standby_location(text) RETURNS boolean
+AS '$libdir/repmgr_funcs', 'repmgr_update_standby_location'
+LANGUAGE C STRICT;
+CREATE FUNCTION repmgr_get_last_standby_location() RETURNS text
+AS '$libdir/repmgr_funcs', 'repmgr_get_last_standby_location'
+LANGUAGE C STRICT;
+CREATE FUNCTION repmgr_update_last_updated() RETURNS TIMESTAMP WITH TIME ZONE
+AS '$libdir/repmgr_funcs', 'repmgr_update_last_updated'
+LANGUAGE C STRICT;
+CREATE FUNCTION repmgr_get_last_updated() RETURNS TIMESTAMP WITH TIME ZONE
+AS '$libdir/repmgr_funcs', 'repmgr_get_last_updated'
+LANGUAGE C STRICT;

expected/repmgr_test.out

@@ -0,0 +1,24 @@
+select * from repmgr_update_standby_location('');
+ repmgr_update_standby_location 
+--------------------------------
+ f
+(1 row)
+
+select * from repmgr_get_last_standby_location();
+ repmgr_get_last_standby_location 
+----------------------------------
+ 
+(1 row)
+
+select * from repmgr_update_last_updated();
+ repmgr_update_last_updated 
+----------------------------
+ 
+(1 row)
+
+select * from repmgr_get_last_updated();
+ repmgr_get_last_updated 
+-------------------------
+ 
+(1 row)
+

log.c

@@ -1,6 +1,6 @@
 /*
  * log.c - Logging methods
- * Copyright (C) 2ndQuadrant, 2010-2016
+ * Copyright (c) 2ndQuadrant, 2010-2017
  *
  * This module is a set of methods for logging (currently only syslog)
  *
@@ -44,8 +44,8 @@ static void _stderr_log_with_level(const char *level_name, int level, const char
 __attribute__((format(PG_PRINTF_ATTRIBUTE, 3, 0)));
 
 int			log_type = REPMGR_STDERR;
-int			log_level = LOG_NOTICE;
-int			last_log_level = LOG_NOTICE;
+int			log_level = LOG_INFO;
+int			last_log_level = LOG_INFO;
 int			verbose_logging = false;
 int			terse_logging = false;
 /*
@@ -71,7 +71,7 @@ _stderr_log_with_level(const char *level_name, int level, const char *fmt, va_li
 
 	/*
 	 * Store the requested level so that if there's a subsequent
-	 * log_hint(), we can suppress that if appropriate.
+	 * log_hint() or log_detail(), we can suppress that if appropriate.
 	 */
 	last_log_level = level;
 
@@ -113,6 +113,20 @@ log_hint(const char *fmt, ...)
 }
 
 
+void
+log_detail(const char *fmt, ...)
+{
+	va_list		ap;
+
+	if (terse_logging == false)
+	{
+		va_start(ap, fmt);
+		_stderr_log_with_level("DETAIL", last_log_level, fmt, ap);
+		va_end(ap);
+	}
+}
+
+
 void
 log_verbose(int level, const char *fmt, ...)
 {

log.h

@@ -1,6 +1,6 @@
 /*
  * log.h
- * Copyright (c) 2ndQuadrant, 2010-2016
+ * Copyright (c) 2ndQuadrant, 2010-2017
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -126,6 +126,8 @@ bool		logger_shutdown(void);
 void		logger_set_verbose(void);
 void		logger_set_terse(void);
 
+void		log_detail(const char *fmt, ...)
+__attribute__((format(PG_PRINTF_ATTRIBUTE, 1, 2)));
 void		log_hint(const char *fmt, ...)
 __attribute__((format(PG_PRINTF_ATTRIBUTE, 1, 2)));
 void		log_verbose(int level, const char *fmt, ...)

repmgr.conf.sample

@@ -26,11 +26,14 @@
                  # the server's hostname or another identifier unambiguously
                  # associated with the server to avoid confusion
 
-# Database connection information as a conninfo string
-# This must be accessible to all servers in the cluster; for details see:
+# Database connection information as a conninfo string (this must be a
+# keyword/value string, not a connection URI).
 #
 #   https://www.postgresql.org/docs/current/static/libpq-connect.html#LIBPQ-CONNSTRING
 #
+# All servers in the cluster must be able to access the database
+# using this connection string.
+#
 #conninfo='host=192.168.204.104 dbname=repmgr user=repmgr'
 #
 # If repmgrd is in use, consider explicitly setting `connect_timeout` in the
@@ -63,8 +66,8 @@
 # -------------------------------
 
 # Log level: possible values are DEBUG, INFO, NOTICE, WARNING, ERR, ALERT, CRIT or EMERG
-# (default: NOTICE)
-#loglevel=NOTICE
+# (default: INFO)
+#loglevel=INFO
 
 # Note that logging facility settings will only apply to `repmgrd` by default;
 # `repmgr` will always write to STDERR unless the switch `--log-to-file` is
@@ -80,12 +83,6 @@
 #
 #logfile='/var/log/repmgr/repmgr.log'
 
-# By default only repmgrd log output will be written to a file,
-# if defined in "logfile"
-# enable this to restore old behaviour where output from the repmgr
-# client will be written to the logfile too
-#log_repmgr_to_file = 0
-
 # event notifications can be passed to an arbitrary external program
 # together with the following parameters:
 #
@@ -149,8 +146,15 @@
 # external command arguments. Values shown are examples.
 
 #pg_ctl_options='-s'
-#pg_basebackup_options='--xlog-method=s'
+#pg_basebackup_options='--label=repmgr_backup'
 
+# This is the host name of the barman server, which is used for connecting over
+# to the barman server (passwordless ssh keys should be in place)
+#barman_server='backup_server'
+# If you are placing the barman.conf file in a non-standard path, or using
+# a name other than barman.conf, use this parameter to specify the path and
+# name of the barman configuration file.
+#barman_config='/path/to/barman.conf'
 
 # Standby clone settings
 # ----------------------

repmgr.h

@@ -1,6 +1,6 @@
 /*
  * repmgr.h
- * Copyright (c) 2ndQuadrant, 2010-2016
+ * Copyright (c) 2ndQuadrant, 2010-2017
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -55,7 +55,6 @@
 #define OPT_COPY_EXTERNAL_CONFIG_FILES   4
 #define OPT_CONFIG_ARCHIVE_DIR           5
 #define OPT_PG_REWIND                    6
-#define OPT_PWPROMPT                     7
 #define OPT_CSV                          8
 #define OPT_NODE                         9
 #define OPT_WITHOUT_BARMAN               10
@@ -118,7 +117,7 @@ typedef struct
 
 	char		recovery_min_apply_delay[MAXLEN];
 
-	/* standby register paarameters */
+	/* standby register parameters */
 	bool		wait_register_sync;
 	int			wait_register_sync_seconds;
 
@@ -193,9 +192,10 @@ typedef struct
 {
 	char		slot[MAXLEN];
 	char		xlog_method[MAXLEN];
+	bool		no_slot; /* from PostgreSQL 10 */
 } t_basebackup_options;
 
-#define T_BASEBACKUP_OPTIONS_INITIALIZER { "", "" }
+#define T_BASEBACKUP_OPTIONS_INITIALIZER { "", "", false }
 
 typedef struct
 {

repmgr.sql

@@ -1,7 +1,7 @@
 /*
  * repmgr.sql
  *
- * Copyright (C) 2ndQuadrant, 2010-2016
+ * Copyright (c) 2ndQuadrant, 2010-2017
  *
  */
 

repmgrd.c

@@ -1,7 +1,7 @@
 /*
  * repmgrd.c - Replication manager daemon
  *
- * Copyright (C) 2ndQuadrant, 2010-2016
+ * Copyright (c) 2ndQuadrant, 2010-2017
  *
  * This module connects to the nodes of a replication cluster and monitors
  * how far are they from master
@@ -30,18 +30,10 @@
 #include <stdlib.h>
 #include <unistd.h>
 
-
-
 #include "repmgr.h"
-#include "config.h"
 #include "log.h"
-#include "strutil.h"
 #include "version.h"
 
-/* Required PostgreSQL headers */
-#include "access/xlogdefs.h"
-#include "pqexpbuffer.h"
-
 /* Message strings passed in repmgrSharedState->location */
 
 #define PASSIVE_NODE "PASSIVE_NODE"
@@ -71,6 +63,7 @@ bool		failover_done = false;
 bool		manual_mode_upstream_disconnected = false;
 
 char	   *pid_file = NULL;
+int			server_version_num = 0;
 
 static void help(void);
 static void usage(void);
@@ -145,8 +138,6 @@ main(int argc, char **argv)
 
 	FILE	   *fd;
 
-	int			server_version_num = 0;
-
 	set_progname(argv[0]);
 
 	/* Disallow running as root to prevent directory ownership problems */
@@ -523,6 +514,33 @@ main(int argc, char **argv)
 				else if (node_info.type == STANDBY)
 				{
 					log_info(_("starting continuous standby node monitoring\n"));
+
+					/*
+					 * Call update_shared_memory() so it's not stuck at 0/0; this
+					 * will otherwise cause an infinite loop on other repmgrds if
+					 * this repmgrd does not enter failover.
+					 *
+					 * NOTE: this is a temporary workaround for a structural
+					 * issue resolved through architectural redesign in repmgr 4.
+					 */
+					if (local_options.failover == MANUAL_FAILOVER)
+					{
+						update_shared_memory(PASSIVE_NODE);
+					}
+					else
+					{
+						PQExpBufferData current_lsn;
+
+						XLogRecPtr last_wal_receive_location = get_last_wal_receive_location(my_local_conn);
+
+						initPQExpBuffer(&current_lsn);
+						appendPQExpBuffer(&current_lsn, "%X/%X",
+										  format_lsn(last_wal_receive_location));
+
+						update_shared_memory(current_lsn.data);
+
+						termPQExpBuffer(&current_lsn);
+					}
 				}
 
 				do
@@ -718,26 +736,46 @@ witness_monitor(void)
 		return;
 	}
 
-	strcpy(monitor_witness_timestamp, PQgetvalue(res, 0, 0));
+	strncpy(monitor_witness_timestamp, PQgetvalue(res, 0, 0), MAXLEN);
 	PQclear(res);
 
 	/*
 	 * Build the SQL to execute on master
 	 */
-	sqlquery_snprintf(sqlquery,
-					  "INSERT INTO %s.repl_monitor "
-					  "           (primary_node, standby_node, "
-					  "            last_monitor_time, last_apply_time, "
-					  "            last_wal_primary_location, last_wal_standby_location, "
-					  "            replication_lag, apply_lag )"
-					  "      VALUES(%d, %d, "
-					  "             '%s'::TIMESTAMP WITH TIME ZONE, NULL, "
-					  "             pg_catalog.pg_current_xlog_location(), NULL, "
-					  "             0, 0) ",
-					  get_repmgr_schema_quoted(my_local_conn),
-					  master_options.node,
-					  local_options.node,
-					  monitor_witness_timestamp);
+	if (server_version_num >= 100000)
+	{
+		sqlquery_snprintf(sqlquery,
+						  "INSERT INTO %s.repl_monitor "
+						  "           (primary_node, standby_node, "
+						  "            last_monitor_time, last_apply_time, "
+						  "            last_wal_primary_location, last_wal_standby_location, "
+						  "            replication_lag, apply_lag )"
+						  "      VALUES(%d, %d, "
+						  "             '%s'::TIMESTAMP WITH TIME ZONE, NULL, "
+						  "             pg_catalog.pg_current_wal_lsn(), NULL, "
+						  "             0, 0) ",
+						  get_repmgr_schema_quoted(my_local_conn),
+						  master_options.node,
+						  local_options.node,
+						  monitor_witness_timestamp);
+	}
+	else
+	{
+		sqlquery_snprintf(sqlquery,
+						  "INSERT INTO %s.repl_monitor "
+						  "           (primary_node, standby_node, "
+						  "            last_monitor_time, last_apply_time, "
+						  "            last_wal_primary_location, last_wal_standby_location, "
+						  "            replication_lag, apply_lag )"
+						  "      VALUES(%d, %d, "
+						  "             '%s'::TIMESTAMP WITH TIME ZONE, NULL, "
+						  "             pg_catalog.pg_current_xlog_location(), NULL, "
+						  "             0, 0) ",
+						  get_repmgr_schema_quoted(my_local_conn),
+						  master_options.node,
+						  local_options.node,
+						  monitor_witness_timestamp);
+	}
 
 	/*
 	 * Execute the query asynchronously, but don't check for a result. We will
@@ -836,6 +874,8 @@ standby_monitor(void)
 			: "upstream";
 	}
 
+
+
 	/*
 	 * Check that the upstream node is still available
 	 * If not, initiate failover process
@@ -844,9 +884,7 @@ standby_monitor(void)
 	 * local_options.reconnect_interval seconds
 	 */
 
-	check_connection(&upstream_conn, upstream_node_type, upstream_conninfo);
-
-	if (PQstatus(upstream_conn) != CONNECTION_OK)
+	if (!check_connection(&upstream_conn, upstream_node_type, upstream_conninfo))
 	{
 		int previous_master_node_id = master_options.node;
 
@@ -1125,21 +1163,42 @@ standby_monitor(void)
 	 * If receive_location is less than replay location, we were streaming WAL but are
 	 *   somehow disconnected and evidently in archive recovery
 	 */
-	sqlquery_snprintf(sqlquery,
-					  " SELECT ts, "
-					  "        CASE WHEN (receive_location IS NULL OR receive_location < replay_location) "
-					  "          THEN replay_location "
-					  "          ELSE receive_location"
-					  "        END AS receive_location,"
-					  "        replay_location, "
-					  "        replay_timestamp, "
-					  "        COALESCE(receive_location, '0/0') >= replay_location AS receiving_streamed_wal "
-					  "   FROM (SELECT CURRENT_TIMESTAMP AS ts, "
-					  "         pg_catalog.pg_last_xlog_receive_location() AS receive_location, "
-					  "         pg_catalog.pg_last_xlog_replay_location()  AS replay_location, "
-					  "         pg_catalog.pg_last_xact_replay_timestamp() AS replay_timestamp "
-					  "        ) q ");
 
+	if (server_version_num >= 100000)
+	{
+		sqlquery_snprintf(sqlquery,
+						  " SELECT ts, "
+						  "        CASE WHEN (receive_location IS NULL OR receive_location < replay_location) "
+						  "          THEN replay_location "
+						  "          ELSE receive_location"
+						  "        END AS receive_location,"
+						  "        replay_location, "
+						  "        replay_timestamp, "
+						  "        COALESCE(receive_location, '0/0') >= replay_location AS receiving_streamed_wal "
+						  "   FROM (SELECT CURRENT_TIMESTAMP AS ts, "
+						  "         pg_catalog.pg_last_wal_receive_lsn()  AS receive_location, "
+						  "         pg_catalog.pg_last_wal_replay_lsn()   AS replay_location, "
+						  "         pg_catalog.pg_last_xact_replay_timestamp() AS replay_timestamp "
+						  "        ) q ");
+
+	}
+	else
+	{
+		sqlquery_snprintf(sqlquery,
+						  " SELECT ts, "
+						  "        CASE WHEN (receive_location IS NULL OR receive_location < replay_location) "
+						  "          THEN replay_location "
+						  "          ELSE receive_location"
+						  "        END AS receive_location,"
+						  "        replay_location, "
+						  "        replay_timestamp, "
+						  "        COALESCE(receive_location, '0/0') >= replay_location AS receiving_streamed_wal "
+						  "   FROM (SELECT CURRENT_TIMESTAMP AS ts, "
+						  "         pg_catalog.pg_last_xlog_receive_location() AS receive_location, "
+						  "         pg_catalog.pg_last_xlog_replay_location()  AS replay_location, "
+						  "         pg_catalog.pg_last_xact_replay_timestamp() AS replay_timestamp "
+						  "        ) q ");
+	}
 
 
 	res = PQexec(my_local_conn, sqlquery);
@@ -1151,9 +1210,9 @@ standby_monitor(void)
 		return;
 	}
 
-	strncpy(monitor_standby_timestamp, PQgetvalue(res, 0, 0), MAXLEN);
+	strncpy(monitor_standby_timestamp,  PQgetvalue(res, 0, 0), MAXLEN);
 	strncpy(last_xlog_receive_location, PQgetvalue(res, 0, 1), MAXLEN);
-	strncpy(last_xlog_replay_location, PQgetvalue(res, 0, 2), MAXLEN);
+	strncpy(last_xlog_replay_location,  PQgetvalue(res, 0, 2), MAXLEN);
 	strncpy(last_xact_replay_timestamp, PQgetvalue(res, 0, 3), MAXLEN);
 
 	receiving_streamed_wal = (strcmp(PQgetvalue(res, 0, 4), "t") == 0)
@@ -1173,7 +1232,11 @@ standby_monitor(void)
 	 * TODO: investigate whether pg_current_xlog_insert_location() would be a better
 	 * choice; see: https://github.com/2ndQuadrant/repmgr/issues/189
 	 */
-	sqlquery_snprintf(sqlquery, "SELECT pg_catalog.pg_current_xlog_location()");
+
+	if (server_version_num >= 100000)
+		sqlquery_snprintf(sqlquery, "SELECT pg_catalog.pg_current_wal_lsn()");
+	else
+		sqlquery_snprintf(sqlquery, "SELECT pg_catalog.pg_current_xlog_location()");
 
 	res = PQexec(master_conn, sqlquery);
 	if (PQresultStatus(res) != PGRES_TUPLES_OK)
@@ -1187,10 +1250,22 @@ standby_monitor(void)
 	PQclear(res);
 
 	lsn_master_current_xlog_location = lsn_to_xlogrecptr(last_wal_primary_location, NULL);
-	lsn_last_xlog_replay_location = lsn_to_xlogrecptr(last_xlog_replay_location, NULL);
 	lsn_last_xlog_receive_location = lsn_to_xlogrecptr(last_xlog_receive_location, NULL);
+	lsn_last_xlog_replay_location = lsn_to_xlogrecptr(last_xlog_replay_location, NULL);
+
+	if (lsn_last_xlog_receive_location >= lsn_last_xlog_replay_location)
+	{
+		apply_lag = (long long unsigned int)lsn_last_xlog_receive_location - lsn_last_xlog_replay_location;
+	}
+	else
+	{
+		/* This should never happen, but in case it does set apply lag to zero */
+		log_warning("Standby receive (%s) location appears less than standby replay location (%s)\n",
+					last_xlog_receive_location,
+					last_xlog_replay_location);
+		apply_lag = 0;
+	}
 
-	apply_lag = (long long unsigned int)lsn_last_xlog_receive_location - lsn_last_xlog_replay_location;
 
 	/* Calculate replication lag */
 	if (lsn_master_current_xlog_location >= lsn_last_xlog_receive_location)
@@ -1199,7 +1274,7 @@ standby_monitor(void)
 	}
 	else
 	{
-		/* This should never happen, but in case it does set lag to zero */
+		/* This should never happen, but in case it does set replication lag to zero */
 		log_warning("Master xlog (%s) location appears less than standby receive location (%s)\n",
 					last_wal_primary_location,
 					last_xlog_receive_location);
@@ -1244,8 +1319,23 @@ standby_monitor(void)
 	log_verbose(LOG_DEBUG, "standby_monitor:() %s\n", sqlquery);
 
 	if (PQsendQuery(master_conn, sqlquery) == 0)
-		log_warning(_("query could not be sent to master. %s\n"),
+	{
+		log_warning(_("query could not be sent to master: %s\n"),
 					PQerrorMessage(master_conn));
+	}
+	else
+	{
+		sqlquery_snprintf(sqlquery,
+						  "SELECT %s.repmgr_update_last_updated();",
+						  get_repmgr_schema_quoted(my_local_conn));
+		res = PQexec(my_local_conn, sqlquery);
+
+		/* not critical if the above query fails*/
+		if (PQresultStatus(res) != PGRES_TUPLES_OK)
+			log_warning(_("unable to set last_updated: %s\n"), PQerrorMessage(my_local_conn));
+
+		PQclear(res);
+	}
 }
 
 
@@ -1309,7 +1399,7 @@ do_master_failover(void)
 	}
 
 	total_active_nodes = PQntuples(res);
-	log_debug(_("%d active nodes registered\n"), total_active_nodes);
+	log_info(_("%d active nodes registered\n"), total_active_nodes);
 
 	/*
 	 * Build an array with the nodes and indicate which ones are visible and
@@ -1358,7 +1448,7 @@ do_master_failover(void)
 			 *
 			 * If the master did come back at this point, the voting algorithm should decide
 			 * it's the "best candidate" anyway and no standby will promote itself or
-			 * attempt to follow* another server.
+			 * attempt to follow another server.
 			 *
 			 * If we don't try and connect to the master here (and the code generally
 			 * assumes it's failed anyway) but it does come back any time from here
@@ -1392,8 +1482,8 @@ do_master_failover(void)
 	}
 	PQclear(res);
 
-	log_debug(_("total nodes counted: registered=%d, visible=%d\n"),
-			  total_active_nodes, visible_nodes);
+	log_info(_("total nodes counted: registered=%d, visible=%d\n"),
+			 total_active_nodes, visible_nodes);
 
 	/*
 	 * Am I on the group that should keep alive? If I see less than half of
@@ -1410,7 +1500,7 @@ do_master_failover(void)
 	/* Query all available nodes to determine readiness and LSN */
 	for (i = 0; i < total_active_nodes; i++)
 	{
-		log_debug("checking node %i...\n", nodes[i].node_id);
+		log_info("checking node %i...\n", nodes[i].node_id);
 
 		/* if the node is not visible, skip it */
 		if (!nodes[i].is_visible)
@@ -1434,27 +1524,25 @@ do_master_failover(void)
 		if (PQstatus(node_conn) != CONNECTION_OK)
 		{
 			log_err(_("It seems new problems are arising, manual intervention is needed\n"));
+			log_detail("%s\n", PQerrorMessage(node_conn));
 			terminate(ERR_FAILOVER_FAIL);
 		}
 
-		sqlquery_snprintf(sqlquery, "SELECT pg_catalog.pg_last_xlog_receive_location()");
-		res = PQexec(node_conn, sqlquery);
-		if (PQresultStatus(res) != PGRES_TUPLES_OK)
+		xlog_recptr = get_last_wal_receive_location(node_conn);
+
+		if (xlog_recptr == InvalidXLogRecPtr)
 		{
-			log_info(_("unable to retrieve node's last standby location: %s\n"),
+			log_info(_("unable to retrieve last standby location for node %i: %s\n"),
+					 nodes[i].node_id,
 					 PQerrorMessage(node_conn));
 
-			log_debug(_("connection details: %s\n"), nodes[i].conninfo_str);
-			PQclear(res);
+			log_detail(_("connection details: %s\n"), nodes[i].conninfo_str);
 			PQfinish(node_conn);
 			terminate(ERR_FAILOVER_FAIL);
 		}
 
-		xlog_recptr = lsn_to_xlogrecptr(PQgetvalue(res, 0, 0), &lsn_format_ok);
+		log_info(_("current LSN of node %i is: %X/%X\n"), nodes[i].node_id, format_lsn(xlog_recptr));
 
-		log_debug(_("LSN of node %i is: %s\n"), nodes[i].node_id, PQgetvalue(res, 0, 0));
-
-		PQclear(res);
 		PQfinish(node_conn);
 
 		/* If position is 0/0, error */
@@ -1469,7 +1557,11 @@ do_master_failover(void)
 	}
 
 	/* last we get info about this node, and update shared memory */
-	sprintf(sqlquery, "SELECT pg_catalog.pg_last_xlog_receive_location()");
+	if (server_version_num >= 100000)
+		sprintf(sqlquery, "SELECT pg_catalog.pg_last_wal_receive_lsn()");
+	else
+		sprintf(sqlquery, "SELECT pg_catalog.pg_last_xlog_receive_location()");
+
 	res = PQexec(my_local_conn, sqlquery);
 	if (PQresultStatus(res) != PGRES_TUPLES_OK)
 	{
@@ -1483,6 +1575,9 @@ do_master_failover(void)
 	}
 	/* write last location in shared memory */
 	update_shared_memory(PQgetvalue(res, 0, 0));
+
+	log_info("local node's LSN is %s\n", PQgetvalue(res, 0, 0));
+
 	PQclear(res);
 
 	/* Wait for each node to come up and report a valid LSN */
@@ -1519,6 +1614,9 @@ do_master_failover(void)
 		 */
 		if (PQstatus(node_conn) != CONNECTION_OK)
 		{
+			log_err(_("connection to node %i has gone away:\n%s\n"),
+					nodes[i].node_id,
+					PQerrorMessage(node_conn));
 			log_info(_("At this point, it could be some race conditions "
 					   "that are acceptable, assume the node is restarting "
 					   "and starting failover procedure\n"));
@@ -1535,6 +1633,9 @@ do_master_failover(void)
 			res = PQexec(node_conn, sqlquery);
 			if (PQresultStatus(res) != PGRES_TUPLES_OK)
 			{
+				/*
+				 * Note: in repmgr4 we handle this kind of situation much more gracefully.
+				 */
 				log_err(_("PQexec failed: %s.\nReport an invalid value to not "
 						  "be considered as new master and exit.\n"),
 						PQerrorMessage(node_conn));
@@ -1567,8 +1668,8 @@ do_master_failover(void)
 					 */
 					if (strcmp(location_value, PASSIVE_NODE) == 0)
 					{
-						log_debug("node %i is passive mode\n", nodes[i].node_id);
 						log_info(_("node %i will not be considered for promotion\n"), nodes[i].node_id);
+						log_detail("node %i indicates it is a passive node\n", nodes[i].node_id);
 						nodes[i].xlog_location = InvalidXLogRecPtr;
 						continue_loop = false;
 					}
@@ -1578,7 +1679,8 @@ do_master_failover(void)
 					 */
 					else if (strcmp(location_value, LSN_QUERY_ERROR) == 0)
 					{
-						log_warning(_("node %i is unable to update its shared memory and will not be considered for promotion\n"), nodes[i].node_id);
+						log_warning(_("node %i is unable to update its shared memory and will not be considered for promotion\n"),
+									nodes[i].node_id);
 						nodes[i].xlog_location = InvalidXLogRecPtr;
 						continue_loop = false;
 					}
@@ -1586,12 +1688,8 @@ do_master_failover(void)
 					/* Unable to parse value returned by `repmgr_get_last_standby_location()` */
 					else if (*location_value == '\0')
 					{
-						log_crit(
-							_("unable to obtain LSN from node %i"), nodes[i].node_id
-							);
-						log_hint(
-							_("please check that 'shared_preload_libraries=repmgr_funcs' is set in postgresql.conf\n")
-							);
+						log_crit(_("unable to obtain LSN from node %i"), nodes[i].node_id);
+						log_hint(_("please check that 'shared_preload_libraries=repmgr_funcs' is set in postgresql.conf\n"));
 
 						PQfinish(node_conn);
 						/* XXX shouldn't we just ignore this node? */
@@ -1603,14 +1701,14 @@ do_master_failover(void)
 					 * strategy keep checking
 					 */
 					else {
-						log_warning(_("unable to parse LSN \"%s\"\n"),
+						log_warning(_("unable to parse shared memory LSN \"%s\"\n"),
 									location_value);
 					}
 				}
 				else
 				{
 					log_debug(
-						_("invalid LSN returned from node %i: '%s'\n"),
+						_("invalid shared memory LSN returned from node %i: '%s'\n"),
 						nodes[i].node_id,
 						location_value);
 				}
@@ -1632,7 +1730,7 @@ do_master_failover(void)
 				nodes[i].xlog_location = xlog_recptr;
 			}
 
-			log_debug(_("LSN of node %i is: %s\n"), nodes[i].node_id, location_value);
+			log_info(_("shared memory LSN of node %i is: %s\n"), nodes[i].node_id, location_value);
 
 			ready_nodes++;
 			nodes[i].is_ready = true;
@@ -1688,7 +1786,7 @@ do_master_failover(void)
 		terminate(ERR_FAILOVER_FAIL);
 	}
 
-	log_debug("best candidate node id is %i\n", best_candidate.node_id);
+	log_info("best candidate node id is %i\n", best_candidate.node_id);
 
 	/* if local node is the best candidate, promote it */
 	if (best_candidate.node_id == local_options.node)
@@ -1704,9 +1802,9 @@ do_master_failover(void)
 		sleep(5);
 
 		log_notice(_("this node is the best candidate to be the new master, promoting...\n"));
-
-		log_debug("promote command is: \"%s\"\n",
-				  local_options.promote_command);
+		log_detail(_("LSN is %X/%X\n"), format_lsn(best_candidate.xlog_location));
+		log_info("promote command is: \"%s\"\n",
+				 local_options.promote_command);
 
 		if (log_type == REPMGR_STDERR && *local_options.logfile)
 		{
@@ -1762,6 +1860,8 @@ do_master_failover(void)
 						  node_info.node_id,
 						  failed_master.node_id);
 
+		log_notice("%s\n", event_details.data);
+
 		/* my_local_conn is now the master */
 		create_event_record(my_local_conn,
 							&local_options,
@@ -1822,7 +1922,7 @@ do_master_failover(void)
 		}
 
 
-		log_debug(_("executing follow command: \"%s\"\n"), local_options.follow_command);
+		log_notice(_("executing follow command: \"%s\"\n"), local_options.follow_command);
 
 		r = system(local_options.follow_command);
 		if (r != 0)
@@ -2040,8 +2140,11 @@ check_connection(PGconn **conn, const char *type, const char *conninfo)
 {
 	int			connection_retries;
 
+	if (conninfo != NULL && is_server_available(conninfo))
+		return true;
+
 	/*
-	 * Check if the node is still available if after
+	 * Check if the node is still available; if after
 	 * local_options.reconnect_attempts * local_options.reconnect_interval
 	 * seconds of retries we cannot reconnect return false
 	 */
@@ -2327,13 +2430,13 @@ lsn_to_xlogrecptr(char *lsn, bool *format_ok)
 	if (format_ok != NULL)
 		*format_ok = true;
 
-	return (((XLogRecPtr) xlogid * 16 * 1024 * 1024 * 255) + xrecoff);
+	return (XLogRecPtr) ((uint64) xlogid) << 32 | (uint64) xrecoff;
 }
 
 void
 usage(void)
 {
-	log_err(_("%s: Replicator manager daemon \n"), progname());
+	log_err(_("%s: replication management daemon for PostgreSQL\n"), progname());
 	log_err(_("Try \"%s --help\" for more information.\n"), progname());
 }
 

sql/Makefile

@@ -1,7 +1,7 @@
 #
 # Makefile
 #
-# Copyright (c) 2ndQuadrant, 2010-2016
+# Copyright (c) 2ndQuadrant, 2010-2017
 #
 
 MODULE_big = repmgr_funcs

sql/repmgr_funcs.sql.in

@@ -1,6 +1,6 @@
 /*
  * repmgr_function.sql
- * Copyright (c) 2ndQuadrant, 2010-2016
+ * Copyright (c) 2ndQuadrant, 2010-2017
  *
  */
 

sql/repmgr_test.sql

@@ -0,0 +1,4 @@
+select * from repmgr_update_standby_location('');
+select * from repmgr_get_last_standby_location();
+select * from repmgr_update_last_updated();
+select * from repmgr_get_last_updated();

sql/uninstall_repmgr_funcs.sql

@@ -1,6 +1,6 @@
 /*
  * uninstall_repmgr_funcs.sql
- * Copyright (c) 2ndQuadrant, 2010-2016
+ * Copyright (c) 2ndQuadrant, 2010-2017
  *
  */
 

strutil.c

@@ -1,7 +1,7 @@
 /*
  * strutil.c
  *
- * Copyright (C) 2ndQuadrant, 2010-2016
+ * Copyright (c) 2ndQuadrant, 2010-2017
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by

strutil.h

@@ -1,6 +1,6 @@
 /*
  * strutil.h
- * Copyright (C) 2ndQuadrant, 2010-2016
+ * Copyright (c) 2ndQuadrant, 2010-2017
  *
  *
  * This program is free software: you can redistribute it and/or modify

uninstall_repmgr.sql

@@ -1,7 +1,7 @@
 /*
  * uninstall_repmgr.sql
  *
- * Copyright (C) 2ndQuadrant, 2010-2016
+ * Copyright (c) 2ndQuadrant, 2010-2017
  *
  */
 

version.h

@@ -1,6 +1,6 @@
 #ifndef _VERSION_H_
 #define _VERSION_H_
 
-#define REPMGR_VERSION "3.3"
+#define REPMGR_VERSION "3.4.0"
 
 #endif