Add configuration option 'event_notification_command'

Command to be executed each time an event is logged.

Following formatting sequences will be interpolated:

      %e - event type
      %d - description
      %s - success (1 or 0)
      %t - timestamp

config.c

@@ -110,7 +110,7 @@ parse_config(const char *config_file, t_configuration_options *options)
 		return false;
 	}
 
-	/* Initialize */
+	/* Initialize configuration options with sensible defaults */
 	memset(options->cluster_name, 0, sizeof(options->cluster_name));
 	options->node = -1;
 	options->upstream_node = NO_UPSTREAM_NODE;
@@ -126,16 +126,18 @@ parse_config(const char *config_file, t_configuration_options *options)
 	memset(options->pgctl_options, 0, sizeof(options->pgctl_options));
 	memset(options->pg_basebackup_options, 0, sizeof(options->pg_basebackup_options));
 
-	/* if nothing has been provided defaults to 60 */
+	/* default master_response_timeout is 60 seconds */
 	options->master_response_timeout = 60;
 
-	/* it defaults to 6 retries with a time between retries of 10s */
+	/* default to 6 reconnection attempts at intervals of 10 seconds */
 	options->reconnect_attempts = 6;
 	options->reconnect_intvl = 10;
 
 	options->monitor_interval_secs = 2;
 	options->retry_promote_interval_secs = 300;
 
+	memset(options->event_notification_command, 0, sizeof(options->event_notification_command));
+
 	options->tablespace_mapping.head = NULL;
 	options->tablespace_mapping.tail = NULL;
 
@@ -216,6 +218,8 @@ parse_config(const char *config_file, t_configuration_options *options)
 			options->use_replication_slots = atoi(value);
 		else if (strcmp(name, "ignore_external_config_files") == 0)
 			options->ignore_external_config_files = atoi(value);
+		else if (strcmp(name, "event_notification_command") == 0)
+			strncpy(options->event_notification_command, value, MAXLEN);
 		else if (strcmp(name, "tablespace_mapping") == 0)
 			tablespace_list_append(options, value);
 		else
@@ -266,7 +270,7 @@ parse_config(const char *config_file, t_configuration_options *options)
 		exit(ERR_BAD_CONFIG);
 	}
 
-	/* The following checks are for value parameter values */
+	/* The following checks are for valid parameter values */
 	if (options->master_response_timeout <= 0)
 	{
 		log_err(_("'master_response_timeout' must be greater than zero. Check the configuration file.\n"));
@@ -279,9 +283,9 @@ parse_config(const char *config_file, t_configuration_options *options)
 		exit(ERR_BAD_CONFIG);
 	}
 
-	if (options->reconnect_intvl <= 0)
+	if (options->reconnect_intvl < 0)
 	{
-		log_err(_("'reconnect_intervals' must be zero or greater. Check the configuration file.\n"));
+		log_err(_("'reconnect_interval' must be zero or greater. Check the configuration file.\n"));
 		exit(ERR_BAD_CONFIG);
 	}
 

config.h

@@ -20,9 +20,11 @@
 #ifndef _REPMGR_CONFIG_H_
 #define _REPMGR_CONFIG_H_
 
-#include "repmgr.h"
+#include "postgres_fe.h"
+
 #include "strutil.h"
 
+
 typedef struct TablespaceListCell
 {
 	struct TablespaceListCell *next;
@@ -62,10 +64,11 @@ typedef struct
 	int			retry_promote_interval_secs;
 	int			use_replication_slots;
 	int         ignore_external_config_files;
+	char		event_notification_command[MAXLEN];
 	TablespaceList tablespace_mapping;
 }	t_configuration_options;
 
-#define T_CONFIGURATION_OPTIONS_INITIALIZER { "", -1, NO_UPSTREAM_NODE, "", MANUAL_FAILOVER, -1, "", "", "", "", "", "", "", -1, -1, -1, "", "", "", "", 0, 0, 0, 0, {NULL, NULL} }
+#define T_CONFIGURATION_OPTIONS_INITIALIZER { "", -1, NO_UPSTREAM_NODE, "", MANUAL_FAILOVER, -1, "", "", "", "", "", "", "", -1, -1, -1, "", "", "", "", 0, 0, 0, 0, "", {NULL, NULL} }
 
 
 bool		parse_config(const char *config_file, t_configuration_options *options);

dbutils.c

@@ -1060,7 +1060,7 @@ create_node_record(PGconn *conn, char *action, int node, char *type, int upstrea
 	res = PQexec(conn, sqlquery);
 	if (!res || PQresultStatus(res) != PGRES_COMMAND_OK)
 	{
-		log_warning(_("Cannot insert node details, %s\n"),
+		log_warning(_("Unable to create node record: %s\n"),
 					PQerrorMessage(conn));
 		PQclear(res);
 		return false;
@@ -1091,7 +1091,7 @@ delete_node_record(PGconn *conn, int node, char *action)
 	res = PQexec(conn, sqlquery);
 	if (!res || PQresultStatus(res) != PGRES_COMMAND_OK)
 	{
-		log_warning(_("Cannot delete node details, %s\n"),
+		log_warning(_("Unable to delete node record: %s\n"),
 					PQerrorMessage(conn));
 		PQclear(res);
 		return false;
@@ -1102,11 +1102,24 @@ delete_node_record(PGconn *conn, int node, char *action)
 }
 
 
+/*
+ * create_event_record()
+ *
+ * Insert a record into the events table.
+ *
+ * If configuration parameter `event_notification_command` is set, also
+ * attempt to execute that command.
+ *
+ * Returns true if all operations succeeded, false if one or more failed.
+ */
+
 bool
-create_event_record(PGconn *conn, int node_id, char *event, bool successful, char *details)
+create_event_record(PGconn *conn, t_configuration_options *options, int node_id, char *event, bool successful, char *details)
 {
 	char		sqlquery[QUERY_STR_LEN];
 	PGresult   *res;
+	char		event_timestamp[MAXLEN] = "";
+	bool		success = true;
 
 	int n_node_id = htonl(node_id);
 	char *t_successful = successful ? "TRUE" : "FALSE";
@@ -1131,7 +1144,8 @@ create_event_record(PGconn *conn, int node_id, char *event, bool successful, cha
 					  "             successful, "
 					  "             details "
 					  "            ) "
-					  "      VALUES ($1, $2, $3, $4) ",
+					  "      VALUES ($1, $2, $3, $4) "
+					  "   RETURNING event_timestamp ",
 					  get_repmgr_schema_quoted(conn));
 
 	res = PQexecParams(conn,
@@ -1143,13 +1157,104 @@ create_event_record(PGconn *conn, int node_id, char *event, bool successful, cha
 					   binary,
 					   0);
 
-	if (!res || PQresultStatus(res) != PGRES_COMMAND_OK)
+	if (!res || PQresultStatus(res) != PGRES_TUPLES_OK)
 	{
-		log_warning(_("Cannot insert event details, %s\n"),
+		time_t now;
+		struct tm ts;
+
+		log_warning(_("Unable to create event record: %s\n"),
 					PQerrorMessage(conn));
-		PQclear(res);
+
-		return false;
+		success = false;
+
+		/*
+		 * If the query fails for whatever reason, generate a
+		 * current timestamp ourselves. This isn't quite the same
+		 * format as PostgreSQL, but is close enough for diagnostic use.
+		 */
+		time(&now);
+		ts = *localtime(&now);
+		strftime(event_timestamp, MAXLEN, "%Y-%m-%d %H:%M:%S%z", &ts);
+	}
+	else
+	{
+		strncpy(event_timestamp, PQgetvalue(res, 0, 0), MAXLEN);
+		log_debug(_("Event timestamp is: %s\n"), event_timestamp);
 	}
 
-	return true;
+	PQclear(res);
+
+	/* an event notification command was provided - parse and execute it */
+	if(strlen(options->event_notification_command))
+	{
+		char		parsed_command[MAXPGPATH];
+		const char *src_ptr;
+		char	   *dst_ptr;
+		char	   *end_ptr;
+		int r;
+
+		dst_ptr = parsed_command;
+		end_ptr = parsed_command + MAXPGPATH - 1;
+		*end_ptr = '\0';
+
+		for(src_ptr = options->event_notification_command; *src_ptr; src_ptr++)
+		{
+			if (*src_ptr == '%')
+			{
+				switch (src_ptr[1])
+				{
+					case 'e':
+						/* %e: event type */
+						src_ptr++;
+						strlcpy(dst_ptr, event, end_ptr - dst_ptr);
+						dst_ptr += strlen(dst_ptr);
+						break;
+					case 'd':
+						/* %d: details */
+						src_ptr++;
+						if(details != NULL)
+						{
+							strlcpy(dst_ptr, details, end_ptr - dst_ptr);
+							dst_ptr += strlen(dst_ptr);
+						}
+						break;
+					case 's':
+						/* %s: successful */
+						src_ptr++;
+						strlcpy(dst_ptr, successful ? "1" : "0", end_ptr - dst_ptr);
+						dst_ptr += strlen(dst_ptr);
+						break;
+					case 't':
+						/* %: timestamp */
+						src_ptr++;
+						strlcpy(dst_ptr, event_timestamp, end_ptr - dst_ptr);
+						dst_ptr += strlen(dst_ptr);
+						break;
+					default:
+						/* otherwise treat the % as not special */
+						if (dst_ptr < end_ptr)
+							*dst_ptr++ = *src_ptr;
+						break;
+				}
+			}
+			else
+			{
+				if (dst_ptr < end_ptr)
+					*dst_ptr++ = *src_ptr;
+			}
+		}
+
+		*dst_ptr = '\0';
+
+		log_debug(_("Executing: %s\n"), parsed_command);
+
+		r = system(parsed_command);
+		if (r != 0)
+		{
+			log_warning(_("Unable to execute event notification command\n"));
+			success = false;
+		}
+	}
+
+	return success;
 }

dbutils.h

@@ -20,8 +20,10 @@
 #ifndef _REPMGR_DBUTILS_H_
 #define _REPMGR_DBUTILS_H_
 
-#include "strutil.h"
 #include "config.h"
+#include "strutil.h"
+
+
 
 PGconn *establish_db_connection(const char *conninfo,
 						const bool exit_on_error);
@@ -64,7 +66,7 @@ bool		set_config_bool(PGconn *conn, const char *config_param, bool state);
 bool		copy_configuration(PGconn *masterconn, PGconn *witnessconn, char *cluster_name);
 bool		create_node_record(PGconn *conn, char *action, int node, char *type, int upstream_node, char *cluster_name, char *node_name, char *conninfo, int priority, char *slot_name);
 bool		delete_node_record(PGconn *conn, int node, char *action);
-bool        create_event_record(PGconn *conn, int node_id, char *event, bool successful, char *details);
+bool        create_event_record(PGconn *conn, t_configuration_options *options, int node_id, char *event, bool successful, char *details);
 
 
 #endif

repmgr.c

@@ -779,6 +779,7 @@ do_master_register(void)
 
 	/* Log the event */
 	record_created = create_event_record(conn,
+										 &options,
 										 options.node,
 										 "master_register",
 										 true,
@@ -885,6 +886,7 @@ do_standby_register(void)
 
 	/* Log the event */
 	record_created = create_event_record(master_conn,
+										 &options,
 										 options.node,
 										 "standby_register",
 										 true,
@@ -1538,6 +1540,7 @@ log_event(PGconn *standby_conn, bool success, char *details)
 										    NULL, NULL);
 
 	retval = create_event_record(primary_conn,
+								 &options,
 								 options.node,
 								 "standby_clone",
 								 success,
@@ -1655,6 +1658,7 @@ do_standby_promote(void)
 						  options.node);
 
 		record_created = create_event_record(old_master_conn,
+											 &options,
 											 options.node,
 											 "standby_promote",
 											 false,
@@ -1675,6 +1679,7 @@ do_standby_promote(void)
 		log_notice(_("STANDBY PROMOTE successful.  You should REINDEX any hash indexes you have.\n"));
 		/* Log the event */
 		record_created = create_event_record(conn,
+											 &options,
 											 options.node,
 											 "standby_promote",
 											 true,
@@ -1864,6 +1869,7 @@ do_witness_create(void)
 		log_err("%s\n", errmsg.data);
 
 		create_event_record(masterconn,
+							&options,
 							options.node,
 							"witness_create",
 							false,
@@ -1885,6 +1891,7 @@ do_witness_create(void)
 		log_err("%s\n", errmsg.data);
 
 		create_event_record(masterconn,
+							&options,
 							options.node,
 							"witness_create",
 							false,
@@ -1903,6 +1910,7 @@ do_witness_create(void)
 						  runtime_options.host);
 		log_err("%s\n", errmsg.data);
 		create_event_record(masterconn,
+							&options,
 							options.node,
 							"witness_create",
 							false,
@@ -1933,6 +1941,7 @@ do_witness_create(void)
 		char *errmsg = _("unable to initialize cluster for witness server");
 		log_err("%s\n", errmsg);
 		create_event_record(masterconn,
+							&options,
 							options.node,
 							"witness_create",
 							false,
@@ -1955,6 +1964,7 @@ do_witness_create(void)
 		log_err("%s\n", errmsg.data);
 
 		create_event_record(masterconn,
+							&options,
 							options.node,
 							"witness_create",
 							false,
@@ -2000,6 +2010,7 @@ do_witness_create(void)
 		log_err("%s\n", errmsg);
 
 		create_event_record(masterconn,
+							&options,
 							options.node,
 							"witness_create",
 							false,
@@ -2025,6 +2036,7 @@ do_witness_create(void)
 			log_err("%s\n", errmsg);
 
 			create_event_record(masterconn,
+								&options,
 								options.node,
 								"witness_create",
 								false,
@@ -2050,6 +2062,7 @@ do_witness_create(void)
 			log_err("%s\n", errmsg);
 
 			create_event_record(masterconn,
+								&options,
 								options.node,
 								"witness_create",
 								false,
@@ -2069,6 +2082,7 @@ do_witness_create(void)
 		log_err("%s\n", errmsg);
 
 		create_event_record(masterconn,
+							&options,
 							options.node,
 							"witness_create",
 							false,
@@ -2085,6 +2099,7 @@ do_witness_create(void)
 		log_err("%s\n", errmsg);
 
 		create_event_record(masterconn,
+							&options,
 							options.node,
 							"witness_create",
 							false,
@@ -2106,6 +2121,7 @@ do_witness_create(void)
 		log_err("%s\n", errmsg);
 
 		create_event_record(masterconn,
+							&options,
 							options.node,
 							"witness_create",
 							false,
@@ -2131,6 +2147,7 @@ do_witness_create(void)
 	if(record_created == false)
 	{
 		create_event_record(masterconn,
+							&options,
 							options.node,
 							"witness_create",
 							false,
@@ -2148,6 +2165,7 @@ do_witness_create(void)
 	if (!create_schema(witnessconn))
 	{
 		create_event_record(masterconn,
+							&options,
 							options.node,
 							"witness_create",
 							false,
@@ -2161,6 +2179,7 @@ do_witness_create(void)
 	if (!copy_configuration(masterconn, witnessconn, options.cluster_name))
 	{
 		create_event_record(masterconn,
+							&options,
 							options.node,
 							"witness_create",
 							false,
@@ -2192,6 +2211,7 @@ do_witness_create(void)
 
 	/* Log the event */
 	record_created = create_event_record(masterconn,
+										 &options,
 										 options.node,
 										 "witness_create",
 										 true,

repmgrd.c

@@ -317,6 +317,7 @@ main(int argc, char **argv)
 				if(startup_event_logged == false)
 				{
 					create_event_record(primary_conn,
+										&local_options,
 										local_options.node,
 										"repmgrd_start",
 										true,
@@ -411,6 +412,7 @@ main(int argc, char **argv)
 				if(startup_event_logged == false)
 				{
 					create_event_record(primary_conn,
+										&local_options,
 										local_options.node,
 										"repmgrd_start",
 										true,
@@ -1360,6 +1362,7 @@ do_primary_failover(void)
 							  failed_primary.node_id);
 
 			create_event_record(my_local_conn,
+								&local_options,
 								node_info.node_id,
 								"repmgrd_failover_promote",
 								false,
@@ -1377,6 +1380,7 @@ do_primary_failover(void)
 						  failed_primary.node_id);
 
 		create_event_record(my_local_conn,
+							&local_options,
 							node_info.node_id,
 							"repmgrd_failover_promote",
 							true,
@@ -1424,6 +1428,7 @@ do_primary_failover(void)
 								  PQerrorMessage(new_primary_conn));
 
 				create_event_record(new_primary_conn,
+									&local_options,
 									node_info.node_id,
 									"repmgrd_failover_follow",
 									false,
@@ -1455,6 +1460,7 @@ do_primary_failover(void)
 							  best_candidate.node_id);
 
 			create_event_record(new_primary_conn,
+								&local_options,
 								node_info.node_id,
 								"repmgrd_failover_follow",
 								false,
@@ -1471,6 +1477,7 @@ do_primary_failover(void)
 						  best_candidate.node_id);
 
 		create_event_record(new_primary_conn,
+							&local_options,
 							node_info.node_id,
 							"repmgrd_failover_follow",
 							true,