github/repmgr
1
0
Fork 0
mirror of https://github.com/EnterpriseDB/repmgr.git synced 2026-03-22 22:56:29 +00:00
Code Issues Packages Projects Releases 1 Wiki Activity

"node rejoin": actively check for node to rejoin cluster

Browse Source 
Previously repmgr was relying on whatever command was configured to
start PostgreSQL to determine whether the node being rejoined had
started correctly. However it's preferable to actively poll the upstream
to confirm it has restarted and actually attached as a standby before
confirming success of the "node rejoin" action.

This can be overridden with the -W/--no-wait option.

(Note that for consistency with other PostgreSQL utilities, the
short form of the --wait option is now "-w"; this is currently
only used in "repmgr standby follow".)

Also update "repmgr node rejoin" documentation with a list of supported
options, and add some useful index entries for "pg_rewind".

Implements GitHub #415.
This commit is contained in:
Ian Barwick
2018-04-03 09:13:32 +09:00
committed by Ian Barwick
parent 1ab16bc6c2
commit 23c99304a6
8 changed files with 234 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

80
doc/repmgr-node-rejoin.sgml
View File

@@ -45,6 +45,77 @@
</para>
</refsect1>
<refsect1>
<title>Options</title>
<variablelist>
<varlistentry>
<term><option>--dry-run</option></term>
<listitem>
<para>
Check prerequisites but don't actually execute the rejoin.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term><option>--force-rewind[=/path/to/pg_rewind]</option></term>
<listitem>
<para>
Execute <application>pg_rewind</application> if necessary.
</para>
<para>
It is only necessary to provide the <application>pg_rewind</application>
if using PostgreSQL 9.3 or 9.4, and <application>pg_rewind</application>
is not installed in the PostgreSQL <filename>bin</filename> directory.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term><option>--config-files</option></term>
<listitem>
<para>
comma-separated list of configuration files to retain after
executing <application>pg_rewind</application>.
</para>
<para>
Currently <application>pg_rewind</application> will overwrite
the local node's configuration files with the files from the source node,
so it's advisable to use this option to ensure they are kept.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term><option>--config-archive-dir</option></term>
<listitem>
<para>
Directory to temporarily store configuration files specified with
<option>--config-files</option>; default: <filename>/tmp</filename>.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term><option>-W/--no-wait</option></term>
<listitem>
<para>
Don't wait for the node to rejoin cluster.
</para>
<para>
If this option is supplied, &repmgr; will restart the node but
not wait for it to connect to the primary.
</para>
</listitem>
</varlistentry>
</variablelist>
</refsect1>
<refsect1>
<title>Event notifications</title>
<para>
@@ -77,11 +148,18 @@
</refsect1>
<refsect1 id="repmgr-node-rejoin-pg-rewind" xreflabel="Using pg_rewind">
<indexterm>
<primary>pg_rewind</primary>
<secondary>using with "repmgr node rejoin"</secondary>
</indexterm>
<title>Using <command>pg_rewind</command></title>
<para>
<command>repmgr node rejoin</command> can optionally use <command>pg_rewind</command> to re-integrate a
node which has diverged from the rest of the cluster, typically a failed primary.
<command>pg_rewind</command> is available in PostgreSQL 9.5 and later.
<command>pg_rewind</command> is available in PostgreSQL 9.5 and later as part of the core distribution,
and can be installed from external sources for PostgreSQL 9.3 and 9.4.
</para>
<note>
<para>

2
doc/repmgr-standby-follow.sgml
View File

@@ -71,7 +71,7 @@
</varlistentry>
<varlistentry>
<term><option>-W</option></term>
<term><option>-w</option></term>
<term><option>--wait</option></term>
<listitem>
<para>

6
doc/switchover.sgml
View File

@@ -180,6 +180,10 @@
</note>
<sect2 id="switchover-pg-rewind" xreflabel="Switchover and pg_rewind">
<indexterm>
<primary>pg_rewind</primary>
<secondary>using with "repmgr standby switchover"</secondary>
</indexterm>
<title>Switchover and pg_rewind</title>
<para>
If the demotion candidate does not shut down smoothly or cleanly, there's a risk it
@@ -206,7 +210,7 @@
</para>
<para>
<application>pg_rewind</application> has been part of the core PostgreSQL distribution since
version 9.5. Users of versions 9.3 and 9.4 will need to manually install; source code available here:
version 9.5. Users of versions 9.3 and 9.4 will need to manually install it; the source code is available here:
<ulink url="https://github.com/vmware/pg_rewind">https://github.com/vmware/pg_rewind</ulink>.
If the <application>pg_rewind</application>
binary is not installed in the PostgreSQL <filename>bin</filename> directory, provide

1
errcode.h
View File

@@ -45,5 +45,6 @@
#define ERR_OUT_OF_MEMORY 21
#define ERR_SWITCHOVER_INCOMPLETE 22
#define ERR_FOLLOW_FAIL 23
#define ERR_REJOIN_FAIL 24
#endif /* _ERRCODE_H_ */

112
repmgr-action-node.c
View File

@@ -1758,7 +1758,17 @@ do_node_rejoin(void)
PQfinish(upstream_conn);
/* connect to registered primary and check it's not in recovery */
upstream_conn = establish_db_connection(primary_node_record.conninfo, true);
upstream_conn = establish_db_connection(primary_node_record.conninfo, false);
if (PQstatus(upstream_conn) != CONNECTION_OK)
{
log_error(_("unable to connect to current primary \"%s\" (node ID: %i)"),
primary_node_record.node_name,
primary_node_record.node_id);
log_detail(_("primay node conninfo is: \"%s\""),
primary_node_record.conninfo);
exit(ERR_BAD_CONFIG);
}
upstream_recovery_type = get_recovery_type(upstream_conn);
@@ -1992,22 +2002,99 @@ do_node_rejoin(void)
}
/*
* XXX add checks that node actually started and connected to primary,
* if not exit with ERR_REJOIN_FAIL
* Actively check that node actually started and connected to primary,
* if not exit with ERR_REJOIN_FAIL.
*
* This check can be overridden with -W/--no-wait, in which case a one-time
* check will be carried out.
*/
if (runtime_options.no_wait == false)
{
int i;
create_event_notification(upstream_conn,
&config_file_options,
config_file_options.node_id,
"node_rejoin",
success,
follow_output.data);
for (i = 0; i < config_file_options.standby_reconnect_timeout; i++)
{
if (is_server_available(config_file_options.conninfo))
{
log_verbose(LOG_INFO, _("demoted primary is pingable"));
break;
}
PQfinish(upstream_conn);
if (i % 5 == 0)
{
log_verbose(LOG_INFO, _("waiting for node %i to respond to pings; %i of max %i attempts"),
config_file_options.node_id,
i + 1, config_file_options.standby_reconnect_timeout);
}
else
{
log_debug("sleeping 1 second waiting for node %i to respond to pings; %i of max %i attempts",
config_file_options.node_id,
i + 1, config_file_options.standby_reconnect_timeout);
}
log_notice(_("NODE REJOIN successful"));
log_detail("%s", follow_output.data);
sleep(1);
}
for (; i < config_file_options.standby_reconnect_timeout; i++)
{
success = is_downstream_node_attached(upstream_conn, config_file_options.node_name);
if (success == true)
{
log_verbose(LOG_INFO, _("node %i has attached to its upstream node"),
config_file_options.node_id);
break;
}
if (i % 5 == 0)
{
log_info(_("waiting for node %i to connect to new primary; %i of max %i attempts"),
config_file_options.node_id,
i + 1, config_file_options.standby_reconnect_timeout);
}
else
{
log_debug("sleeping 1 second waiting for node %i to connect to new primary; %i of max %i attempts",
config_file_options.node_id,
i + 1, config_file_options.standby_reconnect_timeout);
}
sleep(1);
}
create_event_notification(upstream_conn,
&config_file_options,
config_file_options.node_id,
"node_rejoin",
success,
follow_output.data);
if (success == false)
{
termPQExpBuffer(&follow_output);
log_notice(_("NODE REJOIN failed"));
exit(ERR_REJOIN_FAIL);
}
}
else
{
success = is_downstream_node_attached(upstream_conn, config_file_options.node_name);
}
if (success == true)
{
log_notice(_("NODE REJOIN successful"));
log_detail("%s", follow_output.data);
}
else
{
/*
* if we reach here, no record found in upstream node's pg_stat_replication */
log_notice(_("NODE REJOIN has completed but node is not yet reattached to upstream"));
log_hint(_("you will need to manually check the node's replication status"));
}
termPQExpBuffer(&follow_output);
return;
@@ -2474,6 +2561,7 @@ do_node_help(void)
" after executing \"pg_rewind\"\n"));
printf(_(" --config-archive-dir directory to temporarily store retained configuration files\n" \
" (default: /tmp)\n"));
printf(_(" -W/--no-wait don't wait for the node to rejoin cluster\n"));
puts("");
printf(_("NODE SERVICE\n"));

3
repmgr-client-global.h
View File

@@ -42,6 +42,7 @@ typedef struct
bool force;
char pg_bindir[MAXLEN]; /* overrides setting in repmgr.conf */
bool wait;
bool no_wait;
/* logging options */
char log_level[MAXLEN]; /* overrides setting in repmgr.conf */
@@ -134,7 +135,7 @@ typedef struct
/* configuration metadata */ \
false, false, false, false, \
/* general configuration options */ \
"", false, false, "", false, \
"", false, false, "", false, false, \
/* logging options */ \
"", false, false, false, \
/* output options */ \

46
repmgr-client.c
View File

@@ -178,7 +178,7 @@ main(int argc, char **argv)
strncpy(runtime_options.username, pw->pw_name, MAXLEN);
}
while ((c = getopt_long(argc, argv, "?Vb:f:FWd:h:p:U:R:S:D:ck:L:tvC:", long_options,
while ((c = getopt_long(argc, argv, "?Vb:f:FwWd:h:p:U:R:S:D:ck:L:tvC:", long_options,
&optindex)) != -1)
{
/*
@@ -243,11 +243,16 @@ main(int argc, char **argv)
strncpy(runtime_options.replication_user, optarg, MAXLEN);
break;
/* -W/--wait */
case 'W':
/* -w/--wait */
case 'w':
runtime_options.wait = true;
break;
/* -W/--no-wait */
case 'W':
runtime_options.no_wait = true;
break;
/*----------------------------
* database connection options
*----------------------------
@@ -1571,6 +1576,41 @@ check_cli_parameters(const int action)
}
}
/* --wait/--no-wait */
if (runtime_options.wait == true && runtime_options.no_wait == true)
{
item_list_append_format(&cli_errors,
_("both --wait and --no-wait options provided"));
}
else
{
if (runtime_options.wait)
{
switch (action)
{
case STANDBY_FOLLOW:
break;
default:
item_list_append_format(&cli_warnings,
_("--wait will be ignored when executing %s"),
action_name(action));
}
}
else if (runtime_options.wait)
{
switch (action)
{
case NODE_REJOIN:
break;
default:
item_list_append_format(&cli_warnings,
_("--no-wait will be ignored when executing %s"),
action_name(action));
}
}
}
/* repmgr node service --action */
if (runtime_options.action[0] != '\0')
{

4
repmgr-client.h
View File

@@ -86,6 +86,7 @@
#define OPT_REPL_CONN 1037
#define OPT_REMOTE_NODE_ID 1038
#define OPT_RECOVERY_CONF_ONLY 1039
#define OPT_NO_WAIT 1040
/* deprecated since 3.3 */
#define OPT_DATA_DIR 999
@@ -104,7 +105,8 @@ static struct option long_options[] =
{"dry-run", no_argument, NULL, OPT_DRY_RUN},
{"force", no_argument, NULL, 'F'},
{"pg_bindir", required_argument, NULL, 'b'},
{"wait", no_argument, NULL, 'W'},
{"wait", no_argument, NULL, 'w'},
{"no-wait", no_argument, NULL, 'W'},
/* connection options */
{"dbname", required_argument, NULL, 'd'},