github/repmgr
1
0
Fork 0
mirror of https://github.com/EnterpriseDB/repmgr.git synced 2026-03-22 22:56:29 +00:00
Code Issues Packages Projects Releases 1 Wiki Activity

"standby promote": add check for WAL replay status if replay is paused

Browse Source 
If WAL replay is paused but WAL is still pending replay, PostgreSQL will ignore
the promote request until WAL replay is unpaused. This may lead to the standby
being promoted at an unpredictable point in time outside of repmgr's
control. Moreover it may not be obvious that this is happening, or why, and
it will appear that an apparently successful promotion attempt has not
actually worked.

To prevent this from happening, repmgr will now refuse to promote the
standy if WAL replay is paused *and* WAL is still pending replay.

GitHub #540.
This commit is contained in:
Ian Barwick
2019-02-05 13:25:21 +09:00
parent d8048060a2
commit 701944c194
4 changed files with 218 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
dbutils.c
View File

@@ -4946,6 +4946,7 @@ is_downstream_node_attached(PGconn *conn, char *node_name)
return true;
}
void
set_primary_last_seen(PGconn *conn)
{
@@ -5000,6 +5001,55 @@ get_primary_last_seen(PGconn *conn)
}
bool
is_wal_replay_paused(PGconn *conn, bool check_pending_wal)
{
PQExpBufferData query;
PGresult *res = NULL;
bool is_paused = false;
initPQExpBuffer(&query);
if (PQserverVersion(conn) >= 100000)
{
appendPQExpBufferStr(&query,
"SELECT pg_catalog.pg_is_wal_replay_paused()");
if (check_pending_wal == true)
{
appendPQExpBufferStr(&query,
" AND pg_catalog.pg_last_wal_replay_location() < pg_catalog.pg_last_wal_receive_location()");
}
}
else
{
appendPQExpBufferStr(&query,
"SELECT pg_catalog.pg_is_xlog_replay_paused()");
if (check_pending_wal == true)
{
appendPQExpBufferStr(&query,
" AND pg_catalog.pg_last_xlog_replay_location() < pg_catalog.pg_last_xlog_receive_location()");
}
}
res = PQexec(conn, query.data);
if (PQresultStatus(res) != PGRES_TUPLES_OK)
{
log_db_error(conn, query.data, _("unable to execute \"%s\""), query.data);
}
else
{
is_paused = atobool(PQgetvalue(res, 0, 0));
}
termPQExpBuffer(&query);
PQclear(res);
return is_paused;
}
/* ============= */
/* BDR functions */
/* ============= */

1
dbutils.h
View File

@@ -548,6 +548,7 @@ void get_node_replication_stats(PGconn *conn, t_node_info *node_info);
bool is_downstream_node_attached(PGconn *conn, char *node_name);
void set_primary_last_seen(PGconn *conn);
int get_primary_last_seen(PGconn *conn);
bool is_wal_replay_paused(PGconn *conn, bool check_pending_wal);
/* BDR functions */
int get_bdr_version_num(void);

147
doc/repmgr-standby-promote.sgml
View File

@@ -33,21 +33,21 @@
Both values can be defined in <filename>repmgr.conf</filename>.
</para>
<note>
<para>
If WAL replay is paused on the standby, and not all WAL files on the standby have been
replayed, &repmgr; will not attempt to promote it.
</para>
<para>
This is because if WAL replay is paused, PostgreSQL itself will not react to a promote command
until WAL replay is resumed and all pending WAL has been replayed. This means
attempting to promote PostgreSQL in this state will leave PostgreSQL in a condition where the
promotion may occur at a unpredictable point in the future.
</para>
</note>
</refsect1>
<refsect1>
<title>Options</title>
<variablelist>
<varlistentry>
<term><option>--dry-run</option></term>
<listitem>
<para>
Check if this node can be promoted, but don't carry out the promotion
</para>
</listitem>
</varlistentry>
</variablelist>
</refsect1>
<refsect1>
<title>Example</title>
@@ -64,6 +64,127 @@
</refsect1>
<refsect1>
<title>Options</title>
<variablelist>
<varlistentry>
<term><option>--dry-run</option></term>
<listitem>
<para>
Check if this node can be promoted, but don't carry out the promotion
</para>
</listitem>
</varlistentry>
</variablelist>
</refsect1>
<refsect1>
<title>Configuration file settings</title>
<para>
The following parameters in <filename>repmgr.conf</filename> are relevant to the
promote operation:
</para>
<para>
<itemizedlist spacing="compact" mark="bullet">
<listitem>
<indexterm>
<primary>promote_check_interval</primary>
<secondary>with &quot;repmgr standby promote &quot;</secondary>
</indexterm>
<simpara>
<literal>promote_check_interval</literal>:
interval (in seconds, default: 1 seconds) to wait between each check
to determine whether the standby has been promoted.
</simpara>
</listitem>
<listitem>
<indexterm>
<primary>promote_check_timeout</primary>
<secondary>with &quot;repmgr standby promote &quot;</secondary>
</indexterm>
<simpara>
<literal>promote_check_timeout</literal>:
time (in seconds, default: 60 seconds) to wait to verify that the standby has been promoted
before exiting with <literal>ERR_PROMOTION_FAIL</literal>.
</simpara>
</listitem>
</itemizedlist>
</para>
</refsect1>
<refsect1>
<title>Exit codes</title>
<para>
Following exit codes can be emitted by <command>repmgr standby promote</command>:
</para>
<variablelist>
<varlistentry>
<term><option>SUCCESS (0)</option></term>
<listitem>
<para>
The standby was successfully promoted to primary.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term><option>ERR_DB_CONN (6)</option></term>
<listitem>
<para>
&repmgr; was unable to connect to the local PostgreSQL node.
</para>
<para>
PostgreSQL must be running before the node can be promoted.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term><option>ERR_PROMOTION_FAIL (8)</option></term>
<listitem>
<para>
The node could not be promoted to primary for one of the following
reasons:
<itemizedlist spacing="compact" mark="bullet">
<listitem>
<simpara>
there is an existing primary node in the replication cluster
</simpara>
</listitem>
<listitem>
<simpara>
the node is not a standby
</simpara>
</listitem>
<listitem>
<simpara>
WAL replay is paused on the node
</simpara>
</listitem>
<listitem>
<simpara>
execution of the PostgreSQL promote command failed
</simpara>
</listitem>
</itemizedlist>
</para>
</listitem>
</varlistentry>
</variablelist>
</refsect1>
<refsect1 id="repmgr-standby-promote-events">
<title>Event notifications</title>
<para>

34
repmgr-action-standby.c
View File

@@ -1974,7 +1974,7 @@ do_standby_promote(void)
{
log_error(_("STANDBY PROMOTE can only be executed on a standby node"));
PQfinish(conn);
exit(ERR_BAD_CONFIG);
exit(ERR_PROMOTION_FAIL);
}
else
{
@@ -1988,6 +1988,38 @@ do_standby_promote(void)
log_info(_("node is a standby"));
}
/*
* Executing "pg_ctl ... promote" when WAL replay is paused and
* WAL is pending replay will mean the standby will not promote
* until replay is resumed.
*
* As that could happen at any time outside repmgr's control, we
* need to avoid leaving a "ticking timebomb" which might cause
* an unexpected status change in the replication cluster.
*/
if (is_wal_replay_paused(conn, true) == true)
{
ReplInfo replication_info;
init_replication_info(&replication_info);
log_error(_("WAL replay is paused on this node but not all WAL has been replayed"));
if (get_replication_info(conn, &replication_info) == true)
{
log_detail(_("replay paused at %X/%X; last WAL received is %X/%X"),
format_lsn(replication_info.last_wal_replay_lsn),
format_lsn(replication_info.last_wal_receive_lsn));
}
if (PQserverVersion(conn) >= 100000)
log_hint(_("execute \"pg_wal_replay_resume()\" to unpause WAL replay"));
else
log_hint(_("execute \"pg_xlog_replay_resume()\" to unpause WAL replay"));
PQfinish(conn);
exit(ERR_PROMOTION_FAIL);
}
/* check that there's no existing primary */
current_primary_conn = get_primary_connection_quiet(conn, &existing_primary_id, NULL);