From 701944c19488a7891999465323b4e346a5646148 Mon Sep 17 00:00:00 2001 From: Ian Barwick Date: Tue, 5 Feb 2019 13:25:21 +0900 Subject: [PATCH] "standby promote": add check for WAL replay status if replay is paused If WAL replay is paused but WAL is still pending replay, PostgreSQL will ignore the promote request until WAL replay is unpaused. This may lead to the standby being promoted at an unpredictable point in time outside of repmgr's control. Moreover it may not be obvious that this is happening, or why, and it will appear that an apparently successful promotion attempt has not actually worked. To prevent this from happening, repmgr will now refuse to promote the standy if WAL replay is paused *and* WAL is still pending replay. GitHub #540. --- dbutils.c | 50 +++++++++++ dbutils.h | 1 + doc/repmgr-standby-promote.sgml | 147 +++++++++++++++++++++++++++++--- repmgr-action-standby.c | 34 +++++++- 4 files changed, 218 insertions(+), 14 deletions(-) diff --git a/dbutils.c b/dbutils.c index abc3610a..a089523b 100644 --- a/dbutils.c +++ b/dbutils.c @@ -4946,6 +4946,7 @@ is_downstream_node_attached(PGconn *conn, char *node_name) return true; } + void set_primary_last_seen(PGconn *conn) { @@ -5000,6 +5001,55 @@ get_primary_last_seen(PGconn *conn) } +bool +is_wal_replay_paused(PGconn *conn, bool check_pending_wal) +{ + PQExpBufferData query; + PGresult *res = NULL; + bool is_paused = false; + + initPQExpBuffer(&query); + + if (PQserverVersion(conn) >= 100000) + { + appendPQExpBufferStr(&query, + "SELECT pg_catalog.pg_is_wal_replay_paused()"); + + if (check_pending_wal == true) + { + appendPQExpBufferStr(&query, + " AND pg_catalog.pg_last_wal_replay_location() < pg_catalog.pg_last_wal_receive_location()"); + } + } + else + { + appendPQExpBufferStr(&query, + "SELECT pg_catalog.pg_is_xlog_replay_paused()"); + if (check_pending_wal == true) + { + appendPQExpBufferStr(&query, + " AND pg_catalog.pg_last_xlog_replay_location() < pg_catalog.pg_last_xlog_receive_location()"); + } + } + + res = PQexec(conn, query.data); + + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + log_db_error(conn, query.data, _("unable to execute \"%s\""), query.data); + } + else + { + is_paused = atobool(PQgetvalue(res, 0, 0)); + } + + termPQExpBuffer(&query); + PQclear(res); + + return is_paused; +} + + /* ============= */ /* BDR functions */ /* ============= */ diff --git a/dbutils.h b/dbutils.h index a0fa3f54..568ba9fa 100644 --- a/dbutils.h +++ b/dbutils.h @@ -548,6 +548,7 @@ void get_node_replication_stats(PGconn *conn, t_node_info *node_info); bool is_downstream_node_attached(PGconn *conn, char *node_name); void set_primary_last_seen(PGconn *conn); int get_primary_last_seen(PGconn *conn); +bool is_wal_replay_paused(PGconn *conn, bool check_pending_wal); /* BDR functions */ int get_bdr_version_num(void); diff --git a/doc/repmgr-standby-promote.sgml b/doc/repmgr-standby-promote.sgml index b0598b5b..1bd968c5 100644 --- a/doc/repmgr-standby-promote.sgml +++ b/doc/repmgr-standby-promote.sgml @@ -33,21 +33,21 @@ Both values can be defined in repmgr.conf. + + + If WAL replay is paused on the standby, and not all WAL files on the standby have been + replayed, &repmgr; will not attempt to promote it. + + + This is because if WAL replay is paused, PostgreSQL itself will not react to a promote command + until WAL replay is resumed and all pending WAL has been replayed. This means + attempting to promote PostgreSQL in this state will leave PostgreSQL in a condition where the + promotion may occur at a unpredictable point in the future. + + + - - Options - - - - - - Check if this node can be promoted, but don't carry out the promotion - - - - - Example @@ -64,6 +64,127 @@ + + Options + + + + + + Check if this node can be promoted, but don't carry out the promotion + + + + + + + + Configuration file settings + + The following parameters in repmgr.conf are relevant to the + promote operation: + + + + + + + + promote_check_interval + with "repmgr standby promote " + + + promote_check_interval: + interval (in seconds, default: 1 seconds) to wait between each check + to determine whether the standby has been promoted. + + + + + + promote_check_timeout + with "repmgr standby promote " + + + promote_check_timeout: + time (in seconds, default: 60 seconds) to wait to verify that the standby has been promoted + before exiting with ERR_PROMOTION_FAIL. + + + + + + + + + + Exit codes + + Following exit codes can be emitted by repmgr standby promote: + + + + + + + The standby was successfully promoted to primary. + + + + + + + + + &repmgr; was unable to connect to the local PostgreSQL node. + + + PostgreSQL must be running before the node can be promoted. + + + + + + + + + The node could not be promoted to primary for one of the following + reasons: + + + + + there is an existing primary node in the replication cluster + + + + + + the node is not a standby + + + + + + WAL replay is paused on the node + + + + + + execution of the PostgreSQL promote command failed + + + + + + + + + + + + Event notifications diff --git a/repmgr-action-standby.c b/repmgr-action-standby.c index 4763987c..c58a03bf 100644 --- a/repmgr-action-standby.c +++ b/repmgr-action-standby.c @@ -1974,7 +1974,7 @@ do_standby_promote(void) { log_error(_("STANDBY PROMOTE can only be executed on a standby node")); PQfinish(conn); - exit(ERR_BAD_CONFIG); + exit(ERR_PROMOTION_FAIL); } else { @@ -1988,6 +1988,38 @@ do_standby_promote(void) log_info(_("node is a standby")); } + /* + * Executing "pg_ctl ... promote" when WAL replay is paused and + * WAL is pending replay will mean the standby will not promote + * until replay is resumed. + * + * As that could happen at any time outside repmgr's control, we + * need to avoid leaving a "ticking timebomb" which might cause + * an unexpected status change in the replication cluster. + */ + if (is_wal_replay_paused(conn, true) == true) + { + ReplInfo replication_info; + + init_replication_info(&replication_info); + + log_error(_("WAL replay is paused on this node but not all WAL has been replayed")); + + if (get_replication_info(conn, &replication_info) == true) + { + log_detail(_("replay paused at %X/%X; last WAL received is %X/%X"), + format_lsn(replication_info.last_wal_replay_lsn), + format_lsn(replication_info.last_wal_receive_lsn)); + } + + if (PQserverVersion(conn) >= 100000) + log_hint(_("execute \"pg_wal_replay_resume()\" to unpause WAL replay")); + else + log_hint(_("execute \"pg_xlog_replay_resume()\" to unpause WAL replay")); + PQfinish(conn); + exit(ERR_PROMOTION_FAIL); + } + /* check that there's no existing primary */ current_primary_conn = get_primary_connection_quiet(conn, &existing_primary_id, NULL);