github/repmgr
1
0
Fork 0
mirror of https://github.com/EnterpriseDB/repmgr.git synced 2026-03-24 15:46:29 +00:00
Code Issues Packages Projects Releases 1 Wiki Activity

Add "cluster diagnose" mode

Browse Source 
This mode merges the output of "cluster matrix" from each node to
improve node state knowledge.
This commit is contained in:
Gianni Ciolli
2016-06-21 01:02:44 +02:00
committed by Ian Barwick
parent 263128a740
commit 5189488b92
2 changed files with 226 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

57
README.md
View File

@@ -1601,13 +1601,19 @@ which contains connection details for the local database.
The first column is the node's ID, and the second column represents the
node's status (0 = master, 1 = standby, -1 = failed).
* `cluster matrix`
* `cluster matrix` and `cluster diagnose`
Displays connection information for each pair of nodes in the
replication cluster. This command polls each registered server and
asks it to connect to each other node.
These commands display connection information for each pair of
nodes in the replication cluster.
This command requires a valid `repmgr.conf` file on each node.
- `cluster matrix` polls each registered server and asks it to
connect to each other node;
- `cluster diagnose` runs a `cluster matrix` on each node and
combines the results in a single matrix.
These commands require a valid `repmgr.conf` file on each node, and
the optional `ssh_hostname` parameter must be set.
Example 1 (all nodes up):
@@ -1619,6 +1625,10 @@ which contains connection details for the local database.
node2 | 2 | * | * | *
node3 | 3 | * | * | *
Here `cluster matrix` is sufficient to establish the state of each
possible connection.
Example 2 (node1 and node2 up, node3 down):
$ repmgr -f /etc/repmgr.conf cluster matrix
@@ -1641,20 +1651,45 @@ which contains connection details for the local database.
node1 and node2, meaning that inbound connections to these nodes
have succeeded.
Example 3 (all nodes up, firewall dropping packets originating
from node2 and directed to port 5432 on node3)
In this case, `cluster diagnose` gives the same result as `cluster
matrix`, because from any functioning node we can observe the same
state: node1 and node2 are up, node3 is down.
After a long wait (same as before plus two timeouts, by default
one minute each), you will see the following output:
Example 3 (all nodes up, firewall dropping packets originating
from node1 and directed to port 5432 on node3)
Running `cluster matrix` from node1 gives the following output,
after a long wait (two timeouts, by default one minute each):
$ repmgr -f /etc/repmgr.conf cluster matrix
Name | Id | 1 | 2 | 3
-------+----+----+----+----
node1 | 1 | * | * | *
node2 | 2 | * | * | x
node1 | 1 | * | * | x
node2 | 2 | * | * | *
node3 | 3 | ? | ? | ?
The matrix tells us that we cannot connect from node1 to node3,
and that (therefore) we don't know the state of any outbound
connection from node3.
In this case, the `cluster diagnose` command is more informative:
$ repmgr -f /etc/repmgr.conf cluster diagnose
Name | Id | 1 | 2 | 3
-------+----+----+----+----
node1 | 1 | * | * | x
node2 | 2 | * | * | *
node3 | 3 | * | * | *
What happened is that `cluster diagnose` merged its own `cluster
matrix` with the `cluster matrix` output from node2; the latter is
able to connect to node3 and therefore determine the state of
outbound connections from that node.
* `cluster cleanup`
Purges monitoring history from the `repl_monitor` table to prevent excessive

180
repmgr.c
View File

@@ -21,6 +21,7 @@
* WITNESS REGISTER
* WITNESS UNREGISTER
*
* CLUSTER DIAGNOSE
* CLUSTER MATRIX
* CLUSTER SHOW
* CLUSTER CLEANUP
@@ -90,6 +91,7 @@
#define CLUSTER_SHOW 13
#define CLUSTER_CLEANUP 14
#define CLUSTER_MATRIX 15
#define CLUSTER_DIAGNOSE 16
static int test_ssh_connection(char *host, char *remote_user);
static int copy_remote_files(char *host, char *remote_user, char *remote_path,
@@ -132,6 +134,7 @@ static void do_witness_unregister(void);
static void do_cluster_show(void);
static void do_cluster_matrix(void);
static void do_cluster_diagnose(void);
static void do_cluster_cleanup(void);
static void do_check_upstream_config(void);
static void do_help(void);
@@ -713,6 +716,8 @@ main(int argc, char **argv)
action = CLUSTER_SHOW;
else if (strcasecmp(server_cmd, "CLEANUP") == 0)
action = CLUSTER_CLEANUP;
else if (strcasecmp(server_cmd, "DIAGNOSE") == 0)
action = CLUSTER_DIAGNOSE;
else if (strcasecmp(server_cmd, "MATRIX") == 0)
action = CLUSTER_MATRIX;
}
@@ -954,6 +959,9 @@ main(int argc, char **argv)
case WITNESS_UNREGISTER:
do_witness_unregister();
break;
case CLUSTER_DIAGNOSE:
do_cluster_diagnose();
break;
case CLUSTER_MATRIX:
do_cluster_matrix();
break;
@@ -1282,6 +1290,178 @@ do_cluster_matrix(void)
PQclear(res);
}
static void
do_cluster_diagnose(void)
{
PGconn *conn;
PGresult *res;
char sqlquery[QUERY_STR_LEN];
int i, j, k;
const char *node_header = "Name";
int name_length = strlen(node_header);
int x, y, z, u, v;
int n = 0; /* number of nodes */
int *cube;
char *p;
char c;
char command[MAXLEN];
PQExpBufferData command_output;
/* We need to connect to get the list of nodes */
log_info(_("connecting to database\n"));
conn = establish_db_connection(options.conninfo, true);
sqlquery_snprintf(sqlquery,
"SELECT conninfo, ssh_hostname, type, name, upstream_node_name, id"
" FROM %s.repl_show_nodes ORDER BY id",
get_repmgr_schema_quoted(conn));
log_verbose(LOG_DEBUG, "do_cluster_show(): \n%s\n",sqlquery );
res = PQexec(conn, sqlquery);
if (PQresultStatus(res) != PGRES_TUPLES_OK)
{
log_err(_("Unable to retrieve node information from the database\n%s\n"),
PQerrorMessage(conn));
log_hint(_("Please check that all nodes have been registered\n"));
PQclear(res);
PQfinish(conn);
exit(ERR_BAD_CONFIG);
}
PQfinish(conn);
/*
* Allocate an empty cube matrix
*
* -2 == NULL
* -1 == Error
* 0 == OK
*/
n = PQntuples(res);
cube = (int *) pg_malloc(sizeof(int) * n * n * n);
for (i = 0; i < n * n * n; i++)
cube[i] = -2;
/*
* Find the maximum length of a node name
*/
for (i = 0; i < n; i++)
{
int name_length_cur;
name_length_cur = strlen(PQgetvalue(res, i, 3));
if (name_length_cur > name_length)
name_length = name_length_cur;
}
for (i = 0; i < n; i++)
{
maxlen_snprintf(command,
"repmgr cluster matrix --csv");
initPQExpBuffer(&command_output);
if (i + 1 == options.node)
{
(void)local_command(
command,
&command_output);
}
else
{
(void)remote_command(
PQgetvalue(res, i, 1),
"postgres",
command,
&command_output);
}
p = command_output.data;
for (j = 0; j < n * n; j++)
{
if (sscanf(p, "%d,%d,%d", &x, &y, &z) != 3)
{
fprintf(stderr, _("cannot parse --csv output: %s\n"), p);
PQfinish(conn);
exit(ERR_INTERNAL);
}
cube[i * n * n + (x - 1) * n + (y - 1)] =
(z == -1) ? -1 : 0;
while (*p && (*p != '\n'))
p++;
if (*p == '\n')
p++;
}
}
printf("%*s | Id ", name_length, node_header);
for (i = 0; i < n; i++)
printf("| %2d ", i+1);
printf("\n");
for (i = 0; i < name_length; i++)
printf("-");
printf("-+----");
for (i = 0; i < n; i++)
printf("+----");
printf("\n");
for (i = 0; i < n; i++)
{
printf("%*s | %2d ", name_length,
PQgetvalue(res, i, 3), i + 1);
for (j = 0; j < n; j++)
{
u = cube[i * n + j];
for (k = 1; k < n; k++)
{
/*
* The value of entry (i,j) is equal to the
* maximum value of all the (i,j,k). Indeed:
*
* - if one of the (i,j,k) is 0 (node up), then 0
* (the node is up);
*
* - if the (i,j,k) are either -1 (down) or -2
* (unknown), then -1 (the node is down);
*
* - if all the (i,j,k) are -2 (unknown), then -2
* (the node is in an unknown state).
*/
v = cube[k * n * n + i * n + j];
if (v > u) u = v;
}
switch (u)
{
case -2:
c = '?';
break;
case -1:
c = 'x';
break;
case 0:
c = '*';
break;
default:
exit(ERR_INTERNAL);
}
printf("| %c ", c);
}
printf("\n");
}
PQclear(res);
}
static void
do_cluster_cleanup(void)
{