From 5189488b920aabcb444b83a21aad77c6251e139d Mon Sep 17 00:00:00 2001 From: Gianni Ciolli Date: Tue, 21 Jun 2016 01:02:44 +0200 Subject: [PATCH] Add "cluster diagnose" mode This mode merges the output of "cluster matrix" from each node to improve node state knowledge. --- README.md | 57 +++++++++++++---- repmgr.c | 180 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 226 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index b21e9220..78f880a1 100644 --- a/README.md +++ b/README.md @@ -1601,13 +1601,19 @@ which contains connection details for the local database. The first column is the node's ID, and the second column represents the node's status (0 = master, 1 = standby, -1 = failed). -* `cluster matrix` +* `cluster matrix` and `cluster diagnose` - Displays connection information for each pair of nodes in the - replication cluster. This command polls each registered server and - asks it to connect to each other node. + These commands display connection information for each pair of + nodes in the replication cluster. - This command requires a valid `repmgr.conf` file on each node. + - `cluster matrix` polls each registered server and asks it to + connect to each other node; + + - `cluster diagnose` runs a `cluster matrix` on each node and + combines the results in a single matrix. + + These commands require a valid `repmgr.conf` file on each node, and + the optional `ssh_hostname` parameter must be set. Example 1 (all nodes up): @@ -1619,6 +1625,10 @@ which contains connection details for the local database. node2 | 2 | * | * | * node3 | 3 | * | * | * + Here `cluster matrix` is sufficient to establish the state of each + possible connection. + + Example 2 (node1 and node2 up, node3 down): $ repmgr -f /etc/repmgr.conf cluster matrix @@ -1641,20 +1651,45 @@ which contains connection details for the local database. node1 and node2, meaning that inbound connections to these nodes have succeeded. - Example 3 (all nodes up, firewall dropping packets originating - from node2 and directed to port 5432 on node3) + In this case, `cluster diagnose` gives the same result as `cluster + matrix`, because from any functioning node we can observe the same + state: node1 and node2 are up, node3 is down. - After a long wait (same as before plus two timeouts, by default - one minute each), you will see the following output: + + Example 3 (all nodes up, firewall dropping packets originating + from node1 and directed to port 5432 on node3) + + Running `cluster matrix` from node1 gives the following output, + after a long wait (two timeouts, by default one minute each): $ repmgr -f /etc/repmgr.conf cluster matrix Name | Id | 1 | 2 | 3 -------+----+----+----+---- - node1 | 1 | * | * | * - node2 | 2 | * | * | x + node1 | 1 | * | * | x + node2 | 2 | * | * | * + node3 | 3 | ? | ? | ? + + The matrix tells us that we cannot connect from node1 to node3, + and that (therefore) we don't know the state of any outbound + connection from node3. + + In this case, the `cluster diagnose` command is more informative: + + $ repmgr -f /etc/repmgr.conf cluster diagnose + + Name | Id | 1 | 2 | 3 + -------+----+----+----+---- + node1 | 1 | * | * | x + node2 | 2 | * | * | * node3 | 3 | * | * | * + What happened is that `cluster diagnose` merged its own `cluster + matrix` with the `cluster matrix` output from node2; the latter is + able to connect to node3 and therefore determine the state of + outbound connections from that node. + + * `cluster cleanup` Purges monitoring history from the `repl_monitor` table to prevent excessive diff --git a/repmgr.c b/repmgr.c index b09e8703..173b61bc 100644 --- a/repmgr.c +++ b/repmgr.c @@ -21,6 +21,7 @@ * WITNESS REGISTER * WITNESS UNREGISTER * + * CLUSTER DIAGNOSE * CLUSTER MATRIX * CLUSTER SHOW * CLUSTER CLEANUP @@ -90,6 +91,7 @@ #define CLUSTER_SHOW 13 #define CLUSTER_CLEANUP 14 #define CLUSTER_MATRIX 15 +#define CLUSTER_DIAGNOSE 16 static int test_ssh_connection(char *host, char *remote_user); static int copy_remote_files(char *host, char *remote_user, char *remote_path, @@ -132,6 +134,7 @@ static void do_witness_unregister(void); static void do_cluster_show(void); static void do_cluster_matrix(void); +static void do_cluster_diagnose(void); static void do_cluster_cleanup(void); static void do_check_upstream_config(void); static void do_help(void); @@ -713,6 +716,8 @@ main(int argc, char **argv) action = CLUSTER_SHOW; else if (strcasecmp(server_cmd, "CLEANUP") == 0) action = CLUSTER_CLEANUP; + else if (strcasecmp(server_cmd, "DIAGNOSE") == 0) + action = CLUSTER_DIAGNOSE; else if (strcasecmp(server_cmd, "MATRIX") == 0) action = CLUSTER_MATRIX; } @@ -954,6 +959,9 @@ main(int argc, char **argv) case WITNESS_UNREGISTER: do_witness_unregister(); break; + case CLUSTER_DIAGNOSE: + do_cluster_diagnose(); + break; case CLUSTER_MATRIX: do_cluster_matrix(); break; @@ -1282,6 +1290,178 @@ do_cluster_matrix(void) PQclear(res); } +static void +do_cluster_diagnose(void) +{ + PGconn *conn; + PGresult *res; + char sqlquery[QUERY_STR_LEN]; + int i, j, k; + const char *node_header = "Name"; + int name_length = strlen(node_header); + + int x, y, z, u, v; + int n = 0; /* number of nodes */ + int *cube; + char *p; + char c; + + char command[MAXLEN]; + PQExpBufferData command_output; + + /* We need to connect to get the list of nodes */ + log_info(_("connecting to database\n")); + conn = establish_db_connection(options.conninfo, true); + + sqlquery_snprintf(sqlquery, + "SELECT conninfo, ssh_hostname, type, name, upstream_node_name, id" + " FROM %s.repl_show_nodes ORDER BY id", + get_repmgr_schema_quoted(conn)); + + log_verbose(LOG_DEBUG, "do_cluster_show(): \n%s\n",sqlquery ); + + res = PQexec(conn, sqlquery); + + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + log_err(_("Unable to retrieve node information from the database\n%s\n"), + PQerrorMessage(conn)); + log_hint(_("Please check that all nodes have been registered\n")); + + PQclear(res); + PQfinish(conn); + exit(ERR_BAD_CONFIG); + } + PQfinish(conn); + + /* + * Allocate an empty cube matrix + * + * -2 == NULL + * -1 == Error + * 0 == OK + */ + n = PQntuples(res); + cube = (int *) pg_malloc(sizeof(int) * n * n * n); + for (i = 0; i < n * n * n; i++) + cube[i] = -2; + + /* + * Find the maximum length of a node name + */ + for (i = 0; i < n; i++) + { + int name_length_cur; + + name_length_cur = strlen(PQgetvalue(res, i, 3)); + if (name_length_cur > name_length) + name_length = name_length_cur; + } + + for (i = 0; i < n; i++) + { + maxlen_snprintf(command, + "repmgr cluster matrix --csv"); + + initPQExpBuffer(&command_output); + + if (i + 1 == options.node) + { + (void)local_command( + command, + &command_output); + } + else + { + (void)remote_command( + PQgetvalue(res, i, 1), + "postgres", + command, + &command_output); + } + + p = command_output.data; + + for (j = 0; j < n * n; j++) + { + if (sscanf(p, "%d,%d,%d", &x, &y, &z) != 3) + { + fprintf(stderr, _("cannot parse --csv output: %s\n"), p); + PQfinish(conn); + exit(ERR_INTERNAL); + } + cube[i * n * n + (x - 1) * n + (y - 1)] = + (z == -1) ? -1 : 0; + while (*p && (*p != '\n')) + p++; + if (*p == '\n') + p++; + } + } + + printf("%*s | Id ", name_length, node_header); + for (i = 0; i < n; i++) + printf("| %2d ", i+1); + printf("\n"); + + for (i = 0; i < name_length; i++) + printf("-"); + printf("-+----"); + for (i = 0; i < n; i++) + printf("+----"); + printf("\n"); + + for (i = 0; i < n; i++) + { + printf("%*s | %2d ", name_length, + PQgetvalue(res, i, 3), i + 1); + for (j = 0; j < n; j++) + { + u = cube[i * n + j]; + for (k = 1; k < n; k++) + { + /* + * The value of entry (i,j) is equal to the + * maximum value of all the (i,j,k). Indeed: + * + * - if one of the (i,j,k) is 0 (node up), then 0 + * (the node is up); + * + * - if the (i,j,k) are either -1 (down) or -2 + * (unknown), then -1 (the node is down); + * + * - if all the (i,j,k) are -2 (unknown), then -2 + * (the node is in an unknown state). + */ + + v = cube[k * n * n + i * n + j]; + + if (v > u) u = v; + } + + switch (u) + { + case -2: + c = '?'; + break; + case -1: + c = 'x'; + break; + case 0: + c = '*'; + break; + default: + exit(ERR_INTERNAL); + } + + printf("| %c ", c); + } + printf("\n"); + } + + PQclear(res); +} + static void do_cluster_cleanup(void) {