diff --git a/dbutils.c b/dbutils.c index 51931b88..02566ec7 100644 --- a/dbutils.c +++ b/dbutils.c @@ -1142,6 +1142,7 @@ get_node_record_by_name(PGconn *conn, const char *node_name, t_node_info *node_i int result; initPQExpBuffer(&query); + appendPQExpBuffer(&query, "SELECT node_id, type, upstream_node_id, node_name, conninfo, slot_name, priority, active" " FROM repmgr.nodes " @@ -1152,6 +1153,8 @@ get_node_record_by_name(PGconn *conn, const char *node_name, t_node_info *node_i result = _get_node_record(conn, query.data, node_info); + termPQExpBuffer(&query); + if (result == 0) { log_verbose(LOG_DEBUG, "get_node_record(): no record found for node %s", @@ -1670,3 +1673,105 @@ get_slot_record(PGconn *conn, char *slot_name, t_replication_slot *record) return 1; } + + +/* ================ */ +/* backup functions */ +/* ================ */ + +// XXX is first_wal_segment actually used anywhere? +bool +start_backup(PGconn *conn, char *first_wal_segment, bool fast_checkpoint, int server_version_num) +{ + PQExpBufferData query; + PGresult *res; + + initPQExpBuffer(&query); + + if (server_version_num >= 100000) + { + appendPQExpBuffer(&query, + "SELECT pg_catalog.pg_walfile_name(pg_catalog.pg_start_backup('repmgr_standby_clone_%ld', %s))", + time(NULL), + fast_checkpoint ? "TRUE" : "FALSE"); + } + else + { + appendPQExpBuffer(&query, + "SELECT pg_catalog.pg_xlogfile_name(pg_catalog.pg_start_backup('repmgr_standby_clone_%ld', %s))", + time(NULL), + fast_checkpoint ? "TRUE" : "FALSE"); + } + + log_verbose(LOG_DEBUG, "start_backup():\n %s", query.data); + + res = PQexec(conn, query.data); + + termPQExpBuffer(&query); + + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + log_error(_("unable to start backup:\n %s"), PQerrorMessage(conn)); + PQclear(res); + return false; + } + + if (first_wal_segment != NULL) + { + char *first_wal_seg_pq = PQgetvalue(res, 0, 0); + size_t buf_sz = strlen(first_wal_seg_pq); + + first_wal_segment = pg_malloc0(buf_sz + 1); + snprintf(first_wal_segment, buf_sz + 1, "%s", first_wal_seg_pq); + } + + PQclear(res); + + return true; +} + + +bool +stop_backup(PGconn *conn, char *last_wal_segment, int server_version_num) +{ + PQExpBufferData query; + PGresult *res; + + initPQExpBuffer(&query); + + if (server_version_num >= 100000) + { + appendPQExpBuffer(&query, + "SELECT pg_catalog.pg_walfile_name(pg_catalog.pg_stop_backup())"); + } + else + { + appendPQExpBuffer(&query, + "SELECT pg_catalog.pg_xlogfile_name(pg_catalog.pg_stop_backup())"); + } + + res = PQexec(conn, query.data); + termPQExpBuffer(&query); + + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + log_error(_("unable to stop backup:\n %s"), PQerrorMessage(conn)); + PQclear(res); + return false; + } + + if (last_wal_segment != NULL) + { + char *last_wal_seg_pq = PQgetvalue(res, 0, 0); + size_t buf_sz = strlen(last_wal_seg_pq); + + last_wal_segment = pg_malloc0(buf_sz + 1); + snprintf(last_wal_segment, buf_sz + 1, "%s", last_wal_seg_pq); + } + + PQclear(res); + + return true; +} + + diff --git a/dbutils.h b/dbutils.h index 27ef3227..ba226ce7 100644 --- a/dbutils.h +++ b/dbutils.h @@ -184,5 +184,9 @@ bool create_replication_slot(PGconn *conn, char *slot_name, int server_version_ int get_slot_record(PGconn *conn, char *slot_name, t_replication_slot *record); +/* backup functions */ +bool start_backup(PGconn *conn, char *first_wal_segment, bool fast_checkpoint, int server_version_num); +bool stop_backup(PGconn *conn, char *last_wal_segment, int server_version_num); + #endif diff --git a/errcode.h b/errcode.h index 43ae981d..90ee6985 100644 --- a/errcode.h +++ b/errcode.h @@ -26,6 +26,7 @@ #define ERR_SWITCHOVER_FAIL 18 #define ERR_BARMAN 19 #define ERR_REGISTRATION_SYNC 20 +#define ERR_OUT_OF_MEMORY 21 #endif /* _ERRCODE_H_ */ diff --git a/repmgr-action-standby.c b/repmgr-action-standby.c index 1ee75925..8cf3815c 100644 --- a/repmgr-action-standby.c +++ b/repmgr-action-standby.c @@ -6,6 +6,8 @@ * Copyright (c) 2ndQuadrant, 2010-2017 */ +#include + #include "repmgr.h" #include "dirutil.h" #include "compat.h" @@ -13,6 +15,35 @@ #include "repmgr-client-global.h" #include "repmgr-action-standby.h" + +typedef struct TablespaceDataListCell +{ + struct TablespaceDataListCell *next; + char *name; + char *oid; + char *location; + /* optional payload */ + FILE *f; +} TablespaceDataListCell; + +typedef struct TablespaceDataList +{ + TablespaceDataListCell *head; + TablespaceDataListCell *tail; +} TablespaceDataList; + +struct BackupLabel +{ + XLogRecPtr start_wal_location; + char start_wal_file[MAXLEN]; + XLogRecPtr checkpoint_location; + char backup_from[MAXLEN]; + char backup_method[MAXLEN]; + char start_time[MAXLEN]; + char label[MAXLEN]; + XLogRecPtr min_failover_slot_lsn; +}; + static PGconn *primary_conn = NULL; static PGconn *source_conn = NULL; @@ -32,8 +63,15 @@ static t_configfile_list config_files = T_CONFIGFILE_LIST_INITIALIZER; static standy_clone_mode mode; +// XXX these aren't actually used after being set... remove if no purpose can be found +static char *first_wal_segment = NULL; +static char *last_wal_segment = NULL; + +static bool pg_start_backup_executed = false; + /* used by barman mode */ static char local_repmgr_tmp_directory[MAXPGPATH]; +static char barman_command_buf[MAXLEN] = ""; static void check_barman_config(void); @@ -45,13 +83,24 @@ static void initialise_direct_clone(void); static void config_file_list_init(t_configfile_list *list, int max_size); static void config_file_list_add(t_configfile_list *list, const char *file, const char *filename, bool in_data_dir); +static int run_basebackup(void); +static int run_file_backup(void); +static bool read_backup_label(const char *local_data_directory, struct BackupLabel *out_backup_label); +static void parse_lsn(XLogRecPtr *ptr, const char *str); +static XLogRecPtr parse_label_lsn(const char *label_key, const char *label_value); + +static int get_tablespace_data(PGconn *upstream_conn, TablespaceDataList *list); +static void tablespace_data_append(TablespaceDataList *list, const char *name, const char *oid, const char *location); + static void get_barman_property(char *dst, char *name, char *local_repmgr_directory); +static int get_tablespace_data_barman(char *, TablespaceDataList *); static char *make_barman_ssh_command(char *buf); void do_standby_clone(void) { + int r; /* * conninfo params for the actual upstream node (which might be different @@ -68,8 +117,6 @@ do_standby_clone(void) */ if (mode == rsync) { - int r; - r = test_ssh_connection(runtime_options.host, runtime_options.remote_user); if (r != 0) { @@ -236,6 +283,46 @@ do_standby_clone(void) initialise_direct_clone(); } + switch (mode) + { + case rsync: + log_notice(_("starting backup (using rsync)...")); + break; + case pg_basebackup: + log_notice(_("starting backup (using pg_basebackup)...")); + if (runtime_options.fast_checkpoint == false) + { + log_hint(_("this may take some time; consider using the -c/--fast-checkpoint option")); + } + break; + case barman: + log_notice(_("getting backup from Barman...")); + break; + default: + /* should never reach here */ + log_error(_("unknown clone mode")); + } + + if (mode == pg_basebackup) + { + r = run_basebackup(); + if (r != 0) + { + log_warning(_("standby clone: base backup failed")); + + r = ERR_BAD_BASEBACKUP; + } + } + else + { + r = run_file_backup(); + if (r != 0) + { + log_warning(_("standby clone: base backup failed")); + + r = ERR_BAD_BASEBACKUP; + } + } } @@ -244,7 +331,6 @@ void check_barman_config(void) { char datadir_list_filename[MAXLEN]; - char barman_command_buf[MAXLEN] = ""; char command[MAXLEN]; bool command_ok; @@ -265,7 +351,7 @@ check_barman_config(void) { log_error(_("no valid backup for server %s was found in the Barman catalogue"), config_file_options.barman_server); - log_hint(_("refer to the Barman documentation for more information\n")); + log_hint(_("refer to the Barman documentation for more information")); exit(ERR_BARMAN); } @@ -284,7 +370,7 @@ check_barman_config(void) { log_error(_("unable to use directory %s"), local_data_directory); - log_hint(_("use -F/--force option to force this directory to be overwritten\n")); + log_hint(_("use -F/--force option to force this directory to be overwritten")); exit(ERR_BAD_CONFIG); } @@ -377,7 +463,7 @@ check_source_server() { if (server_version_num < 90400) { - log_error(_("PostgreSQL 9.4 or greater required for --recovery-min-apply-delay\n")); + log_error(_("PostgreSQL 9.4 or greater required for --recovery-min-apply-delay")); PQfinish(source_conn); exit(ERR_BAD_CONFIG); } @@ -594,7 +680,7 @@ initialise_direct_clone(void) { log_error(_("unable to use directory %s ..."), local_data_directory); - log_hint(_("use -F/--force to force this directory to be overwritten\n")); + log_hint(_("use -F/--force to force this directory to be overwritten")); exit(ERR_BAD_CONFIG); } @@ -647,7 +733,7 @@ initialise_direct_clone(void) /* TODO: collate errors and output at end of loop */ if (PQntuples(res) == 0) { - log_error(_("no tablespace matching path '%s' found\n"), + log_error(_("no tablespace matching path '%s' found"), cell->old_dir); PQclear(res); PQfinish(source_conn); @@ -735,7 +821,7 @@ initialise_direct_clone(void) if (PQresultStatus(res) != PGRES_TUPLES_OK) { - log_error(_("unable to retrieve configuration file locations: %s\n"), + log_error(_("unable to retrieve configuration file locations:\n %s"), PQerrorMessage(source_conn)); PQclear(res); PQfinish(source_conn); @@ -787,7 +873,663 @@ initialise_direct_clone(void) repmgr_slot_name, upstream_node_id); } +} + +static int +run_basebackup(void) +{ + char script[MAXLEN]; + int r = 0; + PQExpBufferData params; + TablespaceListCell *cell; + t_basebackup_options backup_options = T_BASEBACKUP_OPTIONS_INITIALIZER; + + /* + * Parse the pg_basebackup_options provided in repmgr.conf - we'll want + * to check later whether certain options were set by the user + */ + parse_pg_basebackup_options(config_file_options.pg_basebackup_options, + &backup_options, + server_version_num, + NULL); + + /* Create pg_basebackup command line options */ + + initPQExpBuffer(¶ms); + + appendPQExpBuffer(¶ms, " -D %s", local_data_directory); + + /* + * conninfo string provided - pass it to pg_basebackup as the -d option + * (pg_basebackup doesn't require or want a database name, but for + * consistency with other applications accepts a conninfo string + * under -d/--dbname) + */ + if (runtime_options.conninfo_provided == true) + { + appendPQExpBuffer(¶ms, " -d '%s'", runtime_options.dbname); + } + + /* + * Connection parameters not passed to repmgr as conninfo string - provide + * them individually to pg_basebackup (-d/--dbname not required) + */ + else + { + if (strlen(runtime_options.host)) + { + appendPQExpBuffer(¶ms, " -h %s", runtime_options.host); + } + + if (strlen(runtime_options.port)) + { + appendPQExpBuffer(¶ms, " -p %s", runtime_options.port); + } + + if (strlen(runtime_options.replication_user)) + { + appendPQExpBuffer(¶ms, " -U %s", runtime_options.replication_user); + } + else if (strlen(runtime_options.username)) + { + appendPQExpBuffer(¶ms, " -U %s", runtime_options.username); + } + } + + if (runtime_options.fast_checkpoint) { + appendPQExpBuffer(¶ms, " -c fast"); + } + + if (config_file_options.tablespace_mapping.head != NULL) + { + for (cell = config_file_options.tablespace_mapping.head; cell; cell = cell->next) + { + appendPQExpBuffer(¶ms, " -T %s=%s", cell->old_dir, cell->new_dir); + } + } + + /* + * To ensure we have all the WALs needed during basebackup execution we stream + * them as the backup is taking place. + * + * From 9.6, if replication slots are in use, we'll have previously + * created a slot with reserved LSN, and will stream from that slot to avoid + * WAL buildup on the master using the -S/--slot, which requires -X/--xlog-method=stream + * (from 10, -X/--wal-method=stream) + */ + if (!strlen(backup_options.xlog_method)) + { + appendPQExpBuffer(¶ms, " -X stream"); + } + + /* + * From 9.6, pg_basebackup accepts -S/--slot, which forces WAL streaming to use + * the specified replication slot. If replication slot usage is specified, the + * slot will already have been created. + * + * NOTE: currently there's no way of disabling the --slot option while using + * --xlog-method=stream - it's hard to imagine a use case for this, so no + * provision has been made for doing it. + * + * NOTE: + * It's possible to set 'pg_basebackup_options' with an invalid combination + * of values for --wal-method (--xlog-method) and --slot - we're not checking that, just that + * we're not overriding any user-supplied values + */ + if (server_version_num >= 90600 && config_file_options.use_replication_slots) + { + bool slot_add = true; + + /* + * Check whether 'pg_basebackup_options' in repmgr.conf has the --slot option set, + * or if --wal-method (--xlog-method) is set to a value other than "stream" + * (in which case we can't use --slot). + */ + if (strlen(backup_options.slot) || (strlen(backup_options.xlog_method) && strcmp(backup_options.xlog_method, "stream") != 0)) { + slot_add = false; + } + + if (slot_add == true) + { + appendPQExpBuffer(¶ms, " -S %s", repmgr_slot_name_ptr); + } + } + + maxlen_snprintf(script, + "%s -l \"repmgr base backup\" %s %s", + make_pg_path("pg_basebackup"), + params.data, + config_file_options.pg_basebackup_options); + + termPQExpBuffer(¶ms); + + log_info(_("executing: '%s'"), script); + + /* + * As of 9.4, pg_basebackup only ever returns 0 or 1 + */ + + r = system(script); + + return r; +} + + +static int +run_file_backup(void) +{ + int r = 0, i; + + char command[MAXLEN]; + char filename[MAXLEN]; + char buf[MAXLEN]; + char backup_directory[MAXLEN]; + char backup_id[MAXLEN] = ""; + char *p, *q; + PQExpBufferData command_output; + TablespaceDataList tablespace_list = { NULL, NULL }; + TablespaceDataListCell *cell_t; + + PQExpBufferData tablespace_map; + bool tablespace_map_rewrite = false; + + char datadir_list_filename[MAXLEN]; + + struct BackupLabel backup_label; + + if (mode == barman) + { + + /* + * Locate Barman's backup directory + */ + + get_barman_property(backup_directory, "backup_directory", local_repmgr_tmp_directory); + + /* + * Read the list of backup files into a local file. In the + * process: + * + * - determine the backup ID; + * - check, and remove, the prefix; + * - detect tablespaces; + * - filter files in one list per tablespace; + */ + + { + FILE *fi; /* input stream */ + FILE *fd; /* output for data.txt */ + char prefix[MAXLEN]; + char output[MAXLEN]; + int n; + + maxlen_snprintf(command, "%s list-files --target=data %s latest", + make_barman_ssh_command(barman_command_buf), + config_file_options.barman_server); + + fi = popen(command, "r"); + if (fi == NULL) + { + log_error("cannot launch command: %s", command); + exit(ERR_BARMAN); + } + + fd = fopen(datadir_list_filename, "w"); + if (fd == NULL) + { + log_error("cannot open file: %s", datadir_list_filename); + exit(ERR_INTERNAL); + } + + maxlen_snprintf(prefix, "%s/base/", backup_directory); + while (fgets(output, MAXLEN, fi) != NULL) + { + /* + * Remove prefix + */ + p = string_skip_prefix(prefix, output); + if (p == NULL) + { + log_error("unexpected output from \"barman list-files\": %s", + output); + exit(ERR_BARMAN); + } + + /* + * Remove and note backup ID; copy backup.info + */ + if (! strcmp(backup_id, "")) + { + FILE *fi2; + + n = strcspn(p, "/"); + + strncpy(backup_id, p, n); + + strncat(prefix,backup_id,MAXLEN-1); + strncat(prefix,"/",MAXLEN-1); + p = string_skip_prefix(backup_id, p); + p = string_skip_prefix("/", p); + + /* + * Copy backup.info + */ + maxlen_snprintf(command, + "rsync -a %s:%s/base/%s/backup.info %s", + config_file_options.barman_host, + backup_directory, + backup_id, + local_repmgr_tmp_directory); + (void)local_command( + command, + &command_output); + + /* + * Get tablespace data + */ + maxlen_snprintf(filename, "%s/backup.info", + local_repmgr_tmp_directory); + fi2 = fopen(filename, "r"); + if (fi2 == NULL) + { + log_error("cannot open file: %s", filename); + exit(ERR_INTERNAL); + } + while (fgets(buf, MAXLEN, fi2) != NULL) + { + q = string_skip_prefix("tablespaces=", buf); + if (q != NULL && strncmp(q, "None\n", 5)) + { + get_tablespace_data_barman(q, &tablespace_list); + } + q = string_skip_prefix("version=", buf); + if (q != NULL) + { + server_version_num = strtol(q, NULL, 10); + } + } + fclose(fi2); + unlink(filename); + + continue; + } + + /* + * Skip backup.info + */ + if (string_skip_prefix("backup.info", p)) + continue; + + /* + * Filter data directory files + */ + if ((q = string_skip_prefix("data/", p)) != NULL) + { + fputs(q, fd); + continue; + } + + /* + * Filter other files (i.e. tablespaces) + */ + for (cell_t = tablespace_list.head; cell_t; cell_t = cell_t->next) + { + if ((q = string_skip_prefix(cell_t->oid, p)) != NULL && *q == '/') + { + if (cell_t->f == NULL) + { + maxlen_snprintf(filename, "%s/%s.txt", local_repmgr_tmp_directory, cell_t->oid); + cell_t->f = fopen(filename, "w"); + if (cell_t->f == NULL) + { + log_error("cannot open file: %s", filename); + exit(ERR_INTERNAL); + } + } + fputs(q + 1, cell_t->f); + break; + } + } + } + + fclose(fd); + + pclose(fi); + } + + /* For 9.5 and greater, create our own tablespace_map file */ + if (server_version_num >= 90500) + { + initPQExpBuffer(&tablespace_map); + } + + /* + * As of Barman version 1.6.1, the file structure of a backup + * is as follows: + * + * base/ - base backup + * wals/ - WAL files associated to the backup + * + * base/ - backup files + * + * here ID has the standard timestamp form yyyymmddThhmmss + * + * base//backup.info - backup metadata, in text format + * base//data - data directory + * base// - tablespace with the given oid + */ + + /* + * Copy all backup files from the Barman server + */ + maxlen_snprintf(command, + "rsync --progress -a --files-from=%s %s:%s/base/%s/data %s", + datadir_list_filename, + config_file_options.barman_host, + backup_directory, + backup_id, + local_data_directory); + + (void)local_command( + command, + &command_output); + + unlink(datadir_list_filename); + + /* + * We must create some PGDATA subdirectories because they are + * not included in the Barman backup. + * + * See class RsyncBackupExecutor in the Barman source (barman/backup_executor.py) + * for a definitive list of excluded directories. + */ + { + const char* const dirs[] = { + /* Only from 10 */ + "pg_wal", + /* Only from 9.5 */ + "pg_commit_ts", + /* Only from 9.4 */ + "pg_dynshmem", "pg_logical", "pg_logical/snapshots", "pg_logical/mappings", "pg_replslot", + /* Already in 9.3 */ + "pg_notify", "pg_serial", "pg_snapshots", "pg_stat", "pg_stat_tmp", "pg_tblspc", + "pg_twophase", "pg_xlog", 0 + }; + const int vers[] = { + 100000, + 90500, + 90400, 90400, 90400, 90400, 90400, + 0, 0, 0, 0, 0, 0, + 0, -100000, 0 + }; + for (i = 0; dirs[i]; i++) + { + /* directory exists in newer versions than this server - skip */ + if (vers[i] > 0 && server_version_num < vers[i]) + continue; + + /* directory existed in earlier versions than this server but has been removed/renamed - skip */ + if (vers[i] < 0 && server_version_num >= abs(vers[i])) + continue; + + maxlen_snprintf(filename, "%s/%s", local_data_directory, dirs[i]); + if (mkdir(filename, S_IRWXU) != 0 && errno != EEXIST) + { + log_error(_("unable to create the %s directory"), dirs[i]); + exit(ERR_INTERNAL); + } + } + } + } + else if (mode == rsync) + { + /* For 9.5 and greater, create our own tablespace_map file */ + if (server_version_num >= 90500) + { + initPQExpBuffer(&tablespace_map); + } + + if (start_backup(source_conn, first_wal_segment, runtime_options.fast_checkpoint, server_version_num) == false) + { + r = ERR_BAD_BASEBACKUP; + goto stop_backup; + } + + /* + * Note that we've successfully executed pg_start_backup(), + * so we know whether or not to execute pg_stop_backup() after + * the 'stop_backup' label + */ + pg_start_backup_executed = true; + + /* + * 1. copy data directory, omitting directories which should not be + * copied, or for which copying would serve no purpose. + * + * 2. copy pg_control file + */ + + /* Copy the data directory */ + log_info(_("standby clone: upstream data directory is '%s'"), + upstream_data_directory); + r = copy_remote_files(runtime_options.host, runtime_options.remote_user, + upstream_data_directory, local_data_directory, + true, server_version_num); + /* + * Exit code 0 means no error, but we want to ignore exit code 24 as well + * as rsync returns that code on "Partial transfer due to vanished source files". + * It's quite common for this to happen on the data directory, particularly + * with long running rsync on a busy server. + */ + if (WIFEXITED(r) && WEXITSTATUS(r) && WEXITSTATUS(r) != 24) + { + log_error(_("standby clone: failed copying upstream data directory '%s'"), + upstream_data_directory); + r = ERR_BAD_RSYNC; + goto stop_backup; + } + + /* Read backup label copied from primary */ + if (read_backup_label(local_data_directory, &backup_label) == false) + { + r = ERR_BAD_BACKUP_LABEL; + goto stop_backup; + } + + /* Copy tablespaces and, if required, remap to a new location */ + r = get_tablespace_data(source_conn, &tablespace_list); + if (r != SUCCESS) goto stop_backup; + } + + for (cell_t = tablespace_list.head; cell_t; cell_t = cell_t->next) + { + bool mapping_found = false; + TablespaceListCell *cell; + char *tblspc_dir_dest; + + /* Check if tablespace path matches one of the provided tablespace mappings */ + if (config_file_options.tablespace_mapping.head != NULL) + { + for (cell = config_file_options.tablespace_mapping.head; cell; cell = cell->next) + { + if (strcmp(cell_t->location, cell->old_dir) == 0) + { + mapping_found = true; + break; + } + } + } + + if (mapping_found == true) + { + tblspc_dir_dest = cell->new_dir; + log_debug(_("mapping source tablespace '%s' (OID %s) to '%s'"), + cell_t->location, cell_t->oid, tblspc_dir_dest); + } + else + { + tblspc_dir_dest = cell_t->location; + } + + /* + * Tablespace file copy + */ + + if (mode == barman) + { + create_pg_dir(cell_t->location, false); + + if (cell_t->f != NULL) /* cell_t->f == NULL iff the tablespace is empty */ + { + maxlen_snprintf(command, + "rsync --progress -a --files-from=%s/%s.txt %s:%s/base/%s/%s %s", + local_repmgr_tmp_directory, + cell_t->oid, + config_file_options.barman_host, + backup_directory, + backup_id, + cell_t->oid, + tblspc_dir_dest); + (void)local_command( + command, + &command_output); + fclose(cell_t->f); + maxlen_snprintf(filename, + "%s/%s.txt", + local_repmgr_tmp_directory, + cell_t->oid); + unlink(filename); + } + } + else if (mode == rsync) + { + /* Copy tablespace directory */ + r = copy_remote_files(runtime_options.host, runtime_options.remote_user, + cell_t->location, tblspc_dir_dest, + true, server_version_num); + + /* + * Exit code 0 means no error, but we want to ignore exit code 24 as well + * as rsync returns that code on "Partial transfer due to vanished source files". + * It's quite common for this to happen on the data directory, particularly + * with long running rsync on a busy server. + */ + if (WIFEXITED(r) && WEXITSTATUS(r) && WEXITSTATUS(r) != 24) + { + log_error(_("standby clone: failed copying tablespace directory '%s'"), + cell_t->location); + r = ERR_BAD_RSYNC; + goto stop_backup; + } + } + + /* + * If a valid mapping was provide for this tablespace, arrange for it to + * be remapped + * (if no tablespace mapping was provided, the link will be copied as-is + * by pg_basebackup or rsync and no action is required) + */ + if (mapping_found == true || mode == barman) + { + /* 9.5 and later - append to the tablespace_map file */ + if (server_version_num >= 90500) + { + tablespace_map_rewrite = true; + appendPQExpBuffer(&tablespace_map, + "%s %s\n", + cell_t->oid, + tblspc_dir_dest); + } + /* Pre-9.5, we have to manipulate the symlinks in pg_tblspc/ ourselves */ + else + { + PQExpBufferData tblspc_symlink; + + initPQExpBuffer(&tblspc_symlink); + appendPQExpBuffer(&tblspc_symlink, "%s/pg_tblspc/%s", + local_data_directory, + cell_t->oid); + + if (unlink(tblspc_symlink.data) < 0 && errno != ENOENT) + { + log_error(_("unable to remove tablespace symlink %s"), tblspc_symlink.data); + + r = ERR_BAD_BASEBACKUP; + goto stop_backup; + } + + if (symlink(tblspc_dir_dest, tblspc_symlink.data) < 0) + { + log_error(_("unable to create tablespace symlink from %s to %s"), tblspc_symlink.data, tblspc_dir_dest); + + r = ERR_BAD_BASEBACKUP; + goto stop_backup; + } + } + } + } + + /* + * For 9.5 and later, if tablespace remapping was requested, we'll need + * to rewrite the tablespace map file ourselves. + * The tablespace map file is read on startup and any links created by + * the backend; we could do this ourselves like for pre-9.5 servers, but + * it's better to rely on functionality the backend provides. + */ + + if (server_version_num >= 90500 && tablespace_map_rewrite == true) + { + PQExpBufferData tablespace_map_filename; + FILE *tablespace_map_file; + initPQExpBuffer(&tablespace_map_filename); + appendPQExpBuffer(&tablespace_map_filename, "%s/%s", + local_data_directory, + TABLESPACE_MAP); + + /* Unlink any existing file (it should be there, but we don't care if it isn't) */ + if (unlink(tablespace_map_filename.data) < 0 && errno != ENOENT) + { + log_error(_("unable to remove tablespace_map file %s: %s"), + tablespace_map_filename.data, + strerror(errno)); + + r = ERR_BAD_BASEBACKUP; + goto stop_backup; + } + + tablespace_map_file = fopen(tablespace_map_filename.data, "w"); + if (tablespace_map_file == NULL) + { + log_error(_("unable to create tablespace_map file '%s'"), tablespace_map_filename.data); + + r = ERR_BAD_BASEBACKUP; + goto stop_backup; + } + + if (fputs(tablespace_map.data, tablespace_map_file) == EOF) + { + log_error(_("unable to write to tablespace_map file '%s'"), tablespace_map_filename.data); + + r = ERR_BAD_BASEBACKUP; + goto stop_backup; + } + + fclose(tablespace_map_file); + } + +stop_backup: + + if (mode == rsync && pg_start_backup_executed) + { + log_notice(_("notifying upstream about backup completion...\n")); + if (stop_backup(source_conn, last_wal_segment, server_version_num) == false) + { + r = ERR_BAD_BASEBACKUP; + } + } + + return r; } @@ -810,6 +1552,67 @@ make_barman_ssh_command(char *buf) } +static int +get_tablespace_data_barman +( char *tablespace_data_barman, + TablespaceDataList *tablespace_list) +{ + /* + * Example: + * [('main', 24674, '/var/lib/postgresql/tablespaces/9.5/main'), ('alt', 24678, '/var/lib/postgresql/tablespaces/9.5/alt')] + */ + + char name[MAXLEN]; + char oid[MAXLEN]; + char location[MAXPGPATH]; + char *p = tablespace_data_barman; + int i; + + tablespace_list->head = NULL; + tablespace_list->tail = NULL; + + p = string_skip_prefix("[", p); + if (p == NULL) return -1; + + while (*p == '(') + { + p = string_skip_prefix("('", p); + if (p == NULL) return -1; + + i = strcspn(p, "'"); + strncpy(name, p, i); + name[i] = 0; + + p = string_skip_prefix("', ", p + i); + if (p == NULL) return -1; + + i = strcspn(p, ","); + strncpy(oid, p, i); + oid[i] = 0; + + p = string_skip_prefix(", '", p + i); + if (p == NULL) return -1; + + i = strcspn(p, "'"); + strncpy(location, p, i); + location[i] = 0; + + p = string_skip_prefix("')", p + i); + if (p == NULL) return -1; + + tablespace_data_append (tablespace_list, name, oid, location); + + if (*p == ']') + break; + + p = string_skip_prefix(", ", p); + if (p == NULL) return -1; + } + + return SUCCESS; +} + + void get_barman_property(char *dst, char *name, char *local_repmgr_directory) { @@ -846,6 +1649,12 @@ config_file_list_init(t_configfile_list *list, int max_size) list->size = max_size; list->entries = 0; list->files = pg_malloc0(sizeof(t_configfile_info *) * max_size); + + if (list->files == NULL) + { + log_error(_("unable to allocate memory; terminating")); + exit(ERR_OUT_OF_MEMORY); + } } @@ -858,6 +1667,12 @@ config_file_list_add(t_configfile_list *list, const char *file, const char *file list->files[list->entries] = pg_malloc0(sizeof(t_configfile_info)); + if (list->files[list->entries] == NULL) + { + log_error(_("unable to allocate memory; terminating")); + exit(ERR_OUT_OF_MEMORY); + } + strncpy(list->files[list->entries]->filepath, file, MAXPGPATH); canonicalize_path(list->files[list->entries]->filepath); @@ -868,3 +1683,247 @@ config_file_list_add(t_configfile_list *list, const char *file, const char *file list->entries ++; } + + +static int +get_tablespace_data(PGconn *upstream_conn, TablespaceDataList *list) +{ + PQExpBufferData query; + PGresult *res; + int i; + + initPQExpBuffer(&query); + + appendPQExpBuffer(&query, + " SELECT spcname, oid, pg_catalog.pg_tablespace_location(oid) AS spclocation " + " FROM pg_catalog.pg_tablespace " + " WHERE spcname NOT IN ('pg_default', 'pg_global')"); + + res = PQexec(upstream_conn, query.data); + + termPQExpBuffer(&query); + + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + log_error(_("unable to execute tablespace query:\n %s"), + PQerrorMessage(upstream_conn)); + + PQclear(res); + + return ERR_DB_QUERY; + } + + for (i = 0; i < PQntuples(res); i++) + tablespace_data_append(list, + PQgetvalue(res, i, 0), + PQgetvalue(res, i, 1), + PQgetvalue(res, i, 2)); + + PQclear(res); + return SUCCESS; +} + +static void +tablespace_data_append(TablespaceDataList *list, const char *name, const char *oid, const char *location) +{ + TablespaceDataListCell *cell; + + cell = (TablespaceDataListCell *) pg_malloc0(sizeof(TablespaceDataListCell)); + + if (cell == NULL) + { + log_error(_("unable to allocate memory; terminating")); + exit(ERR_OUT_OF_MEMORY); + } + + cell->oid = pg_malloc(1 + strlen(oid )); + cell->name = pg_malloc(1 + strlen(name )); + cell->location = pg_malloc(1 + strlen(location)); + + strncpy(cell->oid , oid , 1 + strlen(oid )); + strncpy(cell->name , name , 1 + strlen(name )); + strncpy(cell->location, location, 1 + strlen(location)); + + if (list->tail) + list->tail->next = cell; + else + list->head = cell; + + list->tail = cell; +} + + + +static void +parse_lsn(XLogRecPtr *ptr, const char *str) +{ + uint32 high, low; + + if (sscanf(str, "%x/%x", &high, &low) != 2) + return; + + *ptr = (((XLogRecPtr)high) << 32) + (XLogRecPtr)low; + + return; +} + + +static XLogRecPtr +parse_label_lsn(const char *label_key, const char *label_value) +{ + XLogRecPtr ptr = InvalidXLogRecPtr; + + parse_lsn(&ptr, label_value); + + /* parse_lsn() will not modify ptr if it can't parse the label value */ + if (ptr == InvalidXLogRecPtr) + { + log_error(_("couldn't parse backup label entry \"%s: %s\" as lsn"), + label_key, label_value); + } + + return ptr; +} + + + +/*====================================== + * Read entries of interest from the backup label. + * + * Sample backup label (with failover slots): + * + * START WAL LOCATION: 0/6000028 (file 000000010000000000000006) + * CHECKPOINT LOCATION: 0/6000060 + * BACKUP METHOD: streamed + * BACKUP FROM: master + * START TIME: 2016-03-30 12:18:12 AWST + * LABEL: pg_basebackup base backup + * MIN FAILOVER SLOT LSN: 0/5000000 + * + *====================================== + */ +static bool +read_backup_label(const char *local_data_directory, struct BackupLabel *out_backup_label) +{ + char label_path[MAXPGPATH]; + FILE *label_file; + int nmatches = 0; + + char line[MAXLEN]; + + out_backup_label->start_wal_location = InvalidXLogRecPtr; + out_backup_label->start_wal_file[0] = '\0'; + out_backup_label->checkpoint_location = InvalidXLogRecPtr; + out_backup_label->backup_from[0] = '\0'; + out_backup_label->backup_method[0] = '\0'; + out_backup_label->start_time[0] = '\0'; + out_backup_label->label[0] = '\0'; + out_backup_label->min_failover_slot_lsn = InvalidXLogRecPtr; + + maxlen_snprintf(label_path, "%s/backup_label", local_data_directory); + + label_file = fopen(label_path, "r"); + if (label_file == NULL) + { + log_error(_("read_backup_label: could not open backup label file %s: %s"), + label_path, strerror(errno)); + return false; + } + + log_info(_("read_backup_label: parsing backup label file '%s'"), + label_path); + + while(fgets(line, sizeof line, label_file) != NULL) + { + char label_key[MAXLEN]; + char label_value[MAXLEN]; + char newline; + + nmatches = sscanf(line, "%" MAXLEN_STR "[^:]: %" MAXLEN_STR "[^\n]%c", + label_key, label_value, &newline); + + if (nmatches != 3) + break; + + if (newline != '\n') + { + log_error(_("read_backup_label: line too long in backup label file. Line begins \"%s: %s\""), + label_key, label_value); + return false; + } + + log_verbose(LOG_DEBUG, "standby clone: got backup label entry \"%s: %s\"", + label_key, label_value); + + if (strcmp(label_key, "START WAL LOCATION") == 0) + { + char start_wal_location[MAXLEN]; + char wal_filename[MAXLEN]; + + nmatches = sscanf(label_value, "%" MAXLEN_STR "s (file %" MAXLEN_STR "[^)]", start_wal_location, wal_filename); + + if (nmatches != 2) + { + log_error(_("read_backup_label: unable to parse \"START WAL LOCATION\" in backup label")); + return false; + } + + out_backup_label->start_wal_location = + parse_label_lsn(&label_key[0], start_wal_location); + + if (out_backup_label->start_wal_location == InvalidXLogRecPtr) + return false; + + (void) strncpy(out_backup_label->start_wal_file, wal_filename, MAXLEN); + out_backup_label->start_wal_file[MAXLEN-1] = '\0'; + } + else if (strcmp(label_key, "CHECKPOINT LOCATION") == 0) + { + out_backup_label->checkpoint_location = + parse_label_lsn(&label_key[0], &label_value[0]); + + if (out_backup_label->checkpoint_location == InvalidXLogRecPtr) + return false; + } + else if (strcmp(label_key, "BACKUP METHOD") == 0) + { + (void) strncpy(out_backup_label->backup_method, label_value, MAXLEN); + out_backup_label->backup_method[MAXLEN-1] = '\0'; + } + else if (strcmp(label_key, "BACKUP FROM") == 0) + { + (void) strncpy(out_backup_label->backup_from, label_value, MAXLEN); + out_backup_label->backup_from[MAXLEN-1] = '\0'; + } + else if (strcmp(label_key, "START TIME") == 0) + { + (void) strncpy(out_backup_label->start_time, label_value, MAXLEN); + out_backup_label->start_time[MAXLEN-1] = '\0'; + } + else if (strcmp(label_key, "LABEL") == 0) + { + (void) strncpy(out_backup_label->label, label_value, MAXLEN); + out_backup_label->label[MAXLEN-1] = '\0'; + } + else if (strcmp(label_key, "MIN FAILOVER SLOT LSN") == 0) + { + out_backup_label->min_failover_slot_lsn = + parse_label_lsn(&label_key[0], &label_value[0]); + + if (out_backup_label->min_failover_slot_lsn == InvalidXLogRecPtr) + return false; + } + else + { + log_info("read_backup_label: ignored unrecognised backup label entry \"%s: %s\"", + label_key, label_value); + } + } + + (void) fclose(label_file); + + log_debug("read_backup_label: label is %s; start wal file is %s", + out_backup_label->label, out_backup_label->start_wal_file); + + return true; +} diff --git a/repmgr-client-global.h b/repmgr-client-global.h index 7b7596f4..18bea725 100644 --- a/repmgr-client-global.h +++ b/repmgr-client-global.h @@ -109,6 +109,11 @@ extern bool local_command(const char *command, PQExpBufferData *outputbuf); extern bool check_upstream_config(PGconn *conn, int server_version_num, bool exit_on_error); extern standy_clone_mode get_standby_clone_mode(void); +extern int copy_remote_files(char *host, char *remote_user, char *remote_path, + char *local_path, bool is_directory, int server_version_num); + extern void print_error_list(ItemList *error_list, int log_level); +extern char * make_pg_path(char *file); + #endif diff --git a/repmgr-client.c b/repmgr-client.c index 0723727e..69d0281b 100644 --- a/repmgr-client.c +++ b/repmgr-client.c @@ -17,6 +17,7 @@ #include + #include "repmgr.h" #include "repmgr-client.h" #include "repmgr-client-global.h" @@ -24,6 +25,8 @@ #include "repmgr-action-standby.h" #include "repmgr-action-cluster.h" +#include /* for PG_TEMP_FILE_PREFIX */ + /* globally available variables * * ============================ */ @@ -40,6 +43,8 @@ char pg_bindir[MAXLEN] = ""; char repmgr_slot_name[MAXLEN] = ""; char *repmgr_slot_name_ptr = NULL; +char path_buf[MAXLEN] = ""; + /* * if --node-id/--node-name provided, place that node's record here * for later use @@ -1580,3 +1585,123 @@ get_standby_clone_mode(void) return mode; } + + +char * +make_pg_path(char *file) +{ + maxlen_snprintf(path_buf, "%s%s", pg_bindir, file); + + return path_buf; +} + + +int +copy_remote_files(char *host, char *remote_user, char *remote_path, + char *local_path, bool is_directory, int server_version_num) +{ + PQExpBufferData rsync_flags; + char script[MAXLEN]; + char host_string[MAXLEN]; + int r; + + initPQExpBuffer(&rsync_flags); + + if (*config_file_options.rsync_options == '\0') + { + appendPQExpBuffer(&rsync_flags, "%s", + "--archive --checksum --compress --progress --rsh=ssh"); + } + else + { + appendPQExpBuffer(&rsync_flags, "%s", + config_file_options.rsync_options); + } + + if (runtime_options.force) + { + appendPQExpBuffer(&rsync_flags, "%s", + " --delete --checksum"); + } + + if (!remote_user[0]) + { + maxlen_snprintf(host_string, "%s", host); + } + else + { + maxlen_snprintf(host_string, "%s@%s", remote_user, host); + } + + /* + * When copying the main PGDATA directory, certain files and contents + * of certain directories need to be excluded. + * + * See function 'sendDir()' in 'src/backend/replication/basebackup.c' - + * we're basically simulating what pg_basebackup does, but with rsync rather + * than the BASEBACKUP replication protocol command. + * + * *However* currently we'll always copy the contents of the 'pg_replslot' + * directory and delete later if appropriate. + */ + if (is_directory) + { + /* Files which we don't want */ + appendPQExpBuffer(&rsync_flags, "%s", + " --exclude=postmaster.pid --exclude=postmaster.opts --exclude=global/pg_control"); + + appendPQExpBuffer(&rsync_flags, "%s", + " --exclude=recovery.conf --exclude=recovery.done"); + + if (server_version_num >= 90400) + { + /* + * Ideally we'd use PG_AUTOCONF_FILENAME from utils/guc.h, but + * that has too many dependencies for a mere client program. + */ + appendPQExpBuffer(&rsync_flags, "%s", + " --exclude=postgresql.auto.conf.tmp"); + } + + /* Temporary files which we don't want, if they exist */ + appendPQExpBuffer(&rsync_flags, " --exclude=%s*", + PG_TEMP_FILE_PREFIX); + + /* Directories which we don't want */ + + if (server_version_num >= 100000) + { + appendPQExpBuffer(&rsync_flags, "%s", + " --exclude=pg_wal/*"); + } + else + { + appendPQExpBuffer(&rsync_flags, "%s", + " --exclude=pg_xlog/*"); + } + + appendPQExpBuffer(&rsync_flags, "%s", + " --exclude=pg_log/* --exclude=pg_stat_tmp/*"); + + maxlen_snprintf(script, "rsync %s %s:%s/* %s", + rsync_flags.data, host_string, remote_path, local_path); + } + else + { + maxlen_snprintf(script, "rsync %s %s:%s %s", + rsync_flags.data, host_string, remote_path, local_path); + } + + log_info(_("rsync command line: '%s'"), script); + + r = system(script); + + log_debug("copy_remote_files(): r = %i; WIFEXITED: %i; WEXITSTATUS: %i", r, WIFEXITED(r), WEXITSTATUS(r)); + + /* exit code 24 indicates vanished files, which isn't a problem for us */ + if (WIFEXITED(r) && WEXITSTATUS(r) && WEXITSTATUS(r) != 24) + log_verbose(LOG_WARNING, "copy_remote_files(): rsync returned unexpected exit status %i", WEXITSTATUS(r)); + + return r; +} +