github/repmgr
1
0
Fork 0
mirror of https://github.com/EnterpriseDB/repmgr.git synced 2026-03-23 15:16:29 +00:00
Code Issues Packages Projects Releases 1 Wiki Activity

Initial implementation of improved configuration file copying

Browse Source 
GitHub #210.
This commit is contained in:
Ian Barwick
2016-09-27 22:19:45 +09:00
parent 52328b8f33
commit bbb2e2f017
2 changed files with 203 additions and 130 deletions
Download Patch File Download Diff File Expand all files Collapse all files

304
repmgr.c
View File

@@ -153,6 +153,9 @@ static bool parse_conninfo_string(const char *conninfo_str, t_conninfo_param_lis
static void conn_to_param_list(PGconn *conn, t_conninfo_param_list *param_list);
static void parse_pg_basebackup_options(const char *pg_basebackup_options, t_basebackup_options *backup_options);
static void config_file_list_init(t_configfile_list *list, int max_size);
static void config_file_list_add(t_configfile_list *list, const char *file, const char *filename, bool in_data_dir);
/* Global variables */
static PQconninfoOption *opts = NULL;
@@ -216,7 +219,6 @@ main(int argc, char **argv)
{"help", no_argument, NULL, OPT_HELP},
{"check-upstream-config", no_argument, NULL, OPT_CHECK_UPSTREAM_CONFIG},
{"recovery-min-apply-delay", required_argument, NULL, OPT_RECOVERY_MIN_APPLY_DELAY},
{"ignore-external-config-files", no_argument, NULL, OPT_IGNORE_EXTERNAL_CONFIG_FILES},
{"config-archive-dir", required_argument, NULL, OPT_CONFIG_ARCHIVE_DIR},
{"pg_rewind", optional_argument, NULL, OPT_PG_REWIND},
{"pwprompt", optional_argument, NULL, OPT_PWPROMPT},
@@ -224,10 +226,12 @@ main(int argc, char **argv)
{"node", required_argument, NULL, OPT_NODE},
{"without-barman", no_argument, NULL, OPT_WITHOUT_BARMAN},
{"no-upstream-connection", no_argument, NULL, OPT_NO_UPSTREAM_CONNECTION},
{"copy-external-config-files", optional_argument, NULL, OPT_COPY_EXTERNAL_CONFIG_FILES},
{"version", no_argument, NULL, 'V'},
/* Following options deprecated */
{"local-port", required_argument, NULL, 'l'},
{"initdb-no-pwprompt", no_argument, NULL, OPT_INITDB_NO_PWPROMPT},
{"ignore-external-config-files", no_argument, NULL, OPT_IGNORE_EXTERNAL_CONFIG_FILES},
{NULL, 0, NULL, 0}
};
@@ -477,8 +481,23 @@ main(int argc, char **argv)
strncpy(runtime_options.recovery_min_apply_delay, optarg, MAXLEN);
break;
case OPT_IGNORE_EXTERNAL_CONFIG_FILES:
runtime_options.ignore_external_config_files = true;
case OPT_COPY_EXTERNAL_CONFIG_FILES:
runtime_options.copy_external_config_files = true;
if (optarg != NULL)
{
if (strcmp(optarg, "samepath") == 0)
{
runtime_options.copy_external_config_files_destination = CONFIG_FILE_SAMEPATH;
}
else if (strcmp(optarg, "pgdata") == 0)
{
runtime_options.copy_external_config_files_destination = CONFIG_FILE_PGDATA;
}
else
{
item_list_append(&cli_errors, _("Value provided for '--copy-external-config-files' must be 'samepath' or 'pgdata'"));
}
}
break;
case OPT_CONFIG_ARCHIVE_DIR:
strncpy(runtime_options.config_archive_dir, optarg, MAXLEN);
@@ -515,6 +534,9 @@ main(int argc, char **argv)
/* --initdb-no-pwprompt is deprecated */
item_list_append(&cli_warnings, _("--initdb-no-pwprompt is deprecated and has no effect; use -P/--pwprompt instead"));
break;
case OPT_IGNORE_EXTERNAL_CONFIG_FILES:
item_list_append(&cli_warnings, _("--ignore-external-config-files is deprecated and has no effect; use --copy-external-config-file instead"));
break;
default:
unknown_option:
@@ -1725,25 +1747,12 @@ do_standby_clone(void)
int i;
bool pg_start_backup_executed = false;
bool target_directory_provided = false;
bool external_config_file_copy_required = false;
char master_data_directory[MAXPGPATH];
char local_data_directory[MAXPGPATH];
char local_repmgr_directory[MAXPGPATH];
char master_config_file[MAXPGPATH] = "";
char local_config_file[MAXPGPATH] = "";
bool config_file_outside_pgdata = false;
char master_hba_file[MAXPGPATH] = "";
char local_hba_file[MAXPGPATH] = "";
bool hba_file_outside_pgdata = false;
char master_ident_file[MAXPGPATH] = "";
char local_ident_file[MAXPGPATH] = "";
bool ident_file_outside_pgdata = false;
char master_control_file[MAXPGPATH] = "";
char local_control_file[MAXPGPATH] = "";
@@ -1751,7 +1760,7 @@ do_standby_clone(void)
char *last_wal_segment = NULL;
PQExpBufferData event_details;
t_configfile_list config_files = T_CONFIGFILE_LIST_INITIALIZER;
/*
* Detecting the appropriate mode
@@ -1805,9 +1814,6 @@ do_standby_clone(void)
if (target_directory_provided == true)
{
strncpy(local_data_directory, runtime_options.dest_dir, MAXPGPATH);
strncpy(local_config_file, runtime_options.dest_dir, MAXPGPATH);
strncpy(local_hba_file, runtime_options.dest_dir, MAXPGPATH);
strncpy(local_ident_file, runtime_options.dest_dir, MAXPGPATH);
}
/*
* Otherwise use the same data directory as on the remote host
@@ -1815,9 +1821,6 @@ do_standby_clone(void)
else
{
strncpy(local_data_directory, master_data_directory, MAXPGPATH);
strncpy(local_config_file, master_config_file, MAXPGPATH);
strncpy(local_hba_file, master_hba_file, MAXPGPATH);
strncpy(local_ident_file, master_ident_file, MAXPGPATH);
log_notice(_("setting data directory to: %s\n"), local_data_directory);
log_hint(_("use -D/--data-dir to explicitly specify a data directory\n"));
@@ -1931,7 +1934,7 @@ do_standby_clone(void)
}
}
/* By default attempt to connect to the upstream server */
/* By default attempt to connect to the source server */
if (runtime_options.no_upstream_connection == false)
{
/* Attempt to connect to the upstream server to verify its configuration */
@@ -2161,7 +2164,6 @@ do_standby_clone(void)
PQfinish(source_conn);
exit(ERR_BAD_CONFIG);
}
}
if (mode != barman)
@@ -2215,43 +2217,89 @@ do_standby_clone(void)
}
}
if (get_pg_setting(source_conn, "data_directory", master_data_directory) == false)
{
log_err(_("Unable to retrieve upstream node's data directory\n"));
log_hint(_("STANDBY CLONE must be run as a database superuser"));
exit(ERR_BAD_CONFIG);
}
/*
* Obtain data directory and configuration file locations
* Obtain configuration file locations
* We'll check to see whether the configuration files are in the data
* directory - if not we'll have to copy them via SSH
* directory - if not we'll have to copy them via SSH, if copying
* requested.
*
* XXX: if configuration files are symlinks to targets outside the data
* directory, they won't be copied by pg_basebackup, but we can't tell
* this from the below query; we'll probably need to add a check for their
* presence and if missing force copy by SSH
*/
sqlquery_snprintf(sqlquery,
" WITH dd AS ( "
" SELECT setting "
" SELECT setting AS data_directory"
" FROM pg_catalog.pg_settings "
" WHERE name = 'data_directory' "
" ) "
" SELECT ps.name, ps.setting, "
" ps.setting ~ ('^' || dd.setting) AS in_data_dir "
" FROM dd, pg_settings ps "
" WHERE ps.name IN ('data_directory', 'config_file', 'hba_file', 'ident_file') "
" SELECT DISTINCT(sourcefile), "
" regexp_replace(sourcefile, '^.*\\/', '') AS filename, "
" sourcefile ~ ('^' || dd.data_directory) AS in_data_dir "
" FROM dd, pg_catalog.pg_settings ps "
" WHERE sourcefile IS NOT NULL "
" ORDER BY 1 ");
log_debug(_("standby clone: %s\n"), sqlquery);
res = PQexec(source_conn, sqlquery);
if (PQresultStatus(res) != PGRES_TUPLES_OK)
{
log_err(_("can't get info about data directory and configuration files: %s\n"),
log_err(_("unable to retrieve configuration file locations: %s\n"),
PQerrorMessage(source_conn));
PQclear(res);
PQfinish(source_conn);
exit(ERR_BAD_CONFIG);
}
/* We need all 4 parameters, and they can be retrieved only by superusers */
if (PQntuples(res) != 4)
/*
* allocate memory for config file array - number of rows returned from
* above query + 2 for pg_hba.conf, pg_ident.conf
*/
config_file_list_init(&config_files, PQntuples(res) + 2);
for (i = 0; i < PQntuples(res); i++)
{
log_err("STANDBY CLONE should be run by a SUPERUSER\n");
config_file_list_add(&config_files,
PQgetvalue(res, i, 0),
PQgetvalue(res, i, 1),
strcmp(PQgetvalue(res, i, 2), "t") == 1 ? true : false);
printf("file; %s\n", PQgetvalue(res, i, 0));
}
PQclear(res);
/* Fetch locations of pg_hba.conf and pg_ident.conf */
sqlquery_snprintf(sqlquery,
" WITH dd AS ( "
" SELECT setting AS data_directory"
" FROM pg_catalog.pg_settings "
" WHERE name = 'data_directory' "
" ) "
" SELECT ps.setting, "
" regexp_replace(setting, '^.*\\/', '') AS filename, "
" ps.setting ~ ('^' || dd.data_directory) AS in_data_dir "
" FROM dd, pg_catalog.pg_settings ps "
" WHERE ps.name IN ('hba_file', 'ident_file') "
" ORDER BY 1 ");
log_debug(_("standby clone: %s\n"), sqlquery);
res = PQexec(source_conn, sqlquery);
if (PQresultStatus(res) != PGRES_TUPLES_OK)
{
log_err(_("unable to retrieve configuration file locations: %s\n"),
PQerrorMessage(source_conn));
PQclear(res);
PQfinish(source_conn);
exit(ERR_BAD_CONFIG);
@@ -2259,39 +2307,10 @@ do_standby_clone(void)
for (i = 0; i < PQntuples(res); i++)
{
if (strcmp(PQgetvalue(res, i, 0), "data_directory") == 0)
{
strncpy(master_data_directory, PQgetvalue(res, i, 1), MAXPGPATH);
}
else if (strcmp(PQgetvalue(res, i, 0), "config_file") == 0)
{
if (strcmp(PQgetvalue(res, i, 2), "f") == 0)
{
config_file_outside_pgdata = true;
external_config_file_copy_required = true;
strncpy(master_config_file, PQgetvalue(res, i, 1), MAXPGPATH);
}
}
else if (strcmp(PQgetvalue(res, i, 0), "hba_file") == 0)
{
if (strcmp(PQgetvalue(res, i, 2), "f") == 0)
{
hba_file_outside_pgdata = true;
external_config_file_copy_required = true;
strncpy(master_hba_file, PQgetvalue(res, i, 1), MAXPGPATH);
}
}
else if (strcmp(PQgetvalue(res, i, 0), "ident_file") == 0)
{
if (strcmp(PQgetvalue(res, i, 2), "f") == 0)
{
ident_file_outside_pgdata = true;
external_config_file_copy_required = true;
strncpy(master_ident_file, PQgetvalue(res, i, 1), MAXPGPATH);
}
}
else
log_warning(_("unknown parameter: %s\n"), PQgetvalue(res, i, 0));
config_file_list_add(&config_files,
PQgetvalue(res, i, 0),
PQgetvalue(res, i, 1),
strcmp(PQgetvalue(res, i, 2), "t") == 1 ? true : false);
}
PQclear(res);
@@ -2849,67 +2868,67 @@ do_standby_clone(void)
}
}
/*
* If configuration files were not inside the data directory, we'll need to
* copy them via SSH (unless `--ignore-external-config-files` was provided)
*
* TODO: add option to place these files in the same location on the
* standby server as on the primary?
*/
if (mode != barman &&
external_config_file_copy_required &&
!runtime_options.ignore_external_config_files)
/*
* If `--copy-external-config-files` was provided, copy any configuration
* files detected to the appropriate location. Any errors encountered
* will not be treated as fatal.
*/
if (runtime_options.copy_external_config_files && upstream_record_found)
{
log_notice(_("copying configuration files from master\n"));
r = test_ssh_connection(runtime_options.host, runtime_options.remote_user);
int i;
t_configfile_info *file;
char *host;
/* get host from upstream record */
host = param_get(&recovery_conninfo, "host");
if (host == NULL)
host = runtime_options.host;
log_verbose(LOG_DEBUG, "host for config file is: %s\n", host);
log_notice(_("copying external configuration files from upstream node\n"));
r = test_ssh_connection(host, runtime_options.remote_user);
if (r != 0)
{
log_err(_("aborting, remote host %s is not reachable.\n"),
runtime_options.host);
retval = ERR_BAD_SSH;
goto stop_backup;
log_err(_("remote host %s is not reachable via SSH - unable to copy external configuration files\n"),
host);
}
if (config_file_outside_pgdata)
else
{
log_info(_("standby clone: master config file '%s'\n"), master_config_file);
r = copy_remote_files(runtime_options.host, runtime_options.remote_user,
master_config_file, local_config_file, false, server_version_num);
if (r != 0)
for (i = 0; i < config_files.entries; i++)
{
log_err(_("standby clone: failed copying master config file '%s'\n"),
master_config_file);
retval = ERR_BAD_SSH;
goto stop_backup;
}
}
char dest_path[MAXPGPATH];
file = config_files.files[i];
if (hba_file_outside_pgdata)
{
log_info(_("standby clone: master hba file '%s'\n"), master_hba_file);
r = copy_remote_files(runtime_options.host, runtime_options.remote_user,
master_hba_file, local_hba_file, false, server_version_num);
if (r != 0)
{
log_err(_("standby clone: failed copying master hba file '%s'\n"),
master_hba_file);
retval = ERR_BAD_SSH;
goto stop_backup;
}
}
/*
* Skip files in the data directory - these will be copied during
* the main backup
*/
if (file->in_data_directory == true)
continue;
if (ident_file_outside_pgdata)
{
log_info(_("standby clone: master ident file '%s'\n"), master_ident_file);
r = copy_remote_files(runtime_options.host, runtime_options.remote_user,
master_ident_file, local_ident_file, false, server_version_num);
if (r != 0)
{
log_err(_("standby clone: failed copying master ident file '%s'\n"),
master_ident_file);
retval = ERR_BAD_SSH;
goto stop_backup;
if (runtime_options.copy_external_config_files_destination == CONFIG_FILE_SAMEPATH)
{
strncpy(dest_path, file->filepath, MAXPGPATH);
}
else
{
snprintf(dest_path, MAXPGPATH,
"%s/%s",
local_data_directory,
file->filename);
}
r = copy_remote_files(runtime_options.host, runtime_options.remote_user,
file->filepath, dest_path, false, server_version_num);
if (r != 0)
{
log_err(_("standby clone: unable to copying config file '%s'\n"),
file->filename);
}
}
}
}
@@ -3037,6 +3056,7 @@ stop_backup:
/* delete the backup label file copied from the primary */
maxlen_snprintf(label_path, "%s/backup_label", local_data_directory);
// XXX!
if (0 && unlink(label_path) < 0 && errno != ENOENT)
{
log_warning(_("unable to delete backup label file %s\n"), label_path);
@@ -5450,9 +5470,11 @@ do_help(void)
printf(_(" --without-barman (standby clone) do not use Barman even if configured\n"));
printf(_(" --recovery-min-apply-delay=VALUE (standby clone, follow) set recovery_min_apply_delay\n" \
" in recovery.conf (PostgreSQL 9.4 and later)\n"));
printf(_(" --ignore-external-config-files (standby clone) don't copy configuration files located\n" \
" outside the data directory when cloning a standby\n"));
printf(_(" -w, --wal-keep-segments=VALUE (standby clone) minimum value for the GUC\n" \
printf(_(" --copy-external-config-files[={samepath|pgdata}]\n" \
" (standby clone) copy configuration files located outside the \n" \
" data directory to the same path on the standby (default) or to the\n" \
" PostgreSQL data directory\n"));
printf(_(" -w, --wal-keep-segments (standby clone) minimum value for the GUC\n" \
" wal_keep_segments (default: %s)\n"), DEFAULT_WAL_KEEP_SEGMENTS);
printf(_(" -W, --wait (standby follow) wait for a master to appear\n"));
printf(_(" -m, --mode (standby switchover) shutdown mode (\"fast\" - default, \"smart\" or \"immediate\")\n"));
@@ -6077,9 +6099,9 @@ check_parameters_for_action(const int action)
item_list_append(&cli_warnings, _("-c/--fast-checkpoint can only be used when executing STANDBY CLONE"));
}
if (runtime_options.ignore_external_config_files)
if (runtime_options.copy_external_config_files)
{
item_list_append(&cli_warnings, _("--ignore-external-config-files can only be used when executing STANDBY CLONE"));
item_list_append(&cli_warnings, _("--copy-external-config-files can only be used when executing STANDBY CLONE"));
}
if (*runtime_options.recovery_min_apply_delay)
@@ -7404,6 +7426,7 @@ parse_pg_basebackup_options(const char *pg_basebackup_options, t_basebackup_opti
int optindex = 0;
/* We're only interested in these options */
static struct option long_options[] =
{
{"slot", required_argument, NULL, 'S'},
@@ -7476,3 +7499,32 @@ parse_pg_basebackup_options(const char *pg_basebackup_options, t_basebackup_opti
return;
}
static void
config_file_list_init(t_configfile_list *list, int max_size)
{
list->size = max_size;
list->entries = 0;
list->files = pg_malloc0(sizeof(t_configfile_info *) * max_size);
}
static void
config_file_list_add(t_configfile_list *list, const char *file, const char *filename, bool in_data_dir)
{
/* Failsafe to prevent entries being added beyond the end */
if (list->entries == list->size)
return;
list->files[list->entries] = pg_malloc0(sizeof(t_configfile_info));
strncpy(list->files[list->entries]->filepath, file, MAXPGPATH);
canonicalize_path(list->files[list->entries]->filepath);
strncpy(list->files[list->entries]->filename, filename, MAXPGPATH);
list->files[list->entries]->in_data_directory = in_data_dir;
list->entries ++;
}

29
repmgr.h
View File

@@ -52,7 +52,7 @@
#define OPT_HELP 1
#define OPT_CHECK_UPSTREAM_CONFIG 2
#define OPT_RECOVERY_MIN_APPLY_DELAY 3
#define OPT_IGNORE_EXTERNAL_CONFIG_FILES 4
#define OPT_COPY_EXTERNAL_CONFIG_FILES 4
#define OPT_CONFIG_ARCHIVE_DIR 5
#define OPT_PG_REWIND 6
#define OPT_PWPROMPT 7
@@ -63,6 +63,10 @@
/* deprecated command line options */
#define OPT_INITDB_NO_PWPROMPT 999
#define OPT_IGNORE_EXTERNAL_CONFIG_FILES 998
#define CONFIG_FILE_SAMEPATH 1
#define CONFIG_FILE_PGDATA 2
/* Run time options type */
@@ -84,10 +88,11 @@ typedef struct
bool witness_pwprompt;
bool rsync_only;
bool fast_checkpoint;
bool ignore_external_config_files;
bool csv_mode;
bool without_barman;
bool no_upstream_connection;
bool copy_external_config_files;
int copy_external_config_files_destination;
char masterport[MAXLEN];
/*
* configuration file parameters which can be overridden on the
@@ -103,7 +108,7 @@ typedef struct
char config_archive_dir[MAXLEN];
/* parameter used by CLUSTER CLEANUP */
int keep_history;
/* paramater used by {STANDBY|WITNESS} UNREGISTER */
/* parameter used by {STANDBY|WITNESS} UNREGISTER */
int node;
char pg_bindir[MAXLEN];
@@ -111,7 +116,7 @@ typedef struct
char recovery_min_apply_delay[MAXLEN];
} t_runtime_options;
#define T_RUNTIME_OPTIONS_INITIALIZER { "", "", "", "", "", "", "", DEFAULT_WAL_KEEP_SEGMENTS, false, false, false, false, false, false, false, false, false, false, false, false, "", "", "", "", "fast", "", 0, 0, "", ""}
#define T_RUNTIME_OPTIONS_INITIALIZER { "", "", "", "", "", "", "", DEFAULT_WAL_KEEP_SEGMENTS, false, false, false, false, false, false, false, false, false, false, false, false, CONFIG_FILE_SAMEPATH, "", "", "", "", "fast", "", 0, 0, "", ""}
struct BackupLabel
{
@@ -141,4 +146,20 @@ typedef struct
char **values;
} t_conninfo_param_list;
typedef struct
{
char filepath[MAXPGPATH];
char filename[MAXPGPATH];
bool in_data_directory;
} t_configfile_info;
typedef struct
{
int size;
int entries;
t_configfile_info **files;
} t_configfile_list;
#define T_CONFIGFILE_LIST_INITIALIZER { 0, 0, NULL }
#endif