cluster: STABLE3 - config: fix several issues with reload operation
Fabio M. Di Nitto
fabbione at fedoraproject.org
Mon Jul 26 12:49:04 UTC 2010
Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=ea01b37babe400a1f4545bd65e89d0ee6b35880e
Commit: ea01b37babe400a1f4545bd65e89d0ee6b35880e
Parent: 4500cb07c10baa701234656bbf11d8037d276111
Author: Fabio M. Di Nitto <fdinitto at redhat.com>
AuthorDate: Mon Jul 26 14:35:32 2010 +0200
Committer: Fabio M. Di Nitto <fdinitto at redhat.com>
CommitterDate: Mon Jul 26 14:35:32 2010 +0200
config: fix several issues with reload operation
cman-preconfig: in some cases it was possible to trigger cman to load
a config version with random version, trashing the old valid configuration.
- Enforce now a configuration version check before removing the old config
from the objdb and return an error back to cman/corosync instead.
cman corosync plugin: several config reload corner cases where not handled
properly, resulting in different unnecessary config reload attempts, and
unnecessary notification to all cluster daemons.
- move all config reload handling code in one location to avoid duplication.
- add more sanity checks on the configuration reload requests.
- send notification to the cluster only when we are able to load the config
ourselves.
- don't re-notify cluster nodes of config changes, unless the change
request started from our node.
- notify cluster daemons only if we successfully loaded the configuration.
- handle better some startup corner cases (cluster node joins with a newer
or older configuration).
- attempt to do better detection of first cluster transition.
- add some debugging output just in case....
Signed-off-by: Fabio M. Di Nitto <fdinitto at redhat.com>
---
cman/daemon/cman-preconfig.c | 29 +++++++
cman/daemon/commands.c | 166 +++++++++++++++++++++---------------------
2 files changed, 113 insertions(+), 82 deletions(-)
diff --git a/cman/daemon/cman-preconfig.c b/cman/daemon/cman-preconfig.c
index f741a8a..808b611 100644
--- a/cman/daemon/cman-preconfig.c
+++ b/cman/daemon/cman-preconfig.c
@@ -1137,6 +1137,8 @@ static int cmanpre_reloadconfig(struct objdb_iface_ver0 *objdb, int flush, const
hdb_handle_t object_handle;
hdb_handle_t find_handle;
hdb_handle_t cluster_parent_handle_new;
+ unsigned int config_version = 0, config_version_new = 0;
+ char *config_value = NULL;
/* don't reload if we've been told to run configless */
if (getenv("CMAN_NOCONFIG")) {
@@ -1159,6 +1161,33 @@ static int cmanpre_reloadconfig(struct objdb_iface_ver0 *objdb, int flush, const
}
objdb->object_find_destroy(find_handle);
+ if (!objdb->object_key_get(cluster_parent_handle, "config_version", strlen("config_version"), (void *)&config_value, NULL)) {
+ if (config_value) {
+ config_version = atoi(config_value);
+ } else {
+ /* it should never ever happen.. */
+ sprintf (error_reason, "%s", "Cannot find old /cluster/config_version key in configuration\n");
+ goto err;
+ }
+ }
+
+ config_value = NULL;
+
+ if (!objdb->object_key_get(cluster_parent_handle_new, "config_version", strlen("config_version"), (void *)&config_value, NULL)) {
+ if (config_value) {
+ config_version_new = atoi(config_value);
+ } else {
+ sprintf (error_reason, "%s", "Cannot find new /cluster/config_version key in configuration\n");
+ goto err;
+ }
+ }
+
+ if (config_version_new <= config_version) {
+ objdb->object_destroy(cluster_parent_handle_new);
+ sprintf (error_reason, "%s", "New configuration version has to be newer than current running configuration\n");
+ goto err;
+ }
+
/* destroy the old one */
objdb->object_destroy(cluster_parent_handle);
diff --git a/cman/daemon/commands.c b/cman/daemon/commands.c
index 604e400..36bb519 100644
--- a/cman/daemon/commands.c
+++ b/cman/daemon/commands.c
@@ -86,6 +86,7 @@ static int shutdown_yes;
static int shutdown_no;
static int shutdown_expected;
static int ccsd_timer_active = 0;
+static int ccsd_timer_should_broadcast = 0;
static struct cluster_node *find_node_by_nodeid(int nodeid);
static struct cluster_node *find_node_by_name(char *name);
@@ -98,7 +99,7 @@ static void recalculate_quorum(int allow_decrease, int by_current_nodes);
static void send_kill(int nodeid, uint16_t reason);
static const char *killmsg_reason(int reason);
static void ccsd_timer_fn(void *arg);
-static int reread_config(int new_version);
+static int reload_config(int new_version, int should_broadcast);
static void set_port_bit(struct cluster_node *node, uint8_t port)
{
@@ -484,23 +485,8 @@ static int do_cmd_set_version(char *cmdbuf, int *retlen)
version->patch != CNXMAN_PATCH_VERSION)
return -EINVAL;
- if (config_version == version->config)
- return 0;
-
- /* If the passed-in version number is 0 then read the file now, then
- * tell the other nodes to look for that version number.
- * That means we also have to send the notification here, because it will
- * be skipped when we get our own RECONFIGURE message back, as the version
- * number will match.
- */
- if (!version->config) {
- if (!reread_config(0))
- notify_listeners(NULL, EVENT_REASON_CONFIG_UPDATE, config_version);
- version->config = config_version;
- }
+ reload_config(version->config, 1);
- /* We will re-read CCS when we get our own message back */
- send_reconfigure(us->node_id, RECONFIG_PARAM_CONFIG_VERSION, version->config);
return 0;
}
@@ -1179,81 +1165,94 @@ static int do_cmd_unregister_quorum_device(char *cmdbuf, int *retlen)
return 0;
}
-static int reread_config(int new_version)
+static int reload_config(int new_version, int should_broadcast)
{
- int read_err;
const char *reload_err = NULL;
+ if (config_version == new_version) {
+ log_printf(LOG_DEBUG, "We are already using config version [%d]\n",
+ config_version);
+ return 0;
+ }
+
+ if (new_version > 0 && new_version < config_version) {
+ log_printf(LOG_ERR, "Requested version [%d] older than running version [%d]\n",
+ new_version, config_version);
+ return -1;
+ }
+
wanted_config_version = new_version;
/* Tell objdb to reload */
- read_err = corosync->object_reload_config(1, &reload_err);
+ config_error = corosync->object_reload_config(1, &reload_err);
+ if (config_error)
+ log_printf(LOG_ERR, "Unable to load new config in corosync: %s\n",
+ reload_err);
- /* Now get our bits */
- if (!read_err)
- read_err = read_cman_nodes(corosync, &config_version, 0);
+ if (!config_error)
+ config_error = read_cman_nodes(corosync, &config_version, 0);
- if (read_err) {
- config_error = 1;
+ if (config_error) {
log_printf(LOG_ERR, "Can't get updated config version %d: %s. Activity suspended on this node\n",
wanted_config_version, reload_err?reload_err:"version mismatch on this node");
- }
-
- /* Still too old?? */
- if (new_version && config_version < wanted_config_version) {
- log_printf(LOG_ERR, "Can't get updated config version %d, config file is version %d.\n",
- wanted_config_version, config_version);
- }
- /* Keep looking */
- if (read_err || config_version < wanted_config_version) {
- if (!ccsd_timer_active)
+ if (!ccsd_timer_active) {
+ log_printf(LOG_ERR, "Error reloading the configuration, will retry every second\n");
+ ccsd_timer_should_broadcast = should_broadcast;
corosync->timer_add_duration((unsigned long long)ccsd_poll_interval*1000000, NULL,
ccsd_timer_fn, &ccsd_timer);
- ccsd_timer_active = 1;
- }
- else {
+ ccsd_timer_active = 1;
+ }
+ } else {
+
+ /*
+ * at this point we know:
+ * config is loaded in objdb with a newer version than the previous one
+ * we have been able to activate it in cman (via read_cman_nodes)
+ */
+
+ if (should_broadcast) {
+ log_printf(LOG_DEBUG, "Sending reconfigure message to all nodes\n");
+ send_reconfigure(us->node_id, RECONFIG_PARAM_CONFIG_VERSION, config_version);
+ }
+
+ log_printf(LOG_DEBUG, "Recalculating quorum\n");
recalculate_quorum(1, 0);
- send_transition_msg(0,0);
- }
- return read_err;
+ log_printf(LOG_DEBUG, "Notify all listeners\n");
+ notify_listeners(NULL, EVENT_REASON_CONFIG_UPDATE, config_version);
+ }
+ return config_error;
}
static void ccsd_timer_fn(void *arg)
{
log_printf(LOG_DEBUG, "Polling configuration for updated information\n");
- ccsd_timer_active = 0;
+ ccsd_timer_active = 0;
- if (!reread_config(wanted_config_version) && config_version >= wanted_config_version) {
- log_printf(LOG_ERR, "Now got config information version %d, continuing\n", config_version);
+ if (!reload_config(wanted_config_version, ccsd_timer_should_broadcast) &&
+ config_version >= wanted_config_version) {
+ log_printf(LOG_DEBUG, "ccsd_timer_fn got the new config\n");
config_error = 0;
- recalculate_quorum(0, 0);
- notify_listeners(NULL, EVENT_REASON_CONFIG_UPDATE, config_version);
+ return;
}
- else {
- time_t now;
- now = time(NULL);
- log_printf(LOG_DEBUG, "Checking for startup failure: local_first_trans=%d, time=%d\n",
- local_first_trans, (int)(now - join_time));
- /*
- * If we haven't got the 'right' configuration at startup before (default) 30s
- * then quit so the node can boot
- */
+ if (local_first_trans) {
+ time_t now;
+ now = time(NULL);
- if (local_first_trans && now > join_time+startup_config_timeout) {
- log_printf(LOG_ERR, "Failed to get an up-to-date config file, wanted %d, only got %d. Will exit\n",
- wanted_config_version, config_version);
- log_printf(LOG_ERR, "Check your configuration distribution method is working correctly\n");
- cman_finish();
- corosync_shutdown();
- }
+ if (now > join_time+startup_config_timeout) {
+ log_printf(LOG_ERR, "Checking for startup failure: time=%d\n", (int)(now - join_time));
+ log_printf(LOG_ERR, "Failed to get an up-to-date config file, wanted %d, only got %d. Will exit\n",
+ wanted_config_version, config_version);
+ log_printf(LOG_ERR, "Check your configuration distribution method is working correctly\n");
+ cman_finish();
+ corosync_shutdown();
+ }
}
}
-
static void quorum_device_timer_fn(void *arg)
{
struct timeval now;
@@ -1736,25 +1735,31 @@ static int valid_transition_msg(int nodeid, struct cl_transmsg *msg)
return -1;
}
- /* New config version - try to read new file */
- if (msg->config_version > config_version) {
+ if (local_first_trans) {
+ time_t now;
+ now = time(NULL);
- if (!reread_config(msg->config_version)) {
+ if (now > join_time+startup_config_timeout) {
+ log_printf(LOG_DEBUG, "ccs: disable startup transition check\n");
+ local_first_trans = 0;
+ }
+ }
- if (config_version > msg->config_version) {
- /* Tell everyone else to update */
- send_reconfigure(us->node_id, RECONFIG_PARAM_CONFIG_VERSION, config_version);
+ /* New config version - try to read new file */
+ if (msg->config_version > config_version) {
+ log_printf(LOG_DEBUG, "Reloading config from TRANSITION message\n");
+ if (reload_config(msg->config_version, 0)) {
+ if (msg->config_version != config_version) {
+ log_printf(LOG_ERR, "Node %d conflict, remote config version id=%d, local=%d\n",
+ nodeid, msg->config_version, config_version);
+ return -1;
}
- recalculate_quorum(0, 0);
- notify_listeners(NULL, EVENT_REASON_CONFIG_UPDATE, config_version);
}
}
-
- if (msg->config_version != config_version) {
- log_printf(LOG_ERR, "Node %d conflict, remote config version id=%d, local=%d\n",
- nodeid, msg->config_version, config_version);
- return -1;
+ if ((msg->config_version == config_version) && (nodeid != us->node_id)) {
+ log_printf(LOG_DEBUG, "Completed first transition with nodes on the same config versions\n");
+ local_first_trans = 0;
}
return 0;
@@ -1897,10 +1902,7 @@ static void do_reconfigure_msg(void *data)
break;
case RECONFIG_PARAM_CONFIG_VERSION:
- if (config_version != msg->value) {
- if (!reread_config(msg->value))
- notify_listeners(NULL, EVENT_REASON_CONFIG_UPDATE, config_version);
- }
+ reload_config(msg->value, 0);
break;
}
}
More information about the cluster-commits
mailing list