src/cmd.c | 6
src/delta_lease.c | 24 +++
src/delta_lease.h | 6
src/direct.c | 46 ++++++-
src/direct.h | 6
src/lockspace.c | 70 ++++++++++
src/main.c | 317 +++++++++++++++++++++++++++++++++++++++++--------
src/paxos_lease.c | 21 +++
src/paxos_lease.h | 4
src/sanlock_internal.h | 4
10 files changed, 445 insertions(+), 59 deletions(-)
New commits:
commit c0ca80a304c1af1808c674a363802fa522ed0a53
Author: David Teigland <teigland(a)redhat.com>
Date: Wed Aug 20 15:41:36 2014 -0500
sanlock: direct write_leader
Add a command to write a leader record directly to disk.
This is useful for testing, and to fix corrupted leases.
Signed-off-by: David Teigland <teigland(a)redhat.com>
diff --git a/src/delta_lease.c b/src/delta_lease.c
index 648b767..5b6dc9f 100644
--- a/src/delta_lease.c
+++ b/src/delta_lease.c
@@ -224,6 +224,30 @@ int delta_lease_leader_read(struct task *task, int io_timeout,
}
/*
+ * NB. this should not be used to write the leader record, it is meant only
+ * for manually clobbering the disk to corrupt it for testing, or to manually
+ * repair it after it's corrupted.
+ */
+
+int delta_lease_leader_clobber(struct task *task, int io_timeout,
+ struct sync_disk *disk,
+ uint64_t host_id,
+ struct leader_record *leader,
+ const char *caller)
+{
+ struct leader_record leader_end;
+ int rv;
+
+ leader_record_out(leader, &leader_end);
+
+ rv = write_sector(disk, host_id - 1, (char *)&leader_end, sizeof(struct leader_record),
+ task, io_timeout, caller);
+ if (rv < 0)
+ return rv;
+ return SANLK_OK;
+}
+
+/*
* delta_lease_acquire:
* set the owner of host_id to our_host_name.
*
diff --git a/src/delta_lease.h b/src/delta_lease.h
index f9992d4..968c7c7 100644
--- a/src/delta_lease.h
+++ b/src/delta_lease.h
@@ -56,4 +56,10 @@ int delta_read_lockspace(struct task *task,
int io_timeout,
int *io_timeout_ret);
+int delta_lease_leader_clobber(struct task *task, int io_timeout,
+ struct sync_disk *disk,
+ uint64_t host_id,
+ struct leader_record *leader,
+ const char *caller);
+
#endif
diff --git a/src/direct.c b/src/direct.c
index 24777ad..8813b2f 100644
--- a/src/direct.c
+++ b/src/direct.c
@@ -78,6 +78,7 @@ static int do_paxos_action(int action, struct task *task, int io_timeout,
int max_hosts, int num_hosts,
uint64_t local_host_id,
uint64_t local_host_generation,
+ struct leader_record *leader_in,
struct leader_record *leader_ret)
{
struct token *token;
@@ -138,6 +139,10 @@ static int do_paxos_action(int action, struct task *task, int io_timeout,
case ACT_READ_LEADER:
rv = paxos_lease_leader_read(task, token, &leader, "direct_read_leader");
break;
+
+ case ACT_WRITE_LEADER:
+ rv = paxos_lease_leader_clobber(task, token, leader_in, "direct_clobber");
+ break;
}
close_disks(token->disks, token->r.num_disks);
@@ -167,6 +172,7 @@ int direct_acquire(struct task *task, int io_timeout,
return do_paxos_action(ACT_ACQUIRE, task, io_timeout, res,
-1, num_hosts,
local_host_id, local_host_generation,
+ NULL,
leader_ret);
}
@@ -177,6 +183,7 @@ int direct_release(struct task *task, int io_timeout,
return do_paxos_action(ACT_RELEASE, task, io_timeout, res,
-1, -1,
0, 0,
+ NULL,
leader_ret);
}
@@ -186,6 +193,7 @@ static int do_delta_action(int action,
struct sanlk_lockspace *ls,
int max_hosts,
char *our_host_name,
+ struct leader_record *leader_in,
struct leader_record *leader_ret)
{
struct leader_record leader;
@@ -266,6 +274,11 @@ static int do_delta_action(int action,
&leader,
"direct_read");
break;
+ case ACT_WRITE_LEADER:
+ rv = delta_lease_leader_clobber(task, io_timeout, &sd,
+ ls->host_id,
+ leader_in,
+ "direct_clobber");
}
close_disks(&sd, 1);
@@ -289,17 +302,17 @@ static int do_delta_action(int action,
int direct_acquire_id(struct task *task, int io_timeout, struct sanlk_lockspace *ls,
char *our_host_name)
{
- return do_delta_action(ACT_ACQUIRE_ID, task, io_timeout, ls, -1, our_host_name, NULL);
+ return do_delta_action(ACT_ACQUIRE_ID, task, io_timeout, ls, -1, our_host_name, NULL, NULL);
}
int direct_release_id(struct task *task, int io_timeout, struct sanlk_lockspace *ls)
{
- return do_delta_action(ACT_RELEASE_ID, task, io_timeout, ls, -1, NULL, NULL);
+ return do_delta_action(ACT_RELEASE_ID, task, io_timeout, ls, -1, NULL, NULL, NULL);
}
int direct_renew_id(struct task *task, int io_timeout, struct sanlk_lockspace *ls)
{
- return do_delta_action(ACT_RENEW_ID, task, io_timeout, ls, -1, NULL, NULL);
+ return do_delta_action(ACT_RENEW_ID, task, io_timeout, ls, -1, NULL, NULL, NULL);
}
int direct_align(struct sync_disk *disk)
@@ -320,7 +333,7 @@ int direct_write_lockspace(struct task *task, struct sanlk_lockspace *ls,
return -1;
return do_delta_action(ACT_DIRECT_INIT, task, io_timeout, ls,
- max_hosts, NULL, NULL);
+ max_hosts, NULL, NULL, NULL);
}
int direct_write_resource(struct task *task, struct sanlk_resource *res,
@@ -336,7 +349,7 @@ int direct_write_resource(struct task *task, struct sanlk_resource *res,
return -ENODEV;
return do_paxos_action(ACT_DIRECT_INIT, task, 0, res,
- max_hosts, num_hosts, 0, 0, NULL);
+ max_hosts, num_hosts, 0, 0, NULL, NULL);
}
int direct_read_leader(struct task *task,
@@ -348,11 +361,30 @@ int direct_read_leader(struct task *task,
int rv = -1;
if (ls && ls->host_id_disk.path[0])
- rv = do_delta_action(ACT_READ_LEADER, task, io_timeout, ls, -1, NULL, leader_ret);
+ rv = do_delta_action(ACT_READ_LEADER, task, io_timeout, ls, -1, NULL, NULL, leader_ret);
else if (res)
rv = do_paxos_action(ACT_READ_LEADER, task, io_timeout, res,
- -1, -1, 0, 0, leader_ret);
+ -1, -1, 0, 0, NULL, leader_ret);
+ return rv;
+}
+
+int direct_write_leader(struct task *task,
+ int io_timeout,
+ struct sanlk_lockspace *ls,
+ struct sanlk_resource *res,
+ struct leader_record *leader)
+{
+ int rv = -1;
+
+ if (ls && ls->host_id_disk.path[0]) {
+ rv = do_delta_action(ACT_WRITE_LEADER, task, io_timeout, ls, -1, NULL, leader, NULL);
+
+ } else if (res) {
+ rv = do_paxos_action(ACT_WRITE_LEADER, task, io_timeout, res,
+ -1, -1, 0, 0, leader, NULL);
+ }
+
return rv;
}
diff --git a/src/direct.h b/src/direct.h
index dd7abbb..af2c4d1 100644
--- a/src/direct.h
+++ b/src/direct.h
@@ -45,6 +45,12 @@ int direct_read_leader(struct task *task, int io_timeout,
struct sanlk_resource *res,
struct leader_record *leader_ret);
+int direct_write_leader(struct task *task,
+ int io_timeout,
+ struct sanlk_lockspace *ls,
+ struct sanlk_resource *res,
+ struct leader_record *leader);
+
int direct_dump(struct task *task, char *dump_path, int force_mode);
int direct_next_free(struct task *task, char *path);
diff --git a/src/main.c b/src/main.c
index f729540..5d3b270 100644
--- a/src/main.c
+++ b/src/main.c
@@ -53,6 +53,7 @@
#include "cmd.h"
#include "helper.h"
#include "timeouts.h"
+#include "paxos_lease.h"
#define ONEMB 1048576
@@ -2006,6 +2007,8 @@ static int read_command_line(int argc, char *argv[])
com.action = ACT_NEXT_FREE;
else if (!strcmp(act, "read_leader"))
com.action = ACT_READ_LEADER;
+ else if (!strcmp(act, "write_leader"))
+ com.action = ACT_WRITE_LEADER;
else if (!strcmp(act, "acquire"))
com.action = ACT_ACQUIRE;
else if (!strcmp(act, "release"))
@@ -2071,6 +2074,9 @@ static int read_command_line(int argc, char *argv[])
case 'S':
log_syslog_priority = atoi(optionarg);
break;
+ case 'F':
+ com.file_path = strdup(optionarg);
+ break;
case 'a':
com.all = atoi(optionarg);
com.aio_arg = atoi(optionarg);
@@ -2671,23 +2677,269 @@ static int do_client(void)
return rv;
}
+#define MAX_LINE 128
+
+static int read_file_leader(struct leader_record *leader, int is_ls)
+{
+ FILE *file;
+ char line[MAX_LINE];
+ char field[MAX_LINE];
+ char val[MAX_LINE];
+ uint32_t checksum = 0;
+ uint32_t new_checksum;
+ struct leader_record lr;
+ int rv;
+
+ file = fopen(com.file_path, "r");
+ if (!file) {
+ log_tool("open error %d %s", errno, com.file_path);
+ return -1;
+ }
+
+ memcpy(&lr, leader, sizeof(lr));
+
+ memset(line, 0, sizeof(line));
+
+ while (fgets(line, MAX_LINE, file)) {
+
+ memset(field, 0, sizeof(field));
+ memset(val, 0, sizeof(val));
+
+ rv = sscanf(line, "%s %s", field, val);
+ if (rv != 2) {
+ log_tool("ignore line: \"%s\"", line);
+ continue;
+ }
+
+ if (!strcmp(field, "magic")) {
+ sscanf(val, "0x%x", &lr.magic);
+
+ } else if (!strcmp(field, "version")) {
+ sscanf(val, "0x%x", &lr.version);
+
+ } else if (!strcmp(field, "flags")) {
+ sscanf(val, "0x%x", &lr.flags);
+
+ } else if (!strcmp(field, "sector_size")) {
+ sscanf(val, "%u", &lr.sector_size);
+
+ } else if (!strcmp(field, "num_hosts")) {
+ sscanf(val, "%llu", (unsigned long long *)&lr.num_hosts);
+
+ } else if (!strcmp(field, "max_hosts")) {
+ sscanf(val, "%llu", (unsigned long long *)&lr.max_hosts);
+
+ } else if (!strcmp(field, "owner_id")) {
+ sscanf(val, "%llu", (unsigned long long *)&lr.owner_id);
+
+ } else if (!strcmp(field, "owner_generation")) {
+ sscanf(val, "%llu", (unsigned long long *)&lr.owner_generation);
+
+ } else if (!strcmp(field, "lver")) {
+ sscanf(val, "%llu", (unsigned long long *)&lr.lver);
+
+ } else if (!strcmp(field, "space_name")) {
+ strncpy(lr.space_name, val, NAME_ID_SIZE);
+
+ } else if (!strcmp(field, "resource_name")) {
+ strncpy(lr.resource_name, val, NAME_ID_SIZE);
+
+ } else if (!strcmp(field, "timestamp")) {
+ sscanf(val, "%llu", (unsigned long long *)&lr.timestamp);
+
+ } else if (!strcmp(field, "checksum")) {
+ sscanf(val, "0x%x", &checksum);
+
+ } else if (!strcmp(field, "io_timeout")) {
+ sscanf(val, "%hu", &lr.io_timeout);
+
+ } else if (is_ls && !strcmp(field, "extra1")) {
+ sscanf(val, "%llu", (unsigned long long *)&lr.write_id);
+
+ } else if (is_ls && !strcmp(field, "extra2")) {
+ sscanf(val, "%llu", (unsigned long long *)&lr.write_generation);
+
+ } else if (is_ls && !strcmp(field, "extra3")) {
+ sscanf(val, "%llu", (unsigned long long *)&lr.write_timestamp);
+
+ } else if (!is_ls && !strcmp(field, "write_id")) {
+ sscanf(val, "%llu", (unsigned long long *)&lr.write_id);
+
+ } else if (!is_ls && !strcmp(field, "write_generation")) {
+ sscanf(val, "%llu", (unsigned long long *)&lr.write_generation);
+
+ } else if (!is_ls && !strcmp(field, "write_timestamp")) {
+ sscanf(val, "%llu", (unsigned long long *)&lr.write_timestamp);
+ } else {
+ log_tool("ignore field: \"%s\"", field);
+ }
+
+ memset(line, 0, sizeof(line));
+ }
+
+ new_checksum = leader_checksum(&lr);
+
+ if (!com.force_mode) {
+ lr.checksum = new_checksum;
+ log_tool("use new generated checksum %x", new_checksum);
+ } else {
+ lr.checksum = checksum;
+ log_tool("warning: using specified checksum %x (generated is %x)",
+ checksum, new_checksum);
+ }
+
+ memcpy(leader, &lr, sizeof(lr));
+ return 0;
+}
+
+static void print_leader(struct leader_record *leader, int is_ls)
+{
+ log_tool("magic 0x%0x", leader->magic);
+ log_tool("version 0x%x", leader->version);
+ log_tool("flags 0x%x", leader->flags);
+ log_tool("sector_size %u", leader->sector_size);
+ log_tool("num_hosts %llu", (unsigned long long)leader->num_hosts);
+ log_tool("max_hosts %llu", (unsigned long long)leader->max_hosts);
+ log_tool("owner_id %llu", (unsigned long long)leader->owner_id);
+ log_tool("owner_generation %llu", (unsigned long long)leader->owner_generation);
+ log_tool("lver %llu", (unsigned long long)leader->lver);
+ log_tool("space_name %.48s", leader->space_name);
+ log_tool("resource_name %.48s", leader->resource_name);
+ log_tool("timestamp %llu", (unsigned long long)leader->timestamp);
+ log_tool("checksum 0x%0x", leader->checksum);
+ log_tool("io_timeout %u", leader->io_timeout);
+
+ if (!is_ls) {
+ log_tool("write_id %llu", (unsigned long long)leader->write_id);
+ log_tool("write_generation %llu", (unsigned long long)leader->write_generation);
+ log_tool("write_timestamp %llu", (unsigned long long)leader->write_timestamp);
+ } else {
+ log_tool("extra1 %llu", (unsigned long long)leader->write_id);
+ log_tool("extra2 %llu", (unsigned long long)leader->write_generation);
+ log_tool("extra3 %llu", (unsigned long long)leader->write_timestamp);
+ }
+}
+
+static int do_direct_read_leader(void)
+{
+ struct leader_record leader;
+ int rv;
+
+ rv = direct_read_leader(&main_task, com.io_timeout_arg,
+ &com.lockspace, com.res_args[0],
+ &leader);
+
+ log_tool("read_leader done %d", rv);
+
+ print_leader(&leader, com.res_args[0] ? 0 : 1);
+
+ return rv;
+}
+
+/*
+ * read the current leader record from disk, override any values found in
+ * the file, write back the result.
+ */
+
+static int do_direct_write_leader(void)
+{
+ struct leader_record leader;
+ char *res_str = NULL;
+ int is_ls = com.res_args[0] ? 0 : 1;
+ int rv;
+
+ memset(&leader, 0, sizeof(leader));
+
+ direct_read_leader(&main_task, com.io_timeout_arg,
+ &com.lockspace, com.res_args[0],
+ &leader);
+
+ rv = read_file_leader(&leader, is_ls);
+ if (rv < 0)
+ return rv;
+
+ /* make a record in the logs that this has been done */
+
+ if (is_ls) {
+ syslog(LOG_WARNING, "write_leader lockspace %.48s:%llu:%s:%llu",
+ com.lockspace.name,
+ (unsigned long long)com.lockspace.host_id,
+ com.lockspace.host_id_disk.path,
+ (unsigned long long)com.lockspace.host_id_disk.offset);
+ } else {
+ rv = sanlock_res_to_str(com.res_args[0], &res_str);
+ if (rv < 0) {
+ syslog(LOG_WARNING, "write_leader resource %.48s:%.48s",
+ com.res_args[0]->lockspace_name, com.res_args[0]->name);
+ } else {
+ syslog(LOG_WARNING, "write_leader resource %s", res_str);
+ }
+ }
+
+ rv = direct_write_leader(&main_task, com.io_timeout_arg,
+ &com.lockspace, com.res_args[0],
+ &leader);
+
+ log_tool("write_leader done %d", rv);
+
+ print_leader(&leader, is_ls);
+
+ if (res_str)
+ free(res_str);
+
+ return rv;
+}
+
+static int do_direct_init(void)
+{
+ char *res_str = NULL;
+ int rv;
+
+ if (com.lockspace.host_id_disk.path[0]) {
+ syslog(LOG_WARNING, "init lockspace %.48s:%llu:%s:%llu",
+ com.lockspace.name,
+ (unsigned long long)com.lockspace.host_id,
+ com.lockspace.host_id_disk.path,
+ (unsigned long long)com.lockspace.host_id_disk.offset);
+
+ rv = direct_write_lockspace(&main_task, &com.lockspace,
+ com.max_hosts, com.io_timeout_arg);
+ } else {
+ rv = sanlock_res_to_str(com.res_args[0], &res_str);
+ if (rv < 0) {
+ syslog(LOG_WARNING, "init resource %.48s:%.48s",
+ com.res_args[0]->lockspace_name, com.res_args[0]->name);
+ } else {
+ syslog(LOG_WARNING, "init resource %s", res_str);
+ }
+
+ rv = direct_write_resource(&main_task, com.res_args[0],
+ com.max_hosts, com.num_hosts);
+ }
+
+ log_tool("init done %d", rv);
+
+ if (res_str)
+ free(res_str);
+
+ return rv;
+}
+
static int do_direct(void)
{
struct leader_record leader;
int rv;
+ /* we want a record of any out-of-band changes to disk */
+ openlog("sanlock-direct", LOG_CONS | LOG_PID, LOG_DAEMON);
+
setup_task_aio(&main_task, com.aio_arg, DIRECT_AIO_CB_SIZE);
sprintf(main_task.name, "%s", "main_direct");
switch (com.action) {
+
case ACT_DIRECT_INIT:
- if (com.lockspace.host_id_disk.path[0])
- rv = direct_write_lockspace(&main_task, &com.lockspace,
- com.max_hosts, com.io_timeout_arg);
- else
- rv = direct_write_resource(&main_task, com.res_args[0],
- com.max_hosts, com.num_hosts);
- log_tool("init done %d", rv);
+ rv = do_direct_init();
break;
case ACT_DUMP:
@@ -2699,49 +2951,15 @@ static int do_direct(void)
break;
case ACT_READ_LEADER:
- rv = direct_read_leader(&main_task, com.io_timeout_arg,
- &com.lockspace, com.res_args[0],
- &leader);
- log_tool("read_leader done %d", rv);
- log_tool("magic 0x%0x", leader.magic);
- log_tool("version 0x%x", leader.version);
- log_tool("flags 0x%x", leader.flags);
- log_tool("sector_size %u", leader.sector_size);
- log_tool("num_hosts %llu",
- (unsigned long long)leader.num_hosts);
- log_tool("max_hosts %llu",
- (unsigned long long)leader.max_hosts);
- log_tool("owner_id %llu",
- (unsigned long long)leader.owner_id);
- log_tool("owner_generation %llu",
- (unsigned long long)leader.owner_generation);
- log_tool("lver %llu",
- (unsigned long long)leader.lver);
- log_tool("space_name %.48s", leader.space_name);
- log_tool("resource_name %.48s", leader.resource_name);
- log_tool("timestamp %llu",
- (unsigned long long)leader.timestamp);
- log_tool("checksum 0x%0x", leader.checksum);
- log_tool("io_timeout %u", leader.io_timeout);
-
- if (com.res_args[0]) {
- log_tool("write_id %llu",
- (unsigned long long)leader.write_id);
- log_tool("write_generation %llu",
- (unsigned long long)leader.write_generation);
- log_tool("write_timestamp %llu",
- (unsigned long long)leader.write_timestamp);
- } else {
- log_tool("extra1 %llu",
- (unsigned long long)leader.write_id);
- log_tool("extra2 %llu",
- (unsigned long long)leader.write_generation);
- log_tool("extra3 %llu",
- (unsigned long long)leader.write_timestamp);
- }
+ rv = do_direct_read_leader();
+ break;
+
+ case ACT_WRITE_LEADER:
+ rv = do_direct_write_leader();
break;
case ACT_ACQUIRE:
+ syslog(LOG_WARNING, "acquire");
rv = direct_acquire(&main_task, com.io_timeout_arg,
com.res_args[0], com.num_hosts,
com.host_id, com.host_generation,
@@ -2750,12 +2968,14 @@ static int do_direct(void)
break;
case ACT_RELEASE:
+ syslog(LOG_WARNING, "release");
rv = direct_release(&main_task, com.io_timeout_arg,
com.res_args[0], &leader);
log_tool("release done %d", rv);
break;
case ACT_ACQUIRE_ID:
+ syslog(LOG_WARNING, "acquire_id");
setup_host_name();
rv = direct_acquire_id(&main_task, com.io_timeout_arg,
@@ -2764,11 +2984,13 @@ static int do_direct(void)
break;
case ACT_RELEASE_ID:
+ syslog(LOG_WARNING, "release_id");
rv = direct_release_id(&main_task, com.io_timeout_arg, &com.lockspace);
log_tool("release_id done %d", rv);
break;
case ACT_RENEW_ID:
+ syslog(LOG_WARNING, "renew_id");
rv = direct_renew_id(&main_task, com.io_timeout_arg, &com.lockspace);
log_tool("rewew_id done %d", rv);
break;
@@ -2779,6 +3001,7 @@ static int do_direct(void)
}
close_task_aio(&main_task);
+ closelog();
return rv;
}
diff --git a/src/paxos_lease.c b/src/paxos_lease.c
index 33dacb6..7b8e95a 100644
--- a/src/paxos_lease.c
+++ b/src/paxos_lease.c
@@ -215,6 +215,27 @@ static int write_leader(struct task *task,
return rv;
}
+/*
+ * NB. this should not be used to write the leader record, it is meant only
+ * for manually clobbering the disk to corrupt it for testing, or to manually
+ * repair it after it's corrupted.
+ */
+
+int paxos_lease_leader_clobber(struct task *task,
+ struct token *token,
+ struct leader_record *leader,
+ const char *caller)
+{
+ struct leader_record lr_end;
+ int rv;
+
+ leader_record_out(leader, &lr_end);
+
+ rv = write_sector(&token->disks[0], 0, (char *)&lr_end, sizeof(struct leader_record),
+ task, token->io_timeout, caller);
+ return rv;
+}
+
#if 0
static int read_dblock(struct task *task,
struct sync_disk *disk,
diff --git a/src/paxos_lease.h b/src/paxos_lease.h
index 95b7dcd..84082f4 100644
--- a/src/paxos_lease.h
+++ b/src/paxos_lease.h
@@ -60,4 +60,8 @@ int paxos_erase_dblock(struct task *task,
struct token *token,
uint64_t host_id);
+int paxos_lease_leader_clobber(struct task *task,
+ struct token *token,
+ struct leader_record *leader,
+ const char *caller);
#endif
diff --git a/src/sanlock_internal.h b/src/sanlock_internal.h
index c6aabad..475af18 100644
--- a/src/sanlock_internal.h
+++ b/src/sanlock_internal.h
@@ -318,6 +318,7 @@ struct command_line {
int sh_retries;
uint32_t force_mode;
char our_host_name[SANLK_NAME_LEN+1];
+ char *file_path;
char *dump_path;
struct sanlk_lockspace lockspace; /* -s LOCKSPACE */
struct sanlk_resource *res_args[SANLK_MAX_RESOURCES]; /* -r RESOURCE */
@@ -360,6 +361,7 @@ enum {
ACT_VERSION,
ACT_SET_EVENT,
ACT_SET_CONFIG,
+ ACT_WRITE_LEADER,
};
EXTERN int external_shutdown;
commit dbb5e18a0a23a5dfeb8b0b6045e9fa9da8405b33
Author: David Teigland <teigland(a)redhat.com>
Date: Tue Aug 19 15:39:34 2014 -0500
sanlock: log bad delta leases and delta lease owners
Other delta leases in a lockspace are monitored for liveness,
which includes checking and saving the delta lease timestamp.
This adds a check of the delta lease's magic number and
lockspace name, and logs an error if they are not correct.
This will help debugging if a delta lease is corrupted.
This also adds an entry to sanlock.log that shows the owner
information for each delta lease. This is logged whenever
the sanlock daemon encounters new owner information for a
host in a lockspace.
Signed-off-by: David Teigland <teigland(a)redhat.com>
diff --git a/src/cmd.c b/src/cmd.c
index c67b88c..648ea7c 100644
--- a/src/cmd.c
+++ b/src/cmd.c
@@ -2190,14 +2190,16 @@ static int print_state_host(struct host_status *hs, char *str)
"owner_id=%llu "
"owner_generation=%llu "
"timestamp=%llu "
- "io_timeout=%u",
+ "io_timeout=%u "
+ "owner_name=%.48s",
(unsigned long long)hs->last_check,
(unsigned long long)hs->last_live,
(unsigned long long)hs->last_req,
(unsigned long long)hs->owner_id,
(unsigned long long)hs->owner_generation,
(unsigned long long)hs->timestamp,
- hs->io_timeout);
+ hs->io_timeout,
+ hs->owner_name);
return strlen(str) + 1;
}
diff --git a/src/lockspace.c b/src/lockspace.c
index 2db1cce..9322861 100644
--- a/src/lockspace.c
+++ b/src/lockspace.c
@@ -311,16 +311,80 @@ void check_other_leases(struct space *sp, char *buf)
leader_record_in(leader_end, &leader_in);
leader = &leader_in;
+ /*
+ * If this lease has invalid fields, log an error. Limit the logging
+ * frequency to avoid blowing up logs if some lease is left in a bad
+ * state for a long time. Should we check other fields in addition
+ * to magic and space_name?
+ */
+ if ((leader->magic != DELTA_DISK_MAGIC) || (strncmp(leader->space_name, sp->space_name, NAME_ID_SIZE))) {
+ if (!hs->lease_bad || !(hs->lease_bad % 100)) {
+ log_erros(sp, "check_other_lease invalid for host %llu %llu ts %llu name %.48s in %.48s",
+ (unsigned long long)hs->owner_id,
+ (unsigned long long)hs->owner_generation,
+ (unsigned long long)hs->timestamp,
+ hs->owner_name,
+ sp->space_name);
+ log_erros(sp, "check_other_lease leader %x owner %llu %llu ts %llu sn %.48s rn %.48s",
+ leader->magic,
+ (unsigned long long)leader->owner_id,
+ (unsigned long long)leader->owner_generation,
+ (unsigned long long)leader->timestamp,
+ leader->space_name,
+ leader->resource_name);
+ }
+ hs->lease_bad++;
+ if (!hs->lease_bad)
+ hs->lease_bad++;
+ } else {
+ if (hs->lease_bad) {
+ log_erros(sp, "check_other_lease corrected for host %llu %llu ts %llu name %.48s in %.48s",
+ (unsigned long long)hs->owner_id,
+ (unsigned long long)hs->owner_generation,
+ (unsigned long long)hs->timestamp,
+ hs->owner_name,
+ sp->space_name);
+ }
+ hs->lease_bad = 0;
+ }
+
+ /*
+ * Save a record of each new host instance to help with debugging.
+ */
+ if (!hs->lease_bad &&
+ (strncmp(hs->owner_name, leader->resource_name, NAME_ID_SIZE) ||
+ (hs->owner_generation != leader->owner_generation))) {
+ log_level(sp->space_id, 0, NULL, LOG_WARNING,
+ "host %llu %llu %llu %.48s",
+ (unsigned long long)leader->owner_id,
+ (unsigned long long)leader->owner_generation,
+ (unsigned long long)leader->timestamp,
+ leader->resource_name);
+ strncpy(hs->owner_name, leader->resource_name, NAME_ID_SIZE);
+ }
+
if (hs->owner_id == leader->owner_id &&
hs->owner_generation == leader->owner_generation &&
hs->timestamp == leader->timestamp) {
continue;
}
- hs->owner_id = leader->owner_id;
- hs->owner_generation = leader->owner_generation;
+ /*
+ * Replacing good values with potentially bad values
+ * would have no purpose, and would confuse things, so
+ * don't replace these fields if the lease is bad.
+ * But, continue to update the timestamp because we don't
+ * want to consider the host to be dead if the lease
+ * is being renewed, even if the lease has bad fields.
+ */
+ if (!hs->lease_bad) {
+ hs->owner_id = leader->owner_id;
+ hs->owner_generation = leader->owner_generation;
+ strncpy(hs->owner_name, leader->resource_name, NAME_ID_SIZE);
+ hs->io_timeout = leader->io_timeout;
+ }
+
hs->timestamp = leader->timestamp;
- hs->io_timeout = leader->io_timeout;
hs->last_live = now;
if (i+1 == sp->host_id)
diff --git a/src/sanlock_internal.h b/src/sanlock_internal.h
index 07d0029..c6aabad 100644
--- a/src/sanlock_internal.h
+++ b/src/sanlock_internal.h
@@ -156,6 +156,8 @@ struct host_status {
uint64_t timestamp; /* remote monotime */
uint64_t set_bit_time;
uint16_t io_timeout;
+ uint16_t lease_bad;
+ char owner_name[NAME_ID_SIZE];
};
/* The max number of connections that can get events for a lockspace. */
