src/delta_lease.c | 33 ++++++++++++-----
src/delta_lease.h | 1
src/direct.c | 21 +++++++++--
src/direct_lib.c | 15 +++++++
src/host_id.c | 92 ++++++++++++++++++++++++++++++++++++++++++-------
src/host_id.h | 3 +
src/leader.h | 7 +++
src/main.c | 8 +---
src/paxos_lease.c | 12 +++++-
src/sanlock_internal.h | 1
src/sanlock_rv.h | 8 +++-
src/token_manager.c | 24 +++++++++++-
12 files changed, 188 insertions(+), 37 deletions(-)
New commits:
commit ae88c11978cc17f70309c150a1e3115f7f824f1e
Author: David Teigland <teigland(a)redhat.com>
Date: Mon Aug 29 17:37:39 2011 -0500
sanlock: set bitmap for requests
After a request_record for a lease is written,
set the bit of the current lease owner in our
own host_id bitmap the next time we renew.
Leave the bit set for request_finish_seconds.
diff --git a/src/delta_lease.c b/src/delta_lease.c
index f8dc729..53cbd5e 100644
--- a/src/delta_lease.c
+++ b/src/delta_lease.c
@@ -324,17 +324,18 @@ int delta_lease_renew(struct task *task,
struct space *sp,
struct sync_disk *disk,
char *space_name,
+ char *bitmap,
int prev_result,
int *read_result,
struct leader_record *leader_last,
struct leader_record *leader_ret)
{
struct leader_record leader;
- uint64_t host_id, offset;
- uint64_t new_ts;
char **p_iobuf;
- int iobuf_len, io_timeout_save;
- int rv;
+ char **p_wbuf;
+ char *wbuf;
+ uint64_t host_id, id_offset, new_ts;
+ int rv, iobuf_len, sector_size, io_timeout_save;
if (!leader_last)
return -EINVAL;
@@ -345,9 +346,11 @@ int delta_lease_renew(struct task *task,
iobuf_len = sp->align_size;
+ sector_size = disk->sector_size;
+
/* offset of our leader_record */
- offset = (host_id - 1) * disk->sector_size;
- if (offset > iobuf_len)
+ id_offset = (host_id - 1) * sector_size;
+ if (id_offset > iobuf_len)
return -EINVAL;
@@ -429,7 +432,7 @@ int delta_lease_renew(struct task *task,
read_done:
*read_result = SANLK_OK;
- memcpy(&leader, task->iobuf+offset, sizeof(struct leader_record));
+ memcpy(&leader, task->iobuf+id_offset, sizeof(struct leader_record));
rv = verify_leader(disk, space_name, host_id, &leader, "delta_renew");
if (rv < 0)
@@ -466,6 +469,16 @@ int delta_lease_renew(struct task *task,
leader.timestamp = new_ts;
leader.checksum = leader_checksum(&leader);
+ p_wbuf = &wbuf;
+ rv = posix_memalign((void *)p_wbuf, getpagesize(), sector_size);
+ if (rv) {
+ log_erros(sp, "dela_renew write memalign rv %d", rv);
+ return -ENOMEM;
+ }
+ memset(wbuf, 0, sector_size);
+ memcpy(wbuf, &leader, sizeof(struct leader_record));
+ memcpy(wbuf+LEADER_RECORD_MAX, bitmap, HOSTID_BITMAP_SIZE);
+
/* extend io timeout for this one write; we need to give this write
every chance to succeed, and there's no point in letting it time
out. there's nothing we would do but retry it, and timing out and
@@ -474,8 +487,10 @@ int delta_lease_renew(struct task *task,
io_timeout_save = task->io_timeout_seconds;
task->io_timeout_seconds = task->host_dead_seconds;
- rv = write_sector(disk, host_id - 1, (char *)&leader, sizeof(struct leader_record),
- task, "delta_leader");
+ rv = write_iobuf(disk->fd, disk->offset+id_offset, wbuf, sector_size, task);
+
+ if (rv != SANLK_AIO_TIMEOUT)
+ free(wbuf);
task->io_timeout_seconds = io_timeout_save;
diff --git a/src/delta_lease.h b/src/delta_lease.h
index 4a0cad2..4739cc3 100644
--- a/src/delta_lease.h
+++ b/src/delta_lease.h
@@ -28,6 +28,7 @@ int delta_lease_renew(struct task *task,
struct space *sp,
struct sync_disk *disk,
char *space_name,
+ char *bitmap,
int prev_result,
int *read_result,
struct leader_record *leader_last,
diff --git a/src/direct.c b/src/direct.c
index 6383787..a095803 100644
--- a/src/direct.c
+++ b/src/direct.c
@@ -191,8 +191,11 @@ static int do_delta_action(int action,
struct leader_record leader;
struct sync_disk sd;
struct space space;
+ char bitmap[HOSTID_BITMAP_SIZE];
int read_result, rv;
+ memset(bitmap, 0, sizeof(bitmap));
+
/* for log_space in delta functions */
memset(&space, 0, sizeof(space));
@@ -229,6 +232,7 @@ static int do_delta_action(int action,
return rv;
rv = delta_lease_renew(task, &space, &sd,
ls->name,
+ bitmap,
-1,
&read_result,
&leader,
@@ -420,9 +424,11 @@ int direct_read_leader(struct task *task,
return rv;
}
+int test_id_bit(int host_id, char *bitmap);
+
int direct_dump(struct task *task, char *dump_path, int force_mode)
{
- char *data;
+ char *data, *bitmap;
char *colon, *off_str;
struct leader_record *lr;
struct request_record *rr;
@@ -431,7 +437,7 @@ int direct_dump(struct task *task, char *dump_path, int force_mode)
char rname[NAME_ID_SIZE+1];
uint64_t sector_nr;
int sector_count, datalen, align_size;
- int i, rv;
+ int i, rv, b;
memset(&sd, 0, sizeof(struct sync_disk));
@@ -503,14 +509,21 @@ int direct_dump(struct task *task, char *dump_path, int force_mode)
strncpy(sname, lr->space_name, NAME_ID_SIZE);
strncpy(rname, lr->resource_name, NAME_ID_SIZE);
- printf("%08llu %36s %48s %010llu %04llu %04llu\n",
+ printf("%08llu %36s %48s %010llu %04llu %04llu",
(unsigned long long)((sector_nr + i) * sd.sector_size),
sname, rname,
(unsigned long long)lr->timestamp,
(unsigned long long)lr->owner_id,
(unsigned long long)lr->owner_generation);
- /* TODO: if force_mode, print host_ids set in bitmap */
+ if (force_mode) {
+ bitmap = (char *)lr + LEADER_RECORD_MAX;
+ for (b = 0; b < DEFAULT_MAX_HOSTS; b++) {
+ if (test_id_bit(b+1, bitmap))
+ printf(" %d", b+1);
+ }
+ }
+ printf("\n");
}
} else if (lr->magic == PAXOS_DISK_MAGIC) {
strncpy(sname, lr->space_name, NAME_ID_SIZE);
diff --git a/src/direct_lib.c b/src/direct_lib.c
index 2cb79c3..e2c7d7a 100644
--- a/src/direct_lib.c
+++ b/src/direct_lib.c
@@ -38,6 +38,21 @@ int host_id_disk_info(char *name GNUC_UNUSED, struct sync_disk *disk
GNUC_UNUSED
return -1;
}
+/* copied from host_id.c */
+
+int test_id_bit(int host_id, char *bitmap);
+
+int test_id_bit(int host_id, char *bitmap)
+{
+ char *byte = bitmap + ((host_id - 1) / 8);
+ unsigned int bit = (host_id - 1) % 8;
+ char mask;
+
+ mask = 1 << bit;
+
+ return (*byte & mask);
+}
+
int get_rand(int a, int b);
int get_rand(int a, int b)
diff --git a/src/host_id.c b/src/host_id.c
index 0688b93..a1cc468 100644
--- a/src/host_id.c
+++ b/src/host_id.c
@@ -170,22 +170,85 @@ static void clear_bit(int host_id, char *bitmap)
*byte &= ~bit;
}
+#endif
-static void set_bit(int host_id, char *bitmap)
+static void set_id_bit(int host_id, char *bitmap, char *c)
{
char *byte = bitmap + ((host_id - 1) / 8);
- unsigned int bit = host_id % 8;
+ unsigned int bit = (host_id - 1) % 8;
+ char mask;
+
+ mask = 1 << bit;
- *byte |= bit;
+ *byte |= mask;
+
+ *c = *byte;
}
-#endif
-static int test_bit(int host_id, char *bitmap)
+/* FIXME: another copy in direct_lib.c */
+
+int test_id_bit(int host_id, char *bitmap)
{
char *byte = bitmap + ((host_id - 1) / 8);
- unsigned int bit = host_id % 8;
+ unsigned int bit = (host_id - 1) % 8;
+ char mask;
+
+ mask = 1 << bit;
- return *byte & bit;
+ return (*byte & mask);
+}
+
+int host_info_set_bit(char *space_name, uint64_t host_id)
+{
+ struct space *sp;
+ int found = 0;
+
+ if (!host_id || host_id > DEFAULT_MAX_HOSTS)
+ return -EINVAL;
+
+ pthread_mutex_lock(&spaces_mutex);
+ list_for_each_entry(sp, &spaces, list) {
+ if (strncmp(sp->space_name, space_name, NAME_ID_SIZE))
+ continue;
+ found = 1;
+ break;
+ }
+ pthread_mutex_unlock(&spaces_mutex);
+
+ if (!found)
+ return -ENOSPC;
+
+ pthread_mutex_lock(&sp->mutex);
+ sp->host_info[host_id-1].set_bit_time = monotime();
+ pthread_mutex_unlock(&sp->mutex);
+ return 0;
+}
+
+static void create_bitmap(struct task *task, struct space *sp, char *bitmap)
+{
+ uint64_t now;
+ int i;
+ char c;
+
+ now = monotime();
+
+ pthread_mutex_lock(&sp->mutex);
+ for (i = 0; i < DEFAULT_MAX_HOSTS; i++) {
+ if (i+1 == sp->host_id)
+ continue;
+
+ if (!sp->host_info[i].set_bit_time)
+ continue;
+
+ if (now - sp->host_info[i].set_bit_time > task->request_finish_seconds) {
+ log_space(sp, "bitmap clear host_id %d", i+1);
+ sp->host_info[i].set_bit_time = 0;
+ } else {
+ set_id_bit(i+1, bitmap, &c);
+ log_space(sp, "bitmap set host_id %d byte %x", i+1, c);
+ }
+ }
+ pthread_mutex_unlock(&sp->mutex);
}
/*
@@ -206,7 +269,7 @@ void check_other_leases(struct task *task, struct space *sp, char
*buf)
struct host_info *info;
char *bitmap;
uint64_t now;
- int i, new;
+ int i, new = 0;
disk = &sp->host_id_disk;
@@ -235,7 +298,7 @@ void check_other_leases(struct task *task, struct space *sp, char
*buf)
bitmap = (char *)leader + HOSTID_BITMAP_OFFSET;
- if (!test_bit(sp->host_id, bitmap))
+ if (!test_id_bit(sp->host_id, bitmap))
continue;
/* this host has made a request for us, we won't take a new
@@ -246,7 +309,7 @@ void check_other_leases(struct task *task, struct space *sp, char
*buf)
log_space(sp, "request from host_id %d", i+1);
info->last_req = now;
- new = 1;
+ new++;
}
/* TODO: add a thread that will periodically scan spaces and
@@ -331,6 +394,7 @@ static int corrupt_result(int result)
static void *lockspace_thread(void *arg_in)
{
+ char bitmap[HOSTID_BITMAP_SIZE];
struct task task;
struct space *sp;
struct leader_record leader;
@@ -441,11 +505,15 @@ static void *lockspace_thread(void *arg_in)
* and the length of time between successful renewals
*/
+ memset(bitmap, 0, sizeof(bitmap));
+ create_bitmap(&task, sp, bitmap);
+
delta_begin = monotime();
delta_result = delta_lease_renew(&task, sp, &sp->host_id_disk,
- sp->space_name, delta_result,
- &read_result, &leader, &leader);
+ sp->space_name, bitmap,
+ delta_result, &read_result,
+ &leader, &leader);
delta_length = monotime() - delta_begin;
if (delta_result == SANLK_OK) {
diff --git a/src/host_id.h b/src/host_id.h
index 69cb43f..3a289c2 100644
--- a/src/host_id.h
+++ b/src/host_id.h
@@ -13,6 +13,9 @@ int print_space_state(struct space *sp, char *str);
int _get_space_info(char *space_name, struct space *sp_out);
int get_space_info(char *space_name, struct space *sp_out);
int host_id_disk_info(char *name, struct sync_disk *disk);
+int host_info_set_bit(char *space_name, uint64_t host_id);
+int host_info_clear_bit(char *space_name, uint64_t host_id);
+int test_id_bit(int host_id, char *bitmap);
int check_our_lease(struct task *task, struct space *sp, int *check_all, char
*check_buf);
void check_other_leases(struct task *task, struct space *sp, char *buf);
int add_lockspace(struct sanlk_lockspace *ls);
diff --git a/src/leader.h b/src/leader.h
index e808283..48189bc 100644
--- a/src/leader.h
+++ b/src/leader.h
@@ -68,6 +68,7 @@ struct leader_record {
#define LEADER_RECORD_MAX 256
#define HOSTID_BITMAP_OFFSET 256
+#define HOSTID_BITMAP_SIZE 256
#define REQ_DISK_MAGIC 0x08292011
#define REQ_DISK_VERSION_MAJOR 0x00010000
diff --git a/src/main.c b/src/main.c
index a4ba642..8b354b2 100644
--- a/src/main.c
+++ b/src/main.c
@@ -1425,15 +1425,11 @@ static void cmd_request(struct task *task, struct cmd_args *ca)
result = 0;
- /* zero lver and force mode clears the req, don't set bitmap */
-
if (!token->acquire_lver && !force_mode)
goto reply;
-#if 0
- if (owner_id)
- host_bitmap_set(token, owner_id);
-#endif
+ if (owner_id)
+ host_info_set_bit(token->r.lockspace_name, owner_id);
reply:
free(token);
log_debug("cmd_request %d,%d done %d", ca->ci_in, fd, result);
diff --git a/src/sanlock_internal.h b/src/sanlock_internal.h
index cce1947..4e69988 100644
--- a/src/sanlock_internal.h
+++ b/src/sanlock_internal.h
@@ -107,6 +107,7 @@ struct host_info {
uint64_t owner_id;
uint64_t owner_generation;
uint64_t timestamp; /* remote monotime */
+ uint64_t set_bit_time;
};
struct space {
commit aeedc5d7668710b019d703ccc85b8b82d53a168b
Author: David Teigland <teigland(a)redhat.com>
Date: Mon Aug 29 14:17:23 2011 -0500
sanlock: request struct magic and version numbers
standard fields for on disk struct
diff --git a/src/leader.h b/src/leader.h
index c62dced..e808283 100644
--- a/src/leader.h
+++ b/src/leader.h
@@ -69,7 +69,13 @@ struct leader_record {
#define LEADER_RECORD_MAX 256
#define HOSTID_BITMAP_OFFSET 256
+#define REQ_DISK_MAGIC 0x08292011
+#define REQ_DISK_VERSION_MAJOR 0x00010000
+#define REQ_DISK_VERSION_MINOR 0x00000001
+
struct request_record {
+ uint32_t magic;
+ uint32_t version;
uint64_t lver;
uint32_t force_mode;
};
diff --git a/src/paxos_lease.c b/src/paxos_lease.c
index 0834d95..b5a7282 100644
--- a/src/paxos_lease.c
+++ b/src/paxos_lease.c
@@ -1461,7 +1461,9 @@ int paxos_lease_init(struct task *task,
{
char *iobuf, **p_iobuf;
struct leader_record *leader;
+ struct request_record *rr;
int iobuf_len;
+ int sector_size;
int align_size;
int aio_timeout = 0;
int rv, d;
@@ -1471,11 +1473,13 @@ int paxos_lease_init(struct task *task,
if (!max_hosts)
max_hosts = DEFAULT_MAX_HOSTS;
+ sector_size = token->disks[0].sector_size;
+
align_size = direct_align(&token->disks[0]);
if (align_size < 0)
return align_size;
- if (token->disks[0].sector_size * (2 + max_hosts) > align_size)
+ if (sector_size * (2 + max_hosts) > align_size)
return -E2BIG;
iobuf_len = align_size;
@@ -1491,7 +1495,7 @@ int paxos_lease_init(struct task *task,
leader = (struct leader_record *)iobuf;
leader->magic = PAXOS_DISK_MAGIC;
leader->version = PAXOS_DISK_VERSION_MAJOR | PAXOS_DISK_VERSION_MINOR;
- leader->sector_size = token->disks[0].sector_size;
+ leader->sector_size = sector_size;
leader->num_hosts = num_hosts;
leader->max_hosts = max_hosts;
leader->timestamp = LEASE_FREE;
@@ -1499,6 +1503,10 @@ int paxos_lease_init(struct task *task,
strncpy(leader->resource_name, token->r.name, NAME_ID_SIZE);
leader->checksum = leader_checksum(leader);
+ rr = (struct request_record *)(iobuf + sector_size);
+ rr->magic = REQ_DISK_MAGIC;
+ rr->version = REQ_DISK_VERSION_MAJOR | REQ_DISK_VERSION_MINOR;
+
for (d = 0; d < token->r.num_disks; d++) {
rv = write_iobuf(token->disks[d].fd, token->disks[d].offset,
iobuf, iobuf_len, task);
diff --git a/src/sanlock_rv.h b/src/sanlock_rv.h
index 8edd492..3fd53c8 100644
--- a/src/sanlock_rv.h
+++ b/src/sanlock_rv.h
@@ -48,7 +48,6 @@
#define SANLK_RELEASE_LVER -250
#define SANLK_RELEASE_OWNER -251
-#define SANLK_REQUEST_LVER -252
/* delta_lease_renew, delta_lease_acquire */
@@ -56,4 +55,11 @@
#define SANLK_RENEW_DIFF -261
#define SANLK_HOSTID_BUSY -262
+/* request_token */
+
+#define SANLK_REQUEST_MAGIC -270
+#define SANLK_REQUEST_VERSION -271
+#define SANLK_REQUEST_OLD -272
+#define SANLK_REQUEST_LVER -273
+
#endif
diff --git a/src/token_manager.c b/src/token_manager.c
index b75b974..b786e12 100644
--- a/src/token_manager.c
+++ b/src/token_manager.c
@@ -201,7 +201,7 @@ int request_token(struct task *task, struct token *token, uint32_t
force_mode,
}
if (!token->acquire_lver && !force_mode)
- goto do_req;
+ goto req_read;
rv = paxos_lease_leader_read(task, token, &leader, "request");
if (rv < 0)
@@ -216,20 +216,38 @@ int request_token(struct task *task, struct token *token, uint32_t
force_mode,
*owner_id = leader.owner_id;
if (leader.lver >= token->acquire_lver) {
- rv = SANLK_ACQUIRE_LVER;
+ rv = SANLK_REQUEST_OLD;
goto out;
}
+ req_read:
rv = paxos_lease_request_read(task, token, &req);
if (rv < 0)
goto out;
+ if (req.magic != REQ_DISK_MAGIC) {
+ rv = SANLK_REQUEST_MAGIC;
+ goto out;
+ }
+
+ if ((req.version & 0xFFFF0000) != REQ_DISK_VERSION_MAJOR) {
+ rv = SANLK_REQUEST_VERSION;
+ goto out;
+ }
+
+ if (!token->acquire_lver && !force_mode)
+ goto req_write;
+
+ /* > instead of >= so multiple hosts can request the same
+ version at once and all succeed */
+
if (req.lver > token->acquire_lver) {
rv = SANLK_REQUEST_LVER;
goto out;
}
- do_req:
+req_write:
+ req.version = REQ_DISK_VERSION_MAJOR | REQ_DISK_VERSION_MINOR;
req.lver = token->acquire_lver;
req.force_mode = force_mode;