This is an automated email from the git hooks/post-receive script.

teigland pushed a commit to branch master in repository sanlock.

commit bcd94c521c40676a31175998f5697f2a8d76087d Author: David Teigland teigland@redhat.com AuthorDate: Mon Aug 12 16:38:33 2024 -0500

sanlock: use hugepages to avoid splitting read io

Use 2MB transparent hugepages via madvise(MADV_HUGEPAGE) to reduce or eliminate splitting read io's for lease areas. This requires allocating 2MB of memory for 1MB lease areas, leading to some minimal wasted allocation. --- src/cmd.c | 2 + src/delta_lease.c | 50 +++++++++++++++++----- src/diskio.c | 14 +++--- src/lockspace.c | 3 -- src/main.c | 8 ++++ src/paxos_lease.c | 114 ++++++++++++++++++++++++++++++++++++------------- src/sanlock.8 | 11 +++++ src/sanlock.conf | 3 ++ src/sanlock_internal.h | 5 +++ 9 files changed, 160 insertions(+), 50 deletions(-)

diff --git a/src/cmd.c b/src/cmd.c index 561e8d5..01ebb9d 100644 --- a/src/cmd.c +++ b/src/cmd.c @@ -2302,6 +2302,7 @@ static int print_state_daemon(char *str) "max_sectors_kb_align=%d " "max_sectors_kb_num=%d " "max_worker_threads=%d " + "use_hugepages=%s " "write_init_io_timeout=%u " "use_aio=%d " "io_timeout=%d " @@ -2333,6 +2334,7 @@ static int print_state_daemon(char *str) com.max_sectors_kb_align, com.max_sectors_kb_num, com.max_worker_threads, + com.use_hugepages ? "all" : "none", com.write_init_io_timeout, main_task.use_aio, com.io_timeout, diff --git a/src/delta_lease.c b/src/delta_lease.c index 55a5bba..fb4a640 100644 --- a/src/delta_lease.c +++ b/src/delta_lease.c @@ -21,6 +21,7 @@ #include <syslog.h> #include <sys/types.h> #include <sys/time.h> +#include <sys/mman.h>

#include "sanlock_internal.h" #include "sanlock.h" @@ -575,7 +576,11 @@ int delta_lease_renew(struct task *task, uint32_t checksum; uint32_t reap_timeout_msec; uint64_t host_id, id_offset, new_ts, now; - int rv, iobuf_len, sector_size; + int sector_size = sp->sector_size; + int align_size = sp->align_size; /* 1,2,4,8 MB depending on sector_size */ + int iobuf_len = sp->align_size; /* may increase (from 1MB) for hugepage */ + int read_len = sp->align_size; /* not increased (from 1MB) for hugepage */ + int rv;

if (!leader_last) { log_erros(sp, "delta_renew no leader_last"); @@ -589,15 +594,11 @@ int delta_lease_renew(struct task *task,

host_id = leader_last->owner_id;

- iobuf_len = sp->align_size; - - sector_size = sp->sector_size; - /* offset of our leader_record */ id_offset = (host_id - 1) * sector_size; - if (id_offset > iobuf_len) { - log_erros(sp, "delta_renew bad offset %llu iobuf_len %d", - (unsigned long long)id_offset, iobuf_len); + if (id_offset > align_size) { + log_erros(sp, "delta_renew bad offset %llu align_size %d", + (unsigned long long)id_offset, align_size); return -EINVAL; }

@@ -632,7 +633,7 @@ int delta_lease_renew(struct task *task, clock_gettime(CLOCK_MONOTONIC_RAW, &begin);

rv = read_iobuf_reap(disk->fd, disk->offset, - task->iobuf, iobuf_len, task, reap_timeout_msec); + task->iobuf, read_len, task, reap_timeout_msec);

log_space(sp, "delta_renew reap %d", rv);

@@ -676,23 +677,50 @@ int delta_lease_renew(struct task *task, */

if (!task->iobuf) { + int read_mem_align = getpagesize(); + int is_huge = 0; + /* this will happen the first time renew is called, and after a timed out renewal read fails to be reaped (see task->iobuf = NULL above) */

p_iobuf = &task->iobuf;

- rv = posix_memalign((void *)p_iobuf, getpagesize(), iobuf_len); + /* + * Minimum hugepage is 2MB, so for 1MB lease area we increase + * the iobuf size and alignment to 2MB, wasting a MB of space + * to more likely avoid splitting the i/o (read_len remains + * 1MB even though iobuf_len is extended to 2MB.) + * iobuf_len begins being set to align_size (1MB, 2MB, 4MB, 8MB). + */ + if (com.use_hugepages && ((iobuf_len == ONE_MB_IN_BYTES) || !(iobuf_len % TWO_MB_IN_BYTES))) { + read_mem_align = TWO_MB_IN_BYTES; + if (iobuf_len < TWO_MB_IN_BYTES) + iobuf_len = TWO_MB_IN_BYTES; + is_huge = 1; + } + + rv = posix_memalign((void *)p_iobuf, read_mem_align, iobuf_len); if (rv) { log_erros(sp, "dela_renew memalign rv %d", rv); rv = -ENOMEM; } + + if (is_huge) { + char *iobuf = task->iobuf; + char *bp; + + madvise(iobuf, iobuf_len, MADV_HUGEPAGE); + /* allocate the pages */ + for (bp = iobuf; bp < iobuf+iobuf_len; bp += TWO_MB_IN_BYTES) + memset(bp, 0, 1); + } }

if (log_renewal_level != -1) log_level(sp->space_id, 0, NULL, log_renewal_level, "delta_renew begin read");

- rv = read_iobuf(disk->fd, disk->offset, task->iobuf, iobuf_len, task, sp->io_timeout, rd_ms); + rv = read_iobuf(disk->fd, disk->offset, task->iobuf, read_len, task, sp->io_timeout, rd_ms); if (rv) { /* the next time delta_lease_renew() is called, prev_result will be this rv. If this rv is SANLK_AIO_TIMEOUT, we'll diff --git a/src/diskio.c b/src/diskio.c index 697bde7..1d84a6c 100644 --- a/src/diskio.c +++ b/src/diskio.c @@ -914,13 +914,13 @@ int write_sectors(const struct sync_disk *disk, int sector_size, uint64_t sector

/* read aligned io buffer */

-int read_iobuf(int fd, uint64_t offset, char *iobuf, int iobuf_len, +int read_iobuf(int fd, uint64_t offset, char *iobuf, int len, struct task *task, int ioto, int *rd_ms) { if (task && task->use_aio) - return do_read_aio_linux(fd, offset, iobuf, iobuf_len, task, ioto, rd_ms); + return do_read_aio_linux(fd, offset, iobuf, len, task, ioto, rd_ms); else - return do_read(fd, offset, iobuf, iobuf_len, task, rd_ms); + return do_read(fd, offset, iobuf, len, task, rd_ms); }

/* read sector_count sectors starting with sector_nr, where sector_nr @@ -977,7 +977,7 @@ int read_sectors(const struct sync_disk *disk, int sector_size, uint64_t sector_ The aicb used in a task's last timed out read_iobuf is task->read_iobuf_timeout_aicb . */

-int read_iobuf_reap(int fd, uint64_t offset, char *iobuf, int iobuf_len, +int read_iobuf_reap(int fd, uint64_t offset, char *iobuf, int read_len, struct task *task, uint32_t ioto_msec) { struct timespec ts; @@ -995,7 +995,7 @@ int read_iobuf_reap(int fd, uint64_t offset, char *iobuf, int iobuf_len, return -EINVAL; if (iocb->u.c.buf != iobuf) return -EINVAL; - if (iocb->u.c.nbytes != iobuf_len) + if (iocb->u.c.nbytes != read_len) return -EINVAL; if (iocb->u.c.offset != offset) return -EINVAL; @@ -1044,9 +1044,9 @@ int read_iobuf_reap(int fd, uint64_t offset, char *iobuf, int iobuf_len, rv = event.res; goto out; } - if (event.res != iobuf_len) { + if (event.res != read_len) { log_taskw(task, "aio collect %s %p:%p:%p result %ld:%ld match len %d r", - op_str, ev_aicb, ev_iocb, ev_aicb->buf, event.res, event.res2, iobuf_len); + op_str, ev_aicb, ev_iocb, ev_aicb->buf, event.res, event.res2, read_len); rv = -EMSGSIZE; goto out; } diff --git a/src/lockspace.c b/src/lockspace.c index 4d4240d..a2bee3f 100644 --- a/src/lockspace.c +++ b/src/lockspace.c @@ -752,9 +752,6 @@ static void save_renewal_history(struct space *sp, int delta_result, } }

-#define ONE_MB_IN_BYTES 1048576 -#define ONE_MB_IN_KB 1024 - static void set_lockspace_max_sectors_kb(struct space *sp, int sector_size, int align_size) { struct stat st; diff --git a/src/main.c b/src/main.c index f9ebe96..e8bf7c4 100644 --- a/src/main.c +++ b/src/main.c @@ -3069,6 +3069,13 @@ static void read_config_file(void) val = DEFAULT_MIN_WORKER_THREADS; com.max_worker_threads = val;

+ } else if (!strcmp(str, "use_hugepages")) { + memset(str, 0, sizeof(str)); + get_val_str(line, str); + if (!strcmp(str, "none")) + com.use_hugepages = 0; + else if (!strcmp(str, "all")) + com.use_hugepages = 1; } }

@@ -4096,6 +4103,7 @@ int main(int argc, char *argv[]) com.max_sectors_kb_ignore = DEFAULT_MAX_SECTORS_KB_IGNORE; com.max_sectors_kb_align = DEFAULT_MAX_SECTORS_KB_ALIGN; com.max_sectors_kb_num = DEFAULT_MAX_SECTORS_KB_NUM; + com.use_hugepages = 1; com.debug_cmds = ~0LL; com.debug_hosts = 1;

diff --git a/src/paxos_lease.c b/src/paxos_lease.c index 3f11b30..e62a93f 100644 --- a/src/paxos_lease.c +++ b/src/paxos_lease.c @@ -20,6 +20,7 @@ #include <syslog.h> #include <sys/types.h> #include <sys/time.h> +#include <sys/mman.h>

#include "sanlock_internal.h" #include "diskio.h" @@ -80,6 +81,7 @@ static void bk_debug_append(char *bk_debug, char *bk_str) } }

+/* static uint32_t roundup_power_of_two(uint32_t val) { val--; @@ -91,6 +93,7 @@ static uint32_t roundup_power_of_two(uint32_t val) val++; return val; } +*/

uint32_t leader_checksum(struct leader_record *lr) { @@ -492,32 +495,54 @@ static int run_ballot(struct task *task, struct token *token, uint32_t flags, struct paxos_dblock *bk_end; struct paxos_dblock *bk; struct sync_disk *disk; - char *iobuf[SANLK_MAX_DISKS]; + char *iobuf_array[SANLK_MAX_DISKS]; char **p_iobuf[SANLK_MAX_DISKS]; + char *iobuf; uint32_t checksum; int num_disks = token->r.num_disks; int num_writes, num_reads; - int sector_size = token->sector_size; - int sector_count; + int sector_size; + int read_len; int iobuf_len; + int read_mem_align; + int is_huge = 0; int phase2 = 0; int d, q, rv = 0; int q_max = -1; int error;

- sector_count = roundup_power_of_two(num_hosts + 2); + sector_size = token->sector_size; + /* token->align_size: 1,2,4,8 MB depending on sector_size */ + read_len = token->align_size; /* not increased (from 1MB) for hugepage */ + iobuf_len = token->align_size; /* can increase (from 1MB) for hugepage */ + read_mem_align = getpagesize(); /* increases to 2MB for hugepage */

- iobuf_len = sector_count * sector_size; + /* TODO: for hugepages, the task should keep a huge iobuf around to reuse + rather than alloc+madvise+free each time; after a while of heavy use, + madvise may be less likely to successfully turn the iobuf into a hugepage. */

- if (!iobuf_len) - return -EINVAL; + if (com.use_hugepages && ((iobuf_len == ONE_MB_IN_BYTES) || !(iobuf_len % TWO_MB_IN_BYTES))) { + read_mem_align = TWO_MB_IN_BYTES; + if (iobuf_len < TWO_MB_IN_BYTES) + iobuf_len = TWO_MB_IN_BYTES; + is_huge = 1; + }

for (d = 0; d < num_disks; d++) { - p_iobuf[d] = &iobuf[d]; + p_iobuf[d] = &iobuf_array[d];

- rv = posix_memalign((void *)p_iobuf[d], getpagesize(), iobuf_len); + rv = posix_memalign((void *)p_iobuf[d], read_mem_align, iobuf_len); if (rv) return rv; + + if (is_huge) { + char *bp; + iobuf = iobuf_array[d]; + madvise(iobuf, iobuf_len, MADV_HUGEPAGE); + /* allocate the pages */ + for (bp = iobuf; bp < iobuf+iobuf_len; bp += TWO_MB_IN_BYTES) + memset(bp, 0, 1); + } }

/* @@ -567,20 +592,23 @@ static int run_ballot(struct task *task, struct token *token, uint32_t flags, for (d = 0; d < num_disks; d++) { disk = &token->disks[d];

- if (!iobuf[d]) + if (!iobuf_array[d]) continue; - memset(iobuf[d], 0, iobuf_len); + + iobuf = iobuf_array[d]; + + memset(iobuf, 0, read_len);

/* acquire io: read 2 */ - rv = read_iobuf(disk->fd, disk->offset, iobuf[d], iobuf_len, task, token->io_timeout, NULL); + rv = read_iobuf(disk->fd, disk->offset, iobuf, read_len, task, token->io_timeout, NULL); if (rv == SANLK_AIO_TIMEOUT) - iobuf[d] = NULL; + iobuf_array[d] = NULL; if (rv < 0) continue; num_reads++;

for (q = 0; q < num_hosts; q++) { - bk_end = (struct paxos_dblock *)(iobuf[d] + ((2 + q)*sector_size)); + bk_end = (struct paxos_dblock *)(iobuf + ((2 + q)*sector_size));

checksum = dblock_checksum(bk_end);

@@ -782,20 +810,23 @@ static int run_ballot(struct task *task, struct token *token, uint32_t flags, for (d = 0; d < num_disks; d++) { disk = &token->disks[d];

- if (!iobuf[d]) + if (!iobuf_array[d]) continue; - memset(iobuf[d], 0, iobuf_len); + + iobuf = iobuf_array[d]; + + memset(iobuf, 0, read_len);

/* acquire io: read 3 */ - rv = read_iobuf(disk->fd, disk->offset, iobuf[d], iobuf_len, task, token->io_timeout, NULL); + rv = read_iobuf(disk->fd, disk->offset, iobuf, read_len, task, token->io_timeout, NULL); if (rv == SANLK_AIO_TIMEOUT) - iobuf[d] = NULL; + iobuf_array[d] = NULL; if (rv < 0) continue; num_reads++;

for (q = 0; q < num_hosts; q++) { - bk_end = (struct paxos_dblock *)(iobuf[d] + ((2 + q)*sector_size)); + bk_end = (struct paxos_dblock *)(iobuf + ((2 + q)*sector_size));

checksum = dblock_checksum(bk_end);

@@ -910,9 +941,11 @@ static int run_ballot(struct task *task, struct token *token, uint32_t flags, out: for (d = 0; d < num_disks; d++) { /* don't free iobufs that have timed out */ - if (!iobuf[d]) + if (!iobuf_array[d]) continue; - free(iobuf[d]); + + /* TODO: the task should keep huge iobuf and reuse it. */ + free(iobuf_array[d]); }

if (phase2 && (error < 0) && @@ -1375,25 +1408,48 @@ static int _lease_read_one(struct task *task, struct paxos_dblock bk; char *iobuf, **p_iobuf; uint32_t host_id = token->host_id; - uint32_t sector_size = token->sector_size; uint32_t checksum; struct paxos_dblock *bk_end; uint64_t tmp_mbal = 0; - int q, tmp_q = -1, rv, iobuf_len; + int sector_size; + int read_len; + int iobuf_len; + int read_mem_align; + int is_huge = 0; + int q, tmp_q = -1, rv;

- iobuf_len = token->align_size; - if (iobuf_len < 0) - return iobuf_len; + /* FIXME: what case is this? */ + if (token->align_size < 0) + return token->align_size; + + sector_size = token->sector_size; + /* token->align_size: 1,2,4,8 MB depending on sector_size */ + read_len = token->align_size; /* not increased (from 1MB) for hugepage */ + iobuf_len = token->align_size; /* can increase (from 1MB) for hugepage */ + read_mem_align = getpagesize(); /* increases to 2MB for hugepage */ + + if (com.use_hugepages && ((iobuf_len == ONE_MB_IN_BYTES) || !(iobuf_len % TWO_MB_IN_BYTES))) { + read_mem_align = TWO_MB_IN_BYTES; + if (iobuf_len < TWO_MB_IN_BYTES) + iobuf_len = TWO_MB_IN_BYTES; + is_huge = 1; + }

p_iobuf = &iobuf;

- rv = posix_memalign((void *)p_iobuf, getpagesize(), iobuf_len); + rv = posix_memalign((void *)p_iobuf, read_mem_align, iobuf_len); if (rv) return rv;

- memset(iobuf, 0, iobuf_len); + if (is_huge) { + char *bp; + madvise(iobuf, iobuf_len, MADV_HUGEPAGE); + /* allocate the pages */ + for (bp = iobuf; bp < iobuf+iobuf_len; bp += TWO_MB_IN_BYTES) + memset(bp, 0, 1); + }

- rv = read_iobuf(disk->fd, disk->offset, iobuf, iobuf_len, task, token->io_timeout, NULL); + rv = read_iobuf(disk->fd, disk->offset, iobuf, read_len, task, token->io_timeout, NULL); if (rv < 0) goto out;

diff --git a/src/sanlock.8 b/src/sanlock.8 index 75abcf5..f91933b 100644 --- a/src/sanlock.8 +++ b/src/sanlock.8 @@ -1277,6 +1277,9 @@ paxos algorithm to acquire the paxos_lease, and set a new owner.

/etc/sanlock/sanlock.conf

+The current settings in use by the sanlock daemon can be seen +in the output of 'sanlock status -D'. + .IP [bu] 2 quiet_fail = 1 .br @@ -1445,6 +1448,14 @@ Not doing so will invalidate the lease protection provided by sanlock. The io_timeout should usually be tuned along with this value, e.g. watchdog_fire_timeout = 30 with io_timeout = 5.

+.IP [bu] 2 +use_hugepages = <str> +.br +Set to "all" to use transparent hugepages (2MB via MADV_HUGEPAGE.) +This should minimize, or prevent, splitting read io's on lease areas. +2MB is allocated for 1MB lease areas, causing some extra memory usage. +Set to "none" to disable. + .SH SEE ALSO .BR wdmd (8)

diff --git a/src/sanlock.conf b/src/sanlock.conf index d55c280..6fae7dd 100644 --- a/src/sanlock.conf +++ b/src/sanlock.conf @@ -78,3 +78,6 @@ # # debug_hosts = 1 # command line: n/a +# +# use_hugepages = all +# command line: n/a diff --git a/src/sanlock_internal.h b/src/sanlock_internal.h index cb41538..7725eb6 100644 --- a/src/sanlock_internal.h +++ b/src/sanlock_internal.h @@ -35,6 +35,10 @@

#include <libaio.h>

+#define ONE_MB_IN_BYTES 1048576 +#define TWO_MB_IN_BYTES 2097152 +#define ONE_MB_IN_KB 1024 + /* default max number of hosts supported */

#define DEFAULT_MAX_HOSTS 2000 @@ -377,6 +381,7 @@ struct command_line { int clear_arg; int sector_size; int align_size; + int use_hugepages; char *uname; /* -U */ int uid; /* -U */ char *gname; /* -G */