tests/devcount.c | 154 +++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 128 insertions(+), 26 deletions(-)
New commits: commit b064a8e3b3f39524776caf67fa56675ddc7ad4ab Author: David Teigland teigland@redhat.com Date: Thu Apr 21 16:48:37 2011 -0500
devcount: new pause/resume
a last write from the rw process could hit disk after kill(SIGSTOP) returns in the parent. Use a new method to pause the rw child process which should guarantee no last write after the parent completes the pause.
diff --git a/tests/devcount.c b/tests/devcount.c index e2dfc92..4ad389f 100644 --- a/tests/devcount.c +++ b/tests/devcount.c @@ -2,6 +2,7 @@ #include <sys/wait.h> #include <sys/un.h> #include <sys/mount.h> +#include <sys/signalfd.h> #include <inttypes.h> #include <unistd.h> #include <stdio.h> @@ -55,12 +56,45 @@ do { \ } while (0)
+static int kill_pid(int pid) +{ + int rv, status; + + kill(pid, SIGKILL); + + while (1) { + rv = waitpid(pid, &status, 0); + if (rv < 0) + return -1; + if (rv != pid) + return -2; + + if (WIFEXITED(status)) + return 0; + } +} + /* kill(pid, SIGSTOP) would be nice, but that won't guarantee - the pid has finished all i/o when it returns */ + the pid has finished all i/o when it returns. + + Instead, we send SIGUSR1, which child sees after it's done + with a (synchronous) write, and calls SIGSTOP on itself */
-static void pause_pid(int pid) +static void pause_pid(int pid, int child_stderr) { - kill(pid, SIGSTOP); + char buf[64]; + int rv; + + kill(pid, SIGUSR1); + + /* child prints "we_are_paused" to stderr before stopping */ + + memset(buf, 0, sizeof(buf)); + + rv = read(child_stderr, buf, sizeof(buf)); + + if (!strstr(buf, "we_are_paused")) + log_error("pause_pid %d buf %s", pid, buf); }
static void resume_pid(int pid) @@ -68,6 +102,40 @@ static void resume_pid(int pid) kill(pid, SIGCONT); }
+static int check_pause(int fd) +{ + struct signalfd_siginfo fdsi; + ssize_t rv; + + rv = read(fd, &fdsi, sizeof(struct signalfd_siginfo)); + if (rv != sizeof(struct signalfd_siginfo)) { + return 0; + } + if (fdsi.ssi_signo == SIGUSR1) { + return 1; + } + return 0; +} + +static int setup_pause(void) +{ + sigset_t mask; + int fd, rv; + + sigemptyset(&mask); + sigaddset(&mask, SIGUSR1); + + rv = sigprocmask(SIG_BLOCK, &mask, NULL); + if (rv < 0) + return rv; + + fd = signalfd(-1, &mask, SFD_NONBLOCK); + if (fd < 0) + return -errno; + + return fd; +} + static int rand_int(int a, int b) { return a + (int) (((float)(b - a + 1)) * random() / (RAND_MAX+1.0)); @@ -100,12 +168,14 @@ void print_entries(char *path, int pid, char *buf) } }
-void print_our_we(char *path, int pid, int writes, struct entry *our_we) +void print_our_we(char *path, int pid, int writes, struct entry *our_we, + const char *stage) { - log_debug("%s c %d w %d index %d turn %u time %llu %u:%llu:%llu " + log_debug("%s c %d %s w %d index %d turn %u time %llu %u:%llu:%llu " "last %u %llu %u:%llu:%llu", path, pid, + stage, writes, our_hostid - 1, our_we->turn, @@ -133,6 +203,7 @@ static int do_count(int argc, char *argv[]) char *rbuf, **p_rbuf, *wbuf, **p_wbuf, *vbuf, **p_vbuf; struct entry *re, *max_re, *our_we; int i, fd, rv, max_i; + int pause_fd; time_t start; uint32_t our_pid = getpid(); uint32_t max_turn; @@ -143,6 +214,8 @@ static int do_count(int argc, char *argv[]) if (argc < COUNT_ARGS) return -1;
+ pause_fd = setup_pause(); + strcpy(count_path, argv[2]); sec1 = atoi(argv[3]); sec2 = atoi(argv[4]); @@ -287,7 +360,7 @@ static int do_count(int argc, char *argv[]) } writes = 1;
- print_our_we(count_path, our_pid, writes, our_we); + print_our_we(count_path, our_pid, writes, our_we, "begin");
start = time(NULL);
@@ -306,9 +379,17 @@ static int do_count(int argc, char *argv[])
if (write_seconds && (our_we->time - start >= write_seconds)) break; + + if (check_pause(pause_fd)) { + print_our_we(count_path, our_pid, writes, our_we, "pause"); + fprintf(stderr, "we_are_paused\n"); + raise(SIGSTOP); + /* this shouldn't appear until parent does kill(SIGCONT) */ + print_our_we(count_path, our_pid, writes, our_we, "resume"); + } }
- print_our_we(count_path, our_pid, writes, our_we); + print_our_we(count_path, our_pid, writes, our_we, "end");
if (turn_file) { fprintf(turn_file, "turn %03u start %llu end %llu host %u pid %u\n", @@ -402,6 +483,7 @@ static int do_relock(int argc, char *argv[]) char *av[COUNT_ARGS+1]; struct sanlk_resource *res, *res_inq; int i, j, pid, rv, sock, len, status; + int c2p[2]; /* child to parent */ int res_count; uint32_t parent_pid = getpid(); uint64_t lver; @@ -444,6 +526,8 @@ static int do_relock(int argc, char *argv[]) av[j] = NULL;
while (1) { + pipe(c2p); + pid = fork(); if (!pid) { int child_pid = getpid(); @@ -470,6 +554,12 @@ static int do_relock(int argc, char *argv[]) log_debug("%s c %d sanlock_acquire done", count_path, child_pid);
+ /* make child's stderr go to parent c2p[0] */ + close(2); + dup(c2p[1]); + close(c2p[0]); + close(c2p[1]); + execv(av[0], av); perror("execv devcount problem"); exit(EXIT_FAILURE); @@ -488,10 +578,8 @@ static int do_relock(int argc, char *argv[]) /* we expect child to exit when it fails to acquire the lock because it's held by someone else, or rw run time is up */
- if (rv == pid) { - sleep(rand_int(0, 1)); - continue; - } + if (rv == pid) + goto dead_child;
rv = sanlock_inquire(-1, pid, 0, &res_count, &state); if (rv < 0) { @@ -514,7 +602,8 @@ static int do_relock(int argc, char *argv[]) free(res_inq); free(state);
- pause_pid(pid); + pause_pid(pid, c2p[0]); + log_debug("%s p %d paused c %d", count_path, parent_pid, pid);
rv = sanlock_release(-1, pid, SANLK_REL_ALL, 0, NULL); if (rv < 0) { @@ -551,8 +640,10 @@ static int do_relock(int argc, char *argv[]) count_path, parent_pid, pid, (unsigned long long)lver, rv);
kill_child: - kill(pid, SIGKILL); - waitpid(pid, &status, 0); + kill_pid(pid); + dead_child: + close(c2p[0]); + close(c2p[1]); sleep(rand_int(0, 1)); }
@@ -900,8 +991,9 @@ static int do_migrate(int argc, char *argv[]) { char *av[MIGRATE_ARGS+1]; struct sanlk_resource *res; - int i, j, pid, rv, sock, len, status, init; - int pfd[2]; + int i, j, pid, rv, sock, len, init; + int p2c[2]; /* parent to child */ + int c2p[2]; /* child to parent */ int res_count; uint32_t parent_pid = getpid(); uint64_t lver; @@ -945,7 +1037,9 @@ static int do_migrate(int argc, char *argv[]) av[j] = NULL;
while (1) { - pipe(pfd); + pipe(p2c); + pipe(c2p); + pid = fork(); if (!pid) { int child_pid = getpid(); @@ -960,12 +1054,18 @@ static int do_migrate(int argc, char *argv[])
log_debug("%s c %d wait", count_path, child_pid);
- read(pfd[0], &junk, 1); - close(pfd[0]); - close(pfd[1]); + read(p2c[0], &junk, 1); + close(p2c[0]); + close(p2c[1]);
log_debug("%s c %d begin", count_path, child_pid);
+ /* make child's stderr go to parent c2p[0] */ + close(2); + dup(c2p[1]); + close(c2p[0]); + close(c2p[1]); + execv(av[0], av); perror("execv devcount problem"); exit(EXIT_FAILURE); @@ -992,9 +1092,9 @@ static int do_migrate(int argc, char *argv[]) (unsigned long long)lver);
/* tell child to resume */ - write(pfd[1], "\n", 1); - close(pfd[0]); - close(pfd[1]); + write(p2c[1], "\n", 1); + close(p2c[0]); + close(p2c[1]);
/* let the child run for 10 seconds before stopping it; if the child exits before the 10 seconds, the sanlock_inquire @@ -1011,7 +1111,8 @@ static int do_migrate(int argc, char *argv[]) log_debug("%s p %d sanlock_inquire c %d done", count_path, parent_pid, pid);
- pause_pid(pid); + pause_pid(pid, c2p[0]); + log_debug("%s p %d paused c %d", count_path, parent_pid, pid);
rv = sanlock_release(-1, pid, SANLK_REL_ALL, 0, NULL); if (rv < 0) { @@ -1024,8 +1125,9 @@ static int do_migrate(int argc, char *argv[])
write_migrate_incoming(state); /* to dest */
- kill(pid, SIGKILL); - waitpid(pid, &status, 0); + kill_pid(pid); + close(c2p[0]); + close(c2p[1]); free(state); }
sanlock-devel@lists.fedorahosted.org