src/delta_lease.c | 14 +++
src/host_id.c | 181 ++++++++++++++++++++++++-------------------------
src/host_id.h | 2
src/main.c | 4 -
src/sanlock_internal.h | 14 +--
5 files changed, 107 insertions(+), 108 deletions(-)
New commits:
commit 8fc52ae57a456e158ccec97f85e3055b8650ab12
Author: David Teigland <teigland(a)redhat.com>
Date: Wed Apr 20 11:15:53 2011 -0500
sanlock: renewal changes
Remove the pthread cond var and cond_timedwait and
replace with a much simpler sleep. This seems to
eliminates the erratic renewal behavior, in addition
to making the code simpler.
In the renewal thread, only use the leader timestamp
when the renewal was successful.
Add/change debugging messages related to renewls.
Use local variables in renewal thread to guard against
corrupting the disk if a bug causes the sp struct be
corrupted.
diff --git a/src/delta_lease.c b/src/delta_lease.c
index 19a8b61..bf55469 100644
--- a/src/delta_lease.c
+++ b/src/delta_lease.c
@@ -281,7 +281,7 @@ int delta_lease_renew(struct timeout *ti,
uint64_t host_id,
struct leader_record *leader_ret)
{
- struct leader_record leader;
+ struct leader_record leader, leader_read;
uint64_t new_ts;
int error, delay;
@@ -316,13 +316,23 @@ int delta_lease_renew(struct timeout *ti,
/* log_space(sp, "delta_renew sleep 2d %d", delay); */
sleep(delay);
- error = delta_lease_leader_read(ti, disk, space_name, host_id, &leader,
+ error = delta_lease_leader_read(ti, disk, space_name, host_id, &leader_read,
"delta_renew_check");
if (error < 0)
return error;
+ /*
if ((leader.timestamp != new_ts) || (leader.owner_id != our_host_id))
return SANLK_BAD_LEADER;
+ */
+
+ if (memcmp(&leader, &leader_read, sizeof(struct leader_record))) {
+ log_erros(sp, "delta_renew %llu reread mismatch",
+ (unsigned long long)host_id);
+ log_leader_error(0, space_name, host_id, disk, &leader,
"delta_renew_write");
+ log_leader_error(0, space_name, host_id, disk, &leader_read,
"delta_renew_reread");
+ return SANLK_BAD_LEADER;
+ }
memcpy(leader_ret, &leader, sizeof(struct leader_record));
return SANLK_OK;
diff --git a/src/host_id.c b/src/host_id.c
index 928d57b..2ac381b 100644
--- a/src/host_id.c
+++ b/src/host_id.c
@@ -44,29 +44,19 @@ int print_space_state(struct space *sp, char *str)
"killing_pids=%d "
"acquire_last_result=%d "
"renewal_last_result=%d "
- "release_last_result=%d "
- "acquire_last_time=%llu "
- "acquire_good_time=%llu "
- "renewal_last_time=%llu "
- "renewal_good_time=%llu "
- "release_last_time=%llu "
- "release_good_time=%llu "
- "max_renewal_time=%llu "
- "max_renewal_interval=%d",
+ "acquire_last_attempt=%llu "
+ "acquire_last_success=%llu "
+ "renewal_last_attempt=%llu "
+ "renewal_last_success=%llu",
sp->space_id,
(unsigned long long)sp->host_generation,
sp->killing_pids,
sp->lease_status.acquire_last_result,
sp->lease_status.renewal_last_result,
- sp->lease_status.release_last_result,
- (unsigned long long)sp->lease_status.acquire_last_time,
- (unsigned long long)sp->lease_status.acquire_good_time,
- (unsigned long long)sp->lease_status.renewal_last_time,
- (unsigned long long)sp->lease_status.renewal_good_time,
- (unsigned long long)sp->lease_status.release_last_time,
- (unsigned long long)sp->lease_status.release_good_time,
- (unsigned long long)sp->lease_status.max_renewal_time,
- sp->lease_status.max_renewal_interval);
+ (unsigned long long)sp->lease_status.acquire_last_attempt,
+ (unsigned long long)sp->lease_status.acquire_last_success,
+ (unsigned long long)sp->lease_status.renewal_last_attempt,
+ (unsigned long long)sp->lease_status.renewal_last_success);
return strlen(str);
}
@@ -135,54 +125,65 @@ int host_id_leader_read(struct timeout *ti,
* check if our_host_id_thread has renewed within timeout
*/
-int host_id_renewed(struct space *sp)
+int host_id_check(struct space *sp)
{
- uint64_t good_time;
- int good_diff;
+ uint64_t last_success;
+ int gap;
pthread_mutex_lock(&sp->mutex);
- good_time = sp->lease_status.renewal_good_time;
+ last_success = sp->lease_status.renewal_last_success;
pthread_mutex_unlock(&sp->mutex);
- good_diff = time(NULL) - good_time;
+ gap = time(NULL) - last_success;
- if (good_diff >= to.host_id_renewal_fail_seconds) {
- log_erros(sp, "host_id_renewed failed %d", good_diff);
+ if (gap >= to.host_id_renewal_fail_seconds) {
+ log_erros(sp, "host_id_check failed %d", gap);
return 0;
}
- if (good_diff >= to.host_id_renewal_warn_seconds) {
- log_erros(sp, "host_id_renewed warning %d last good %llu",
- good_diff,
- (unsigned long long)good_time);
+ if (gap >= to.host_id_renewal_warn_seconds) {
+ log_erros(sp, "host_id_check warning %d last_success %llu",
+ gap, (unsigned long long)last_success);
}
+ /*
+ log_space(sp, "host_id_check good %d %llu",
+ gap, (unsigned long long)last_success);
+ */
+
return 1;
}
static void *host_id_thread(void *arg_in)
{
+ struct space *sp;
+ char space_name[NAME_ID_SIZE];
+ struct sync_disk host_id_disk;
struct leader_record leader;
- struct timespec renew_time;
- struct space *sp = (struct space *)arg_in;
- uint64_t our_host_id;
- uint64_t t;
- uint64_t good_time;
- int good_diff;
- int rv, stop, result, dl_result;
+ uint64_t host_id;
+ time_t last_attempt, last_success;
+ int rv, stop = 0, result, delta_result, delta_length, gap;
- our_host_id = sp->host_id;
+ sp = (struct space *)arg_in;
+ host_id = sp->host_id;
+ memcpy(&space_name, sp->space_name, NAME_ID_SIZE);
+ memcpy(&host_id_disk, &sp->host_id_disk, sizeof(host_id_disk));
- result = delta_lease_acquire(&to, sp, &sp->host_id_disk, sp->space_name,
- our_host_id, sp->host_id, &leader);
- dl_result = result;
- t = leader.timestamp;
+ last_attempt = time(NULL);
+
+ result = delta_lease_acquire(&to, sp, &host_id_disk, space_name,
+ host_id, host_id, &leader);
+ delta_result = result;
+ delta_length = time(NULL) - last_attempt;
+
+ if (result == SANLK_OK)
+ last_success = leader.timestamp;
/* we need to start the watchdog after we acquire the host_id but
before we allow any pid's to begin running */
if (result == SANLK_OK) {
- rv = create_watchdog_file(sp, t);
+ rv = create_watchdog_file(sp, last_success);
if (rv < 0) {
log_erros(sp, "create_watchdog failed %d", rv);
result = SANLK_ERROR;
@@ -191,92 +192,86 @@ static void *host_id_thread(void *arg_in)
pthread_mutex_lock(&sp->mutex);
sp->lease_status.acquire_last_result = result;
- sp->lease_status.acquire_last_time = t;
+ sp->lease_status.acquire_last_attempt = last_attempt;
if (result == SANLK_OK)
- sp->lease_status.acquire_good_time = t;
+ sp->lease_status.acquire_last_success = last_success;
sp->lease_status.renewal_last_result = result;
- sp->lease_status.renewal_last_time = t;
+ sp->lease_status.renewal_last_attempt = last_attempt;
if (result == SANLK_OK)
- sp->lease_status.renewal_good_time = t;
- pthread_cond_broadcast(&sp->cond);
+ sp->lease_status.renewal_last_success = last_success;
pthread_mutex_unlock(&sp->mutex);
if (result < 0) {
log_erros(sp, "host_id %llu acquire failed %d",
- (unsigned long long)sp->host_id, result);
+ (unsigned long long)host_id, result);
goto out;
}
log_erros(sp, "host_id %llu generation %llu acquire %llu",
- (unsigned long long)sp->host_id,
+ (unsigned long long)host_id,
(unsigned long long)leader.owner_generation,
- (unsigned long long)t);
+ (unsigned long long)leader.timestamp);
sp->host_generation = leader.owner_generation;
- good_time = t;
- good_diff = 0;
- renew_time.tv_sec = t;
-
while (1) {
+ if (stop)
+ break;
+
+ sleep(1);
+
pthread_mutex_lock(&sp->mutex);
- renew_time.tv_sec += to.host_id_renewal_seconds;
- rv = 0;
- while (!sp->thread_stop && rv == 0) {
- rv = pthread_cond_timedwait(&sp->cond,
- &sp->mutex,
- &renew_time);
- }
stop = sp->thread_stop;
pthread_mutex_unlock(&sp->mutex);
+
if (stop)
break;
- clock_gettime(CLOCK_REALTIME, &renew_time);
+ if (time(NULL) - last_success < to.host_id_renewal_seconds)
+ continue;
+
+ last_attempt = time(NULL);
+
+ result = delta_lease_renew(&to, sp, &host_id_disk, space_name,
+ host_id, host_id, &leader);
+ delta_result = result;
+ delta_length = time(NULL) - last_attempt;
- result = delta_lease_renew(&to, sp, &sp->host_id_disk,
- sp->space_name, our_host_id,
- sp->host_id, &leader);
- dl_result = result;
- t = leader.timestamp;
+ if (result == SANLK_OK)
+ last_success = leader.timestamp;
pthread_mutex_lock(&sp->mutex);
sp->lease_status.renewal_last_result = result;
- sp->lease_status.renewal_last_time = t;
+ sp->lease_status.renewal_last_attempt = last_attempt;
if (result == SANLK_OK) {
- sp->lease_status.renewal_good_time = t;
-
- good_diff = t - good_time;
- good_time = t;
-
- if (good_diff > sp->lease_status.max_renewal_interval) {
- sp->lease_status.max_renewal_interval = good_diff;
- sp->lease_status.max_renewal_time = t;
- }
+ gap = last_success - sp->lease_status.renewal_last_success;
+ sp->lease_status.renewal_last_success = last_success;
/*
- log_space(sp, "host_id %llu renewal %llu interval %d",
- (unsigned long long)sp->host_id,
- (unsigned long long)t, good_diff);
+ log_space(sp, "host_id %llu renewed %llu len %d interval %d",
+ (unsigned long long)host_id,
+ (unsigned long long)last_success,
+ delta_length, gap);
*/
if (!sp->thread_stop)
- update_watchdog_file(sp, t);
+ update_watchdog_file(sp, last_success);
} else {
- log_erros(sp, "host_id %llu renewal error %d last good %llu",
- (unsigned long long)sp->host_id, result,
- (unsigned long long)sp->lease_status.renewal_good_time);
+ log_erros(sp, "host_id %llu renewal error %d len %d last_success %llu",
+ (unsigned long long)host_id, result, delta_length,
+ (unsigned long long)sp->lease_status.renewal_last_success);
}
+ stop = sp->thread_stop;
pthread_mutex_unlock(&sp->mutex);
}
/* unlink called below to get it done ASAP */
close_watchdog_file(sp);
out:
- if (dl_result == SANLK_OK)
- delta_lease_release(&to, sp, &sp->host_id_disk, sp->space_name,
- sp->host_id, &leader, &leader);
+ if (delta_result == SANLK_OK)
+ delta_lease_release(&to, sp, &host_id_disk, space_name,
+ host_id, &leader, &leader);
return NULL;
}
@@ -334,12 +329,14 @@ int add_space(struct space *sp)
goto fail_close;
}
- pthread_mutex_lock(&sp->mutex);
- while (!sp->lease_status.acquire_last_result) {
- pthread_cond_wait(&sp->cond, &sp->mutex);
+ while (1) {
+ pthread_mutex_lock(&sp->mutex);
+ result = sp->lease_status.acquire_last_result;
+ pthread_mutex_unlock(&sp->mutex);
+ if (result)
+ break;
+ sleep(1);
}
- result = sp->lease_status.acquire_last_result;
- pthread_mutex_unlock(&sp->mutex);
if (result != SANLK_OK) {
/* the thread exits right away if acquire fails */
diff --git a/src/host_id.h b/src/host_id.h
index eec3780..ae230c8 100644
--- a/src/host_id.h
+++ b/src/host_id.h
@@ -13,7 +13,7 @@ int print_space_state(struct space *sp, char *str);
int _get_space_info(char *space_name, struct space *sp_out);
int get_space_info(char *space_name, struct space *sp_out);
int host_id_leader_read(struct timeout *ti, char *space_name, uint64_t host_id, struct
leader_record *leader_ret);
-int host_id_renewed(struct space *sp);
+int host_id_check(struct space *sp);
int add_space(struct space *sp);
int rem_space(char *name, struct sync_disk *disk, uint64_t host_id);
void clear_spaces(int wait);
diff --git a/src/main.c b/src/main.c
index a5d251b..5587957 100644
--- a/src/main.c
+++ b/src/main.c
@@ -579,7 +579,6 @@ static int main_loop(void)
pthread_mutex_lock(&sp->mutex);
sp->thread_stop = 1;
unlink_watchdog_file(sp);
- pthread_cond_broadcast(&sp->cond);
pthread_mutex_unlock(&sp->mutex);
list_move(&sp->list, &spaces_remove);
} else {
@@ -588,7 +587,7 @@ static int main_loop(void)
check_interval = RECOVERY_CHECK_INTERVAL;
} else {
if (external_shutdown || sp->external_remove ||
- !host_id_renewed(sp)) {
+ !host_id_check(sp)) {
log_space(sp, "set killing_pids");
sp->killing_pids = 1;
kill_pids(sp);
@@ -1345,7 +1344,6 @@ static void *cmd_add_lockspace_thread(void *args_in)
memcpy(&sp->host_id_disk, &lockspace.host_id_disk,
sizeof(struct sanlk_disk));
pthread_mutex_init(&sp->mutex, NULL);
- pthread_cond_init(&sp->cond, NULL);
pthread_mutex_lock(&spaces_mutex);
sp->space_id = space_id_counter++;
diff --git a/src/sanlock_internal.h b/src/sanlock_internal.h
index 60ffb2d..f0108ea 100644
--- a/src/sanlock_internal.h
+++ b/src/sanlock_internal.h
@@ -91,15 +91,10 @@ struct token {
struct lease_status {
int acquire_last_result;
int renewal_last_result;
- int release_last_result;
- int max_renewal_interval;
- uint64_t acquire_last_time;
- uint64_t acquire_good_time;
- uint64_t renewal_last_time;
- uint64_t renewal_good_time;
- uint64_t release_last_time;
- uint64_t release_good_time;
- uint64_t max_renewal_time;
+ uint64_t acquire_last_attempt;
+ uint64_t acquire_last_success;
+ uint64_t renewal_last_attempt;
+ uint64_t renewal_last_success;
};
struct space {
@@ -114,7 +109,6 @@ struct space {
int thread_stop;
pthread_t thread;
pthread_mutex_t mutex; /* protects lease_status, thread_stop */
- pthread_cond_t cond;
struct lease_status lease_status;
int wd_fd;
};