src/main.c wdmd/main.c
by David Teigland
src/main.c | 2 +-
wdmd/main.c | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
New commits:
commit 0b23224e2efc8cc680563e66a23cec65743aa87c
Author: David Teigland <teigland(a)redhat.com>
Date: Fri Aug 5 14:10:58 2011 -0500
release: sanlock 1.7
and wdmd 1.1
diff --git a/src/main.c b/src/main.c
index 7d29499..1ea5a36 100644
--- a/src/main.c
+++ b/src/main.c
@@ -2338,7 +2338,7 @@ static void parse_arg_timeout(char *optstr)
}
#endif
-#define RELEASE_VERSION "1.6"
+#define RELEASE_VERSION "1.7"
/*
* daemon: acquires leases for the local host_id, associates them with a local
diff --git a/wdmd/main.c b/wdmd/main.c
index 95dcd3b..db8c3bc 100644
--- a/wdmd/main.c
+++ b/wdmd/main.c
@@ -885,7 +885,7 @@ static void print_usage(void)
would be for wdmd to fail starting if it found a pid file left over from
its previous run. */
-#define RELEASE_VERSION "1.0"
+#define RELEASE_VERSION "1.1"
int main(int argc, char *argv[])
{
12 years, 8 months
src/main.c
by David Teigland
src/main.c | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
New commits:
commit 85709d277cefc0fc58b879d8a0c5be6d18cde249
Author: David Teigland <teigland(a)redhat.com>
Date: Fri Aug 5 13:56:08 2011 -0500
sanlock: print connections limit
in limits section of help output
diff --git a/src/main.c b/src/main.c
index 69525b7..7d29499 100644
--- a/src/main.c
+++ b/src/main.c
@@ -2438,10 +2438,12 @@ static void print_usage(void)
printf(" <lver> optional leader version\n");
printf("\n");
printf("Limits:\n");
- printf("offsets must be 1MB aligned for storage with 512 byte sectors (8MB for 4096)\n");
+ printf("offset alignment with 512 byte sectors: %d (1MB)\n", 1024 * 1024);
+ printf("offset alignment with 4096 byte sectors: %d (8MB)\n", 1024 * 1024 * 8);
printf("maximum name length for lockspaces and resources: %d\n", SANLK_NAME_LEN);
printf("maximum path length: %d\n", SANLK_PATH_LEN);
printf("maximum host_id: %d\n", DEFAULT_MAX_HOSTS);
+ printf("maximum client process connections: 1000\n"); /* NALLOC */
printf("\n");
}
12 years, 8 months
src/main.c src/sanlock.8
by David Teigland
src/main.c | 134 +++++++++++++++++-------------------------
src/sanlock.8 | 181 ++++++++++++++++++++++++++++++++--------------------------
2 files changed, 156 insertions(+), 159 deletions(-)
New commits:
commit 26ec45bd741c1d4ccdac3b697e6cb11546559a91
Author: David Teigland <teigland(a)redhat.com>
Date: Fri Aug 5 12:20:40 2011 -0500
sanlock: update man page and help text
diff --git a/src/main.c b/src/main.c
index 1dc268e..69525b7 100644
--- a/src/main.c
+++ b/src/main.c
@@ -2353,15 +2353,16 @@ static void parse_arg_timeout(char *optstr)
static void print_usage(void)
{
printf("Usage:\n");
- printf("sanlock <type> <action> [options]\n\n");
-
- printf("types:\n");
- printf(" version print version\n");
- printf(" help print usage\n");
- printf(" daemon start daemon\n");
- printf(" client send request to daemon (default type if none given)\n");
- printf(" direct access storage directly (no coordination with daemon)\n");
+ printf("sanlock <command> <action> ...\n\n");
+
+ printf("commands:\n");
+ printf(" daemon start daemon\n");
+ printf(" client send request to daemon (default type if none given)\n");
+ printf(" direct access storage directly (no coordination with daemon)\n");
+ printf(" help print this usage (defaults in parens)\n");
+ printf(" version print version\n");
printf("\n");
+#if 0
printf("client actions: ask daemon to:\n");
printf(" status send internal state\n");
printf(" log_dump send internal debug buffer\n");
@@ -2385,87 +2386,62 @@ static void print_usage(void)
printf(" read_id read a host_id lease, print the state\n");
printf(" live_id monitor a host_id lease for liveness\n");
printf("\n");
- printf("daemon\n");
- printf(" -D debug: no fork and print all logging to stderr\n");
- printf(" -R <num> debug renewal: log debug info about renewals\n");
- printf(" -Q <num> quiet error messages for common lock contention\n");
- printf(" -L <level> write logging at level and up to logfile (-1 none)\n");
- printf(" -S <level> write logging at level and up to syslog (-1 none)\n");
- printf(" -t <num> max worker threads (default %d)\n", DEFAULT_MAX_WORKER_THREADS);
- printf(" -w <num> use watchdog through wdmd (1 yes, 0 no, default %d)\n", DEFAULT_USE_WATCHDOG);
- printf(" -a <num> use async io (1 yes, 0 no, default %d)\n", DEFAULT_USE_AIO);
- printf(" -h <num> use high priority features (1 yes, 0 no, default %d)\n", DEFAULT_HIGH_PRIORITY);
- printf(" includes max realtime scheduling priority, mlockall\n");
- printf(" -o <num> io timeout in seconds (default %d)\n", DEFAULT_IO_TIMEOUT);
- printf("\n");
- printf("client status\n");
- printf(" -D debug: print extra internal state for debugging\n");
- printf("\n");
- printf("client log_dump\n");
- printf("\n");
- printf("client shutdown\n");
- printf("\n");
- printf("client add_lockspace -s LOCKSPACE\n");
- printf("\n");
- printf("client rem_lockspace -s LOCKSPACE\n");
- printf("\n");
- printf("client command -r RESOURCE -c <path> <args>\n");
- printf(" -n <num_hosts> change num_hosts in leases when acquired\n");
- printf(" -c <path> <args> run command with args, -c must be final option\n");
- printf("\n");
- printf("client acquire -p <pid> -r RESOURCE\n");
- printf(" -p <pid> process that lease should be added for\n");
- printf("\n");
- printf("client release -p <pid> -r RESOURCE\n");
- printf(" -p <pid> process whose lease should be released\n");
- printf("\n");
- printf("client inquire -p <pid>\n");
- printf(" -p <pid> process whose resource leases should be displayed\n");
- printf("\n");
-
- printf("direct init [-s LOCKSPACE] [-r RESOURCE]\n");
- printf(" -a <num> use async io (1 yes, 0 no)\n");
- printf(" -n <num_hosts> host_id's from 1 to num_hosts will be able to acquire\n");
- printf(" a resource lease.\n");
- printf(" (default %d)\n", DEFAULT_MAX_HOSTS);
- printf(" -m <max_hosts> disk space is initialized for this many hosts\n");
- printf(" (default %d)\n", DEFAULT_MAX_HOSTS);
- printf(" -s LOCKSPACE initialize host_id leases for a lockspace\n");
- printf(" (host_id in the LOCKSPACE arg is ignored for init)\n");
- printf(" -r RESOURCE initialize a resource lease\n");
- printf("\n");
- printf("direct dump <path>[:<offset>] [options]\n");
- printf(" -D debug: print extra info for debugging\n");
- printf(" -a <num> use async io (1 yes, 0 no)\n");
- printf("\n");
- printf("direct read_leader [-s LOCKSPACE] [-r RESOURCE]\n");
- printf(" -a <num> use async io (1 yes, 0 no)\n");
+#endif
+ printf("sanlock daemon [options]\n");
+ printf(" -D no fork and print all logging to stderr\n");
+ printf(" -Q 0|1 quiet error messages for common lock contention (0)\n");
+ printf(" -R 0|1 renewal debugging, log debug info about renewals (0)\n");
+ printf(" -L <pri> write logging at priority level and up to logfile (3 LOG_ERR))\n");
+ printf(" (use -1 for none)\n");
+ printf(" -S <pri> write logging at priority level and up to syslog (3 LOG_ERR)\n");
+ printf(" (use -1 for none)\n");
+ printf(" -U <uid> user id\n");
+ printf(" -G <gid> group id\n");
+ printf(" -t <num> max worker threads (%d)\n", DEFAULT_MAX_WORKER_THREADS);
+ printf(" -w 0|1 use watchdog through wdmd (%d)\n", DEFAULT_USE_WATCHDOG);
+ printf(" -h 0|1 use high priority features (%d)\n", DEFAULT_HIGH_PRIORITY);
+ printf(" (realtime scheduling, mlockall)\n");
+ printf(" -a 0|1 use async io (%d)\n", DEFAULT_USE_AIO);
+ printf(" -o 0|1 io timeout in seconds (%d)\n", DEFAULT_IO_TIMEOUT);
printf("\n");
- printf("direct acquire|release -i <num> -g <num> -r RESOURCE\n");
- printf(" -a <num> use async io (1 yes, 0 no)\n");
- printf(" -n <num_hosts> change num_hosts in leases when acquired\n");
- printf(" -i <num> host_id of local host\n");
- printf(" -g <num> host_id generation of local host\n");
+ printf("sanlock client <action> [options]\n");
+ printf("sanlock client status [-D]\n");
+ printf("sanlock client log_dump\n");
+ printf("sanlock client shutdown\n");
+ printf("sanlock client add_lockspace -s LOCKSPACE\n");
+ printf("sanlock client rem_lockspace -s LOCKSPACE\n");
+ printf("sanlock client command -r RESOURCE -c <path> <args>\n");
+ printf("sanlock client acquire -r RESOURCE -p <pid>\n");
+ printf("sanlock client release -r RESOURCE -p <pid>\n");
+ printf("sanlock client inquire -p <pid>\n");
printf("\n");
- printf("direct acquire_id|renew_id|release_id|read_id|live_id -s LOCKSPACE\n");
- printf(" -a <num> use async io (1 yes, 0 no)\n");
+ printf("sanlock direct <action> [-a 0|1] [-o 0|1]\n");
+ printf("sanlock direct init -s LOCKSPACE\n");
+ printf("sanlock direct init -s RESOURCE\n");
+ printf("sanlock direct read_leader -s LOCKSPACE\n");
+ printf("sanlock direct read_leader -r RESOURCE\n");
+ printf("sanlock direct read_id -s LOCKSPACE\n");
+ printf("sanlock direct live_id -s LOCKSPACE\n");
+ printf("sanlock direct dump <path>[:<offset>]\n");
printf("\n");
-
printf("LOCKSPACE = <lockspace_name>:<host_id>:<path>:<offset>\n");
printf(" <lockspace_name> name of lockspace\n");
printf(" <host_id> local host identifier in lockspace\n");
- printf(" <path> disk path where host_id leases are written\n");
- printf(" <offset> offset on disk, in bytes\n");
+ printf(" <path> disk to storage reserved for leases\n");
+ printf(" <offset> offset on path (bytes)\n");
printf("\n");
printf("RESOURCE = <lockspace_name>:<resource_name>:<path>:<offset>[:<lver>]\n");
printf(" <lockspace_name> name of lockspace\n");
- printf(" <resource_name> name of resource being leased\n");
- printf(" <path> disk path where resource leases are written\n");
- printf(" <offset> offset on disk in bytes\n");
- printf(" <lver> optional disk leader version of resource for acquire\n");
+ printf(" <resource_name> name of resource\n");
+ printf(" <path> disk to storage reserved for leases\n");
+ printf(" <offset> offset on path (bytes)\n");
+ printf(" <lver> optional leader version\n");
printf("\n");
- printf("<offset> must be 1MB aligned for disks with 512 byte sectors\n");
- printf(" and 8MB aligned for disks with 4096 byte sectors\n");
+ printf("Limits:\n");
+ printf("offsets must be 1MB aligned for storage with 512 byte sectors (8MB for 4096)\n");
+ printf("maximum name length for lockspaces and resources: %d\n", SANLK_NAME_LEN);
+ printf("maximum path length: %d\n", SANLK_PATH_LEN);
+ printf("maximum host_id: %d\n", DEFAULT_MAX_HOSTS);
printf("\n");
}
diff --git a/src/sanlock.8 b/src/sanlock.8
index 43caaed..6a9787a 100644
--- a/src/sanlock.8
+++ b/src/sanlock.8
@@ -1,4 +1,4 @@
-.TH SANLOCK 8 2011-08-02
+.TH SANLOCK 8 2011-08-05
.SH NAME
sanlock \- shared storage lock manager
@@ -14,14 +14,13 @@ hosts with shared storage. All lease management and coordination is done
through reading and writing blocks on the shared storage. Two types of
leases are used, each based on a different algorithm:
-"delta leases" are slow to acquire and demand regular i/o to shared
-storage. A delta lease exists in a single sector of shared storage.
-Acquiring a delta lease involves reads and writes to that sector separated
-by specific delays. Once acquired, the lease must be renewed by writing a
-new timestamp to the sector regularly. sanlock only uses delta leases
-internally to hold a lease on its host_id. The host_id leases prevent two
-hosts from using the same host_id and provide basic host liveness
-information based on the renewals.
+"delta leases" are slow to acquire and require regular i/o to shared
+storage. A delta lease exists in a single sector of storage. Acquiring a
+delta lease involves reads and writes to that sector separated by specific
+delays. Once acquired, a lease must be renewed by updating a timestamp in
+the sector regularly. sanlock uses a delta lease internally to hold a
+lease on a host_id. host_id leases prevent two hosts from using the same
+host_id and provide basic host liveness information based on the renewals.
"paxos leases" are generally fast to acquire and sanlock makes them
available to applications as general purpose resource leases. A paxos
@@ -29,13 +28,13 @@ lease exists in 1MB of shared storage (8MB for 4k sectors). Acquiring a
paxos lease involves reads and writes to max_hosts (2000) sectors in a
specific sequence specified by the Disk Paxos algorithm. paxos leases use
host_id's internally to indicate the owner of the lease, and the algorithm
-will not work if different hosts use the same host_id. So, the delta
-leases provide the unique host_id's used in paxos leases. paxos leases
-also use delta leases to check if a host_id is alive.
+fails if different hosts use the same host_id. So, delta leases provide
+the unique host_id's used in paxos leases. paxos leases also refer to
+delta leases to check if a host_id is alive.
Before sanlock can be used, the user must assign each host a host_id,
which is a number between 1 and 2000. Two hosts should not be given the
-same host_id.
+same host_id (even though delta leases attempt to detect this mistake.)
sanlock views a pool of storage as a "lockspace". Each distinct pool of
storage, e.g. from different sources, would typically be defined as a
@@ -45,71 +44,73 @@ Part of this storage space must be reserved and initialized for sanlock to
store delta leases. Each host that wants to use the lockspace must first
acquire a delta lease on its host_id number within the lockspace. (See
the add_lockspace action/api.) The space required for 2000 delta leases
-in the lockspace (for all possible hosts, max_hosts, which is 2000) is 1MB
-(8MB for 4k sectors). (This is the same size as that required for a
-single paxos lease.)
+in the lockspace (for 2000 possible host_id's) is 1MB (8MB for 4k
+sectors). (This is the same size required for a single paxos lease.)
-The following example illustrates these ideas.
+More storage space must be reserved and initialized for paxos leases,
+according to the needs of the applications using sanlock.
+
+The following steps illustrate these concepts using the command line.
+Applications may choose to do these same steps through libsanlock.
1. Create storage pools and reserve and initialize host_id leases
.br
two different LUNs on a SAN: /dev/sdb, /dev/sdc
.br
-vgcreate pool1 /dev/sdb
+# vgcreate pool1 /dev/sdb
.br
-vgcreate pool2 /dev/sdc
+# vgcreate pool2 /dev/sdc
.br
-lvcreate -n hostid_leases -L 1MB pool1
+# lvcreate -n hostid_leases -L 1MB pool1
.br
-lvcreate -n hostid_leases -L 1MB pool2
+# lvcreate -n hostid_leases -L 1MB pool2
.br
-sanlock direct init -s lockspace1:0:/dev/pool1/hostid_leases:0
+# sanlock direct init -s LS1:0:/dev/pool1/hostid_leases:0
.br
-sanlock direct init -s lockspace2:0:/dev/pool2/hostid_leases:0
+# sanlock direct init -s LS2:0:/dev/pool2/hostid_leases:0
.br
2. Start the sanlock daemon on each host
.br
-sanlock daemon
+# sanlock daemon
.br
3. Add each lockspace to be used
.br
host1:
.br
-sanlock client add_lockspace -s lockspace1:1:/dev/pool1/hostid_leases:0
+# sanlock client add_lockspace -s LS1:1:/dev/pool1/hostid_leases:0
.br
-sanlock client add_lockspace -s lockspace2:1:/dev/pool2/hostid_leases:0
+# sanlock client add_lockspace -s LS2:1:/dev/pool2/hostid_leases:0
.br
host2:
.br
-sanlock client add_lockspace -s lockspace1:2:/dev/pool1/hostid_leases:0
+# sanlock client add_lockspace -s LS1:2:/dev/pool1/hostid_leases:0
.br
-sanlock client add_lockspace -s lockspace2:2:/dev/pool2/hostid_leases:0
+# sanlock client add_lockspace -s LS2:2:/dev/pool2/hostid_leases:0
.br
4. Applications can now reserve/initialize space for resource leases, and
then acquire the leases as they need to access the resources.
The resource leases that are created and how they are used depends on the
-application. For example, say application A, running on multiple hosts,
+application. For example, say application A, running on host1 and host2,
needs to synchronize access to data it stores on /dev/pool1/Adata. A
could use a resource lease as follows:
5. Reserve and initialize a single resource lease for Adata
.br
-lvcreate -n Adata_lease -L 1MB pool1
+# lvcreate -n Adata_lease -L 1MB pool1
.br
-sanlock direct init -r lockspace1:Adata:/dev/pool1/Adata_lease:0
+# sanlock direct init -r LS1:Adata:/dev/pool1/Adata_lease:0
.br
6. Acquire the lease from the app using libsanlock (see sanlock_register,
-sanlock_acquire). If the app is already running as pid 1234, and has
+sanlock_acquire). If the app is already running as pid 123, and has
registered with the sanlock daemon, the lease can be added for it
manually.
.br
-sanlock client acquire
- -p 1234 -r lockspace1:Adata:/dev/pool1/Adata_lease:0
+# sanlock client acquire -r LS1:Adata:/dev/pool1/Adata_lease:0 -p 123
.br
.B offsets
@@ -121,16 +122,16 @@ offsets may be used to place leases on the same device rather than using
separate devices and offset 0 as shown in examples above, e.g. these
commands above:
.br
-sanlock direct init -s lockspace1:0:/dev/pool1/hostid_leases:0
+# sanlock direct init -s LS1:0:/dev/pool1/hostid_leases:0
.br
-sanlock direct init -r lockspace1:Adata:/dev/pool1/Adata_lease:0
+# sanlock direct init -r LS1:Adata:/dev/pool1/Adata_lease:0
.br
could be replaced by:
.br
.br
-sanlock direct init -s lockspace1:0:/dev/pool1/leases:0
+# sanlock direct init -s LS1:0:/dev/pool1/leases:0
.br
-sanlock direct init -r lockspace1:Adata:/dev/pool1/leases:1048576
+# sanlock direct init -r LS1:Adata:/dev/pool1/leases:1048576
.B failures
@@ -142,28 +143,42 @@ period of time (usually because storage access is lost), sanlock will kill
any process holding a resource lease within the lockspace.
If the sanlock daemon crashes or gets stuck, it will no longer renew the
-expiry time of the host_id's in the wdmd daemon, and the watchdog device
-will reset the host.
+expiry time of its per-host_id connections to the wdmd daemon, and the
+watchdog device will reset the host.
.B watchdog
-The watchdog device is used to further protect shared resources in case
-processes cannot be killed when a host_id fails to be renewed. The
-watchdog device will reset the host withing a known fixed time period from
-the last successful host_id renewal. This known time is the length of
-time that sanlock on another host must wait before granting the resource
-lease that was held by a process on the failed host.
-
-The sanlock daemon uses the wdmd daemon to multiplex petting the watchdog
-device for all host_id's that are held and being renewed. The watchdog is
-managed so it will reset the host if any host_id fails to be renewed for
-fixed period of time.
-
-.B levels of protection
+sanlock uses the
+.BR wdmd (8)
+daemon to access /dev/watchdog. A separate wdmd connection is maintained
+with wdmd for each host_id being renewed. Each host_id connection has an
+expiry time for some seconds in the future. After each successful host_id
+renewal, sanlock updates the associated expiry time in wdmd. If wdmd
+finds any connection expired, it will not pet /dev/watchdog. After enough
+successive expired/failed checks, the watchdog device will fire and reset
+the host.
+
+After a number of failed attempts to renew a host_id, sanlock kills any
+process using that lockspace. Once all those processes have exited,
+sanlock will unregister the associated wdmd connection. wdmd will no
+longer find the expired connection, and will resume petting /dev/watchdog
+(assuming it finds no other failed/expired tests.) If the killed
+processes did not exit quickly enough, the expired wdmd connection will
+not be unregistered, and /dev/watchdog will reset the host.
+
+Based on these known timeout values, sanlock on another host can
+calculate, based on the last host_id renewal, when the failed host will
+have been reset by its watchdog (or killed all the necessary processes).
+
+If the sanlock daemon itself fails, crashes, get stuck, it will no longer
+update the expiry time for its host_id connections to wdmd, which will
+also lead to the watchdog resetting the host.
+
+.B safety
sanlock leases are meant to guarantee that two process on two hosts are
never allowed to hold the same resource lease at once. If they were, the
-resource being protected could be corrupted. There are three levels of
+resource being protected may be corrupted. There are three levels of
protection built into sanlock itself:
1. The paxos leases and delta leases themselves.
@@ -175,7 +190,6 @@ leases in the lockspace.
3. If the pids do not exit after being killed, or if the sanlock daemon
fails, the watchdog device resets the host.
-
.SH OPTIONS
.P
@@ -192,38 +206,38 @@ COMMAND can be one of three primary top level choices
.BR -D " "
no fork and print all logging to stderr
-.BI -Q " num"
+.BR -Q " 0|1"
quiet error messages for common lock contention
-.BI -R " num"
+.BR -R " 0|1"
renewal debugging, log debug info for each renewal
-.BI -L " num"
-write logging at level and up to logfile (-1 none)
+.BI -L " pri"
+write logging at priority level and up to logfile (-1 none)
-.BI -S " num"
-write logging at level and up to syslog (-1 none)
+.BI -S " pri"
+write logging at priority level and up to syslog (-1 none)
-.BI -U " num"
-uid
+.BI -U " uid"
+user id
-.BI -G " num"
-gid
+.BI -G " gid"
+group id
.BI -t " num"
max worker threads
-.BI -w " num"
-use watchdog through wdmd (1 yes, 0 no, default 1)
+.BR -w " 0|1"
+use watchdog through wdmd
-.BI -h " num"
-use high priority features (1 yes, 0 no, default 1)
+.BR -h " 0|1"
+use high priority features (realtime scheduling, mlockall)
-.BI \-a " num"
-use async i/o (1 yes, 0 no, default 1)
+.BR \-a " 0|1"
+use async i/o
-.BI -o " num"
-io timeout in seconds (default 10)
+.BI -o " sec"
+io timeout in seconds
.B "sanlock client"
.I action
@@ -232,7 +246,7 @@ io timeout in seconds (default 10)
.B sanlock client status
Print processes, lockspaces, and resources being manged by the sanlock
-daemon.
+daemon. Add -D to show extra internal daemon status for debugging.
.B sanlock client log_dump
@@ -282,11 +296,11 @@ held.
.I action
[options]
-.BI \-a " num"
-use async i/o (1 yes, 0 no, default 1)
+.BR \-a " 0|1"
+use async i/o
-.BI -o " num"
-io timeout in seconds (default 10)
+.BI -o " sec"
+io timeout in seconds
.BR "sanlock direct init -s" " LOCKSPACE"
.br
@@ -341,13 +355,12 @@ sanlock daemon's conclusion unless the configured timeouts match.)
Read disk sectors and print leader records for delta or paxos leases.
-
.SS
LOCKSPACE option string
.BR \-s " " \fIlockspace_name\fP:\fIhost_id\fP:\fIpath\fP:\fIoffset\fP
.P
-.IR lockspace_name " name of the lockspace"
+.IR lockspace_name " name of lockspace"
.br
.IR host_id " local host identifier in lockspace"
.br
@@ -371,6 +384,14 @@ RESOURCE option string
.br
.IR lver " optional leader version"
+.SS Defaults
+
+.B sanlock help
+shows the default values for the options above.
+
+.B sanlock version
+shows the build version.
+
.SH SEE ALSO
.BR wdmd (8)
12 years, 8 months
src/sanlock.8
by David Teigland
src/sanlock.8 | 286 ++++++++++++++++++++++++++++++++++++++++++++--------------
1 file changed, 217 insertions(+), 69 deletions(-)
New commits:
commit 005583d6b7b5a7af8b9be30f3582337a9fee2203
Author: David Teigland <teigland(a)redhat.com>
Date: Thu Aug 4 17:33:38 2011 -0500
sanlock: man page update
diff --git a/src/sanlock.8 b/src/sanlock.8
index 7d7d2ed..43caaed 100644
--- a/src/sanlock.8
+++ b/src/sanlock.8
@@ -5,7 +5,7 @@ sanlock \- shared storage lock manager
.SH SYNOPSIS
.B sanlock
-[COMMAND] [ACTION] [OPTIONS]
+[COMMAND] [ACTION] ...
.SH DESCRIPTION
@@ -49,7 +49,7 @@ in the lockspace (for all possible hosts, max_hosts, which is 2000) is 1MB
(8MB for 4k sectors). (This is the same size as that required for a
single paxos lease.)
-.B example
+The following example illustrates these ideas.
1. Create storage pools and reserve and initialize host_id leases
.br
@@ -91,16 +91,16 @@ sanlock client add_lockspace -s lockspace2:2:/dev/pool2/hostid_leases:0
4. Applications can now reserve/initialize space for resource leases, and
then acquire the leases as they need to access the resources.
-What resource leases are created and how they are used depends on the
-application. For example, say application foo, running on multiple hosts,
-needs to synchronize access to data it stores on /dev/pool1/foo_data. foo
+The resource leases that are created and how they are used depends on the
+application. For example, say application A, running on multiple hosts,
+needs to synchronize access to data it stores on /dev/pool1/Adata. A
could use a resource lease as follows:
-5. Reserve and initialize a single resource lease for foo_data
+5. Reserve and initialize a single resource lease for Adata
.br
-lvcreate -n foo_data -L 1MB pool1
+lvcreate -n Adata_lease -L 1MB pool1
.br
-sanlock direct init -r lockspace1:foo_data:/dev/pool1/foo_data:0
+sanlock direct init -r lockspace1:Adata:/dev/pool1/Adata_lease:0
.br
6. Acquire the lease from the app using libsanlock (see sanlock_register,
@@ -109,21 +109,43 @@ registered with the sanlock daemon, the lease can be added for it
manually.
.br
sanlock client acquire
- -p 1234 -r lockspace1:foo_data:/dev/pool1/foo_data:0
+ -p 1234 -r lockspace1:Adata:/dev/pool1/Adata_lease:0
.br
-.B client failure
+.B offsets
+
+offsets must be 1MB aligned for disks with 512 byte sectors, and
+8MB aligned for disks with 4096 byte sectors.
+
+offsets may be used to place leases on the same device rather than using
+separate devices and offset 0 as shown in examples above, e.g. these
+commands above:
+.br
+sanlock direct init -s lockspace1:0:/dev/pool1/hostid_leases:0
+.br
+sanlock direct init -r lockspace1:Adata:/dev/pool1/Adata_lease:0
+.br
+could be replaced by:
+.br
+.br
+sanlock direct init -s lockspace1:0:/dev/pool1/leases:0
+.br
+sanlock direct init -r lockspace1:Adata:/dev/pool1/leases:1048576
+
+.B failures
If a process holding resource leases fails or exits without releasing its
leases, sanlock will release the leases for it automatically.
-.B lockspace failure
-
If the sanlock daemon cannot renew a lockspace host_id for a specific
period of time (usually because storage access is lost), sanlock will kill
any process holding a resource lease within the lockspace.
-.B watchdog usage
+If the sanlock daemon crashes or gets stuck, it will no longer renew the
+expiry time of the host_id's in the wdmd daemon, and the watchdog device
+will reset the host.
+
+.B watchdog
The watchdog device is used to further protect shared resources in case
processes cannot be killed when a host_id fails to be renewed. The
@@ -132,12 +154,17 @@ the last successful host_id renewal. This known time is the length of
time that sanlock on another host must wait before granting the resource
lease that was held by a process on the failed host.
+The sanlock daemon uses the wdmd daemon to multiplex petting the watchdog
+device for all host_id's that are held and being renewed. The watchdog is
+managed so it will reset the host if any host_id fails to be renewed for
+fixed period of time.
+
.B levels of protection
-sanlock leases are meant to guarantee that two different process on two
-different hosts are never allowed to hold the same resource lease at once.
-If they were, the resource being protected could be corrupted. There are
-three levels of protection built into sanlock itself:
+sanlock leases are meant to guarantee that two process on two hosts are
+never allowed to hold the same resource lease at once. If they were, the
+resource being protected could be corrupted. There are three levels of
+protection built into sanlock itself:
1. The paxos leases and delta leases themselves.
@@ -145,83 +172,204 @@ three levels of protection built into sanlock itself:
cannot be renewed), the sanlock daemon kills any pids using resource
leases in the lockspace.
-3. If the pids do not exit after being killed, the watchdog device resets
-the host.
+3. If the pids do not exit after being killed, or if the sanlock daemon
+fails, the watchdog device resets the host.
+
-.SH COMMANDS
+.SH OPTIONS
-sanlock is a multi purpose binary, with three top level commands:
+.P
+COMMAND can be one of three primary top level choices
+.P
+.BR "sanlock daemon" " start daemon"
+.br
+.BR "sanlock client" " send request to daemon (default command if none given)"
+.br
+.BR "sanlock direct" " access storage directly (no coordination with daemon)"
-.TP
-.B daemon
-start daemon
+.BR "sanlock daemon" " [options]"
-.TP
-.B client
-send request to daemon (default command if none given)
+.BR -D " "
+no fork and print all logging to stderr
-.TP
-.B direct
-access storage directly (no coordination with daemon)
+.BI -Q " num"
+quiet error messages for common lock contention
-.SH OPTIONS
+.BI -R " num"
+renewal debugging, log debug info for each renewal
-.TP
-.nf
-LOCKSPACE = <lockspace_name>:<host_id>:<path>:<offset>
-.fi
+.BI -L " num"
+write logging at level and up to logfile (-1 none)
-.TP
-.nf
-RESOURCE = <lockspace_name>:<resource_name>:<path>:<offset>[:<lver>]
-.fi
+.BI -S " num"
+write logging at level and up to syslog (-1 none)
-.SS client actions
+.BI -U " num"
+uid
-.TP
-.B status
+.BI -G " num"
+gid
-.TP
-.B log_dump
+.BI -t " num"
+max worker threads
-.TP
-.B shutdown
+.BI -w " num"
+use watchdog through wdmd (1 yes, 0 no, default 1)
-.TP
-.B client add_lockspace -s LOCKSPACE
+.BI -h " num"
+use high priority features (1 yes, 0 no, default 1)
-.TP
-.B client rem_lockspace -s LOCKSPACE
+.BI \-a " num"
+use async i/o (1 yes, 0 no, default 1)
-.TP
-.B client command -r RESOURCE -c <path> <args>
+.BI -o " num"
+io timeout in seconds (default 10)
-.TP
-.B client acquire -p <pid> -r RESOURCE
+.B "sanlock client"
+.I action
+[options]
-.TP
-.B client release -p <pid> -r RESOURCE
+.B sanlock client status
-.TP
-.B client inquire -p <pid>
+Print processes, lockspaces, and resources being manged by the sanlock
+daemon.
-.SS direct actions
+.B sanlock client log_dump
-.TP
-.B direct init [-s LOCKSPACE] [-r RESOURCE]
+Print the sanlock daemon internal debug log.
-.TP
-.B direct dump <path>[:<offset>] [options]
+.B sanlock client shutdown
-.TP
-.B direct read_leader [-s LOCKSPACE] [-r RESOURCE]
+Tell the sanlock daemon to exit. Any registered processes will be killed,
+their resource leases released, and lockspaces removed.
-.TP
-.B direct acquire|release -i <num> -g <num> -r RESOURCE
+.BR "sanlock client add_lockspace -s" " LOCKSPACE"
-.TP
-.B direct acquire_id|renew_id|release_id|read_id|live_id -s LOCKSPACE
+Tell the sanlock daemon to acquire the specified host_id in the lockspace.
+This will allow resources to be acquired in the lockspace.
+
+.BR "sanlock client rem_lockspace -s" " LOCKSPACE"
+
+Tell the sanlock daemon to release the specified host_id in the lockspace.
+Any processes holding resource leases in this lockspace will be killed,
+and the leases released.
+
+.BR "sanlock client command -r" " RESOURCE " \
+\fB-c\fP " " \fIpath\fP " " \fIargs\fP
+
+Register with the sanlock daemon, acquire the specified resource lease,
+and exec the command at path with args. When the command exits, the
+sanlock daemon will release the lease. -c must be the final option.
+
+.BR "sanlock client acquire -r" " RESOURCE " \
+\fB-p\fP " " \fIpid\fP
+.br
+.BR "sanlock client release -r" " RESOURCE " \
+\fB-p\fP " " \fIpid\fP
+Tell the sanlock daemon to acquire or release the specified resource lease
+for the given pid. The pid must be registered with the sanlock daemon.
+acquire can optionally take a versioned RESOURCE string RESOURCE:lver,
+where lver is the version of the lease that must be acquired, or fail.
+
+.BI "sanlock client inquire -p" " pid"
+
+Print the resource leases held the given pid. The format is a versioned
+RESOURCE string "RESOURCE:lver" where lver is the version of the lease
+held.
+
+.B "sanlock direct"
+.I action
+[options]
+
+.BI \-a " num"
+use async i/o (1 yes, 0 no, default 1)
+
+.BI -o " num"
+io timeout in seconds (default 10)
+
+.BR "sanlock direct init -s" " LOCKSPACE"
+.br
+.BR "sanlock direct init -r" " RESOURCE"
+
+Initialize storage for 2000 host_id (delta) leases for the given
+lockspace, or initialize storage for one resource (paxos) lease. Both
+options require 1MB of space. The host_id in the LOCKSPACE string is not
+relevant to initialization, so the value is ignored. (The default of 2000
+host_ids can be changed for special cases using the -n num_hosts and -m
+max_hosts options.)
+
+.BR "sanlock direct read_leader -s" " LOCKSPACE"
+.br
+.BR "sanlock direct read_leader -r" " RESOURCE"
+
+Read a leader record from disk and print the fields. The leader record is
+the single sector of a delta lease, or the first sector of a paxos lease.
+
+.BR "sanlock direct read_id -s" " LOCKSPACE"
+.br
+.BR "sanlock direct live_id -s" " LOCKSPACE"
+
+read_id reads a host_id and prints the owner. live_id reads a host_id
+once a second until it the timestamp or owner change (prints live 1), or
+until host_dead_seconds (prints live 0). (host_dead_seconds is derived
+from the io_timeout option. The live 0|1 conclusion will not match the
+sanlock daemon's conclusion unless the configured timeouts match.)
+./" .P
+./" .BR "sanlock direct acquire_id -s" " LOCKSPACE"
+./" .br
+./" .BR "sanlock direct renew_id -s" " LOCKSPACE"
+./" .br
+./" .BR "sanlock direct release_id -s" " LOCKSPACE"
+./"
+./" Acquire, renew, or release a host_id directly to disk, independent from
+./" the sanlock daemon. Not for general use. This should only be used for
+./" testing or for manual recovery in an emergency.
+./"
+./" .P
+./" .BR "sanlock direct acquire -r" " RESOURCE " \
+./" \fB-i\fP " " \fInum\fP " " \fB-g\fP " " \fInum\fP
+./" .br
+./" .BR "sanlock direct release -r" " RESOURCE " \
+./" \fB-i\fP " " \fInum\fP " " \fB-g\fP " " \fInum\fP
+./"
+./" Not supported. Not for general use.
+./"
+
+.BI "sanlock direct dump" " path" \
+\fR[\fP\fB:\fP\fIoffset\fP\fR]\fP
+
+Read disk sectors and print leader records for delta or paxos leases.
+
+
+.SS
+LOCKSPACE option string
+
+.BR \-s " " \fIlockspace_name\fP:\fIhost_id\fP:\fIpath\fP:\fIoffset\fP
+.P
+.IR lockspace_name " name of the lockspace"
+.br
+.IR host_id " local host identifier in lockspace"
+.br
+.IR path " path to storage reserved for leases"
+.br
+.IR offset " offset on path (bytes)"
+.br
+
+.SS
+RESOURCE option string
+
+.BR \-r " " \fIlockspace_name\fP:\fIresource_name\fP:\fIpath\fP:\fIoffset\fP
+.P
+.IR lockspace_name " name of lockspace"
+.br
+.IR resource_name " name of resource"
+.br
+.IR path " path to storage reserved for leases"
+.br
+.IR offset " offset on path (bytes)"
+.br
+.IR lver " optional leader version"
.SH SEE ALSO
.BR wdmd (8)
12 years, 8 months
src/sanlock.8
by David Teigland
src/sanlock.8 | 170 ++++++++++++++++++++++++++++++++++++++++++++++++----------
1 file changed, 143 insertions(+), 27 deletions(-)
New commits:
commit eb27ff9918c4ed834685c045931c55c9fa4d816f
Author: David Teigland <teigland(a)redhat.com>
Date: Wed Aug 3 17:08:23 2011 -0500
sanlock: update man page
diff --git a/src/sanlock.8 b/src/sanlock.8
index b7405cf..7d7d2ed 100644
--- a/src/sanlock.8
+++ b/src/sanlock.8
@@ -11,39 +11,45 @@ sanlock \- shared storage lock manager
The sanlock daemon manages leases for applications running on a cluster of
hosts with shared storage. All lease management and coordination is done
-through reading and writing blocks on the shared storage. Two basic types
-of leases are used, each based on a different algorithm:
+through reading and writing blocks on the shared storage. Two types of
+leases are used, each based on a different algorithm:
"delta leases" are slow to acquire and demand regular i/o to shared
-storage. sanlock only uses them internally to hold a lease on its
-"host_id" (an integer host identifier from 1-2000). They simply prevent
-two hosts from using the same host identifier. They also serve in a
-limited way as host "liveness" indicators since they are renewed.
+storage. A delta lease exists in a single sector of shared storage.
+Acquiring a delta lease involves reads and writes to that sector separated
+by specific delays. Once acquired, the lease must be renewed by writing a
+new timestamp to the sector regularly. sanlock only uses delta leases
+internally to hold a lease on its host_id. The host_id leases prevent two
+hosts from using the same host_id and provide basic host liveness
+information based on the renewals.
"paxos leases" are generally fast to acquire and sanlock makes them
-available to applications as "general purpose" resource leases. The disk
-paxos algorithm uses host_id's internally, and does not work if different
-hosts use the same host_id. So the delta leases provide unique host_id's
-for implementing paxos leases.
-
-Before sanlock can be used, the user must assign each host an identifier,
-or "host_id", which is an integer value between 1 and 2000. Two hosts
-should not be given the same host_id.
+available to applications as general purpose resource leases. A paxos
+lease exists in 1MB of shared storage (8MB for 4k sectors). Acquiring a
+paxos lease involves reads and writes to max_hosts (2000) sectors in a
+specific sequence specified by the Disk Paxos algorithm. paxos leases use
+host_id's internally to indicate the owner of the lease, and the algorithm
+will not work if different hosts use the same host_id. So, the delta
+leases provide the unique host_id's used in paxos leases. paxos leases
+also use delta leases to check if a host_id is alive.
+
+Before sanlock can be used, the user must assign each host a host_id,
+which is a number between 1 and 2000. Two hosts should not be given the
+same host_id.
sanlock views a pool of storage as a "lockspace". Each distinct pool of
storage, e.g. from different sources, would typically be defined as a
-separate lockspace, with a separate lockspace name.
+separate lockspace, with a unique lockspace name.
-A part of this storage space must be reserved and initialized for sanlock
-to store delta leases. Each host that wants to use the lockspace must
-first acquire a delta lease on its host_id number within the lockspace.
-(See the add_lockspace action/api.) These delta leases prevent two hosts
-from using the same host_id, and allow hosts to detect which other hosts
-are currently using the lockspace. The space required for 2000 delta
-leases in the lockspace (for all possible host_id's) is 1GB for storage
-with 512 byte sectors, or 8GB for storage with 4k sectors.
+Part of this storage space must be reserved and initialized for sanlock to
+store delta leases. Each host that wants to use the lockspace must first
+acquire a delta lease on its host_id number within the lockspace. (See
+the add_lockspace action/api.) The space required for 2000 delta leases
+in the lockspace (for all possible hosts, max_hosts, which is 2000) is 1MB
+(8MB for 4k sectors). (This is the same size as that required for a
+single paxos lease.)
-Example:
+.B example
1. Create storage pools and reserve and initialize host_id leases
.br
@@ -53,9 +59,9 @@ vgcreate pool1 /dev/sdb
.br
vgcreate pool2 /dev/sdc
.br
-lvcreate -n hostid_leases -L 1GB pool1
+lvcreate -n hostid_leases -L 1MB pool1
.br
-lvcreate -n hostid_leases -L 1GB pool2
+lvcreate -n hostid_leases -L 1MB pool2
.br
sanlock direct init -s lockspace1:0:/dev/pool1/hostid_leases:0
.br
@@ -85,7 +91,64 @@ sanlock client add_lockspace -s lockspace2:2:/dev/pool2/hostid_leases:0
4. Applications can now reserve/initialize space for resource leases, and
then acquire the leases as they need to access the resources.
+What resource leases are created and how they are used depends on the
+application. For example, say application foo, running on multiple hosts,
+needs to synchronize access to data it stores on /dev/pool1/foo_data. foo
+could use a resource lease as follows:
+
+5. Reserve and initialize a single resource lease for foo_data
+.br
+lvcreate -n foo_data -L 1MB pool1
+.br
+sanlock direct init -r lockspace1:foo_data:/dev/pool1/foo_data:0
+.br
+
+6. Acquire the lease from the app using libsanlock (see sanlock_register,
+sanlock_acquire). If the app is already running as pid 1234, and has
+registered with the sanlock daemon, the lease can be added for it
+manually.
+.br
+sanlock client acquire
+ -p 1234 -r lockspace1:foo_data:/dev/pool1/foo_data:0
+.br
+
+.B client failure
+
+If a process holding resource leases fails or exits without releasing its
+leases, sanlock will release the leases for it automatically.
+
+.B lockspace failure
+
+If the sanlock daemon cannot renew a lockspace host_id for a specific
+period of time (usually because storage access is lost), sanlock will kill
+any process holding a resource lease within the lockspace.
+
+.B watchdog usage
+
+The watchdog device is used to further protect shared resources in case
+processes cannot be killed when a host_id fails to be renewed. The
+watchdog device will reset the host withing a known fixed time period from
+the last successful host_id renewal. This known time is the length of
+time that sanlock on another host must wait before granting the resource
+lease that was held by a process on the failed host.
+.B levels of protection
+
+sanlock leases are meant to guarantee that two different process on two
+different hosts are never allowed to hold the same resource lease at once.
+If they were, the resource being protected could be corrupted. There are
+three levels of protection built into sanlock itself:
+
+1. The paxos leases and delta leases themselves.
+
+2. If the leases cannot function because storage access is lost (host_id's
+cannot be renewed), the sanlock daemon kills any pids using resource
+leases in the lockspace.
+
+3. If the pids do not exit after being killed, the watchdog device resets
+the host.
+
+.SH COMMANDS
sanlock is a multi purpose binary, with three top level commands:
@@ -101,11 +164,64 @@ send request to daemon (default command if none given)
.B direct
access storage directly (no coordination with daemon)
+.SH OPTIONS
+
+.TP
+.nf
+LOCKSPACE = <lockspace_name>:<host_id>:<path>:<offset>
+.fi
+
+.TP
+.nf
+RESOURCE = <lockspace_name>:<resource_name>:<path>:<offset>[:<lver>]
+.fi
+
.SS client actions
+.TP
+.B status
+
+.TP
+.B log_dump
+
+.TP
+.B shutdown
+
+.TP
+.B client add_lockspace -s LOCKSPACE
+
+.TP
+.B client rem_lockspace -s LOCKSPACE
+
+.TP
+.B client command -r RESOURCE -c <path> <args>
+
+.TP
+.B client acquire -p <pid> -r RESOURCE
+
+.TP
+.B client release -p <pid> -r RESOURCE
+
+.TP
+.B client inquire -p <pid>
+
.SS direct actions
-.SH OPTIONS
+.TP
+.B direct init [-s LOCKSPACE] [-r RESOURCE]
+
+.TP
+.B direct dump <path>[:<offset>] [options]
+
+.TP
+.B direct read_leader [-s LOCKSPACE] [-r RESOURCE]
+
+.TP
+.B direct acquire|release -i <num> -g <num> -r RESOURCE
+
+.TP
+.B direct acquire_id|renew_id|release_id|read_id|live_id -s LOCKSPACE
+
.SH SEE ALSO
.BR wdmd (8)
12 years, 8 months
3 commits - src/Makefile src/main.c src/monotime.c src/monotime.h wdmd/main.c
by David Teigland
src/Makefile | 11 +++++------
src/main.c | 4 +++-
src/monotime.c | 13 +++++++++++++
src/monotime.h | 14 ++++++++++++++
wdmd/main.c | 5 ++++-
5 files changed, 39 insertions(+), 8 deletions(-)
New commits:
commit 1b7f6c1ff6468066c173e77bb11c6cfc23ae3140
Author: David Teigland <teigland(a)redhat.com>
Date: Wed Aug 3 15:11:07 2011 -0500
sanlock: add missing monotime files
forgot to add in earlier commit
diff --git a/src/monotime.c b/src/monotime.c
new file mode 100644
index 0000000..79e7a58
--- /dev/null
+++ b/src/monotime.c
@@ -0,0 +1,13 @@
+#include <unistd.h>
+#include <stdint.h>
+#include <time.h>
+#include "monotime.h"
+
+uint64_t monotime(void)
+{
+ struct timespec ts;
+
+ clock_gettime(CLOCK_MONOTONIC, &ts);
+
+ return ts.tv_sec;
+}
diff --git a/src/monotime.h b/src/monotime.h
new file mode 100644
index 0000000..c375111
--- /dev/null
+++ b/src/monotime.h
@@ -0,0 +1,14 @@
+/*
+ * Copyright (C) 2011 Red Hat, Inc. All rights reserved.
+ *
+ * This copyrighted material is made available to anyone wishing to use,
+ * modify, copy, or redistribute it subject to the terms and conditions
+ * of the GNU General Public License v.2.
+ */
+
+#ifndef __MONOTIME_H__
+#define __MONOTIME_H__
+
+uint64_t monotime(void);
+
+#endif
commit e5755f76327b05a1b980d1739013cb7a767156b8
Author: David Teigland <teigland(a)redhat.com>
Date: Wed Aug 3 14:34:06 2011 -0500
sanlock/wdmd: nonblocking listening/accept
Undo previous commits that changed accept() to accept4(SOCK_NONBLOCK).
That causes the fd returned from accept to be nonblocking, which is
not what I want (it causes recv's to start returning EAGAIN). What
I intended was the listening fd to be nonblocking.
diff --git a/src/main.c b/src/main.c
index b58ecd4..1dc268e 100644
--- a/src/main.c
+++ b/src/main.c
@@ -2033,7 +2033,7 @@ static void process_listener(int ci GNUC_UNUSED)
int fd;
int on = 1;
- fd = accept4(client[ci].fd, NULL, NULL, SOCK_NONBLOCK);
+ fd = accept(client[ci].fd, NULL, NULL);
if (fd < 0)
return;
@@ -2072,6 +2072,8 @@ static int setup_listener(void)
if (rv < 0)
goto exit_fail;
+ fcntl(fd, F_SETFL, fcntl(fd, F_GETFL, 0) | O_NONBLOCK);
+
ci = client_add(fd, process_listener, NULL);
if (ci < 0)
goto exit_fail;
diff --git a/wdmd/main.c b/wdmd/main.c
index 8977a49..95dcd3b 100644
--- a/wdmd/main.c
+++ b/wdmd/main.c
@@ -280,7 +280,7 @@ static void process_listener(int ci)
int fd;
int on = 1;
- fd = accept4(client[ci].fd, NULL, NULL, SOCK_NONBLOCK);
+ fd = accept(client[ci].fd, NULL, NULL);
if (fd < 0)
return;
@@ -327,6 +327,9 @@ static int setup_listener_socket(int *listener_socket)
close(s);
return rv;
}
+
+ fcntl(s, F_SETFL, fcntl(s, F_GETFL, 0) | O_NONBLOCK);
+
*listener_socket = s;
return 0;
}
commit 4329415cd8387118072a90ee09a4829920457b31
Author: David Teigland <teigland(a)redhat.com>
Date: Wed Aug 3 14:11:43 2011 -0500
sanlock: build with pie
diff --git a/src/Makefile b/src/Makefile
index edc299b..ec318d3 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -62,15 +62,14 @@ CFLAGS += -D_GNU_SOURCE -g \
-fexceptions \
-fasynchronous-unwind-tables \
-fdiagnostics-show-option
-# -fPIE -DPIE
+
+CMD_CFLAGS = $(CFLAGS) -fPIE -DPIE
CMD_LDFLAGS += -lpthread -luuid -lrt -laio -lblkid -lsanlock -L../wdmd -lwdmd
-CMD_LDFLAGS += -Wl,-z,now -Wl,-z,relro
-#CMD_LDFLAGS += -Wl,-z,now -Wl,-z,relro -pie
+CMD_LDFLAGS += -Wl,-z,now -Wl,-z,relro -pie
LIB_LDFLAGS += -lpthread -lrt -laio -lblkid -L../wdmd -lwdmd
-LIB_LDFLAGS += -Wl,-z,relro
-#LIB_LDFLAGS += -Wl,-z,relro -pie
+LIB_LDFLAGS += -Wl,-z,relro -pie
all: $(SHLIB_TARGET) $(CMD_TARGET)
@@ -80,7 +79,7 @@ $(SHLIB_TARGET): $(LIB_SOURCE)
ln -sf $(SHLIB_TARGET) $(LIB_TARGET).so.$(SOMAJOR)
$(CMD_TARGET): $(SHLIB_TARGET) $(CMD_SOURCE)
- $(CC) $(CFLAGS) $(CMD_LDFLAGS) $(CMD_SOURCE) -o $@ -L.
+ $(CC) $(CMD_CFLAGS) $(CMD_LDFLAGS) $(CMD_SOURCE) -o $@ -L.
clean:
rm -f *.o *.so *.so.* $(CMD_TARGET)
12 years, 8 months
src/Makefile src/delta_lease.c src/direct.c src/host_id.c src/paxos_lease.c src/sanlock_internal.h wdmd/Makefile wdmd/main.c
by David Teigland
src/Makefile | 2 ++
src/delta_lease.c | 4 ++--
src/direct.c | 4 ++--
src/host_id.c | 12 ++++++------
src/paxos_lease.c | 16 ++++++++--------
src/sanlock_internal.h | 1 +
wdmd/Makefile | 2 +-
wdmd/main.c | 26 +++++++++++++++++---------
8 files changed, 39 insertions(+), 28 deletions(-)
New commits:
commit f95d08fd484e3c5caba4e2d7152e530d658c7af0
Author: David Teigland <teigland(a)redhat.com>
Date: Wed Aug 3 12:35:22 2011 -0500
sanlock/wdmd: use monotonic time
to avoid problems if the real time is changed
diff --git a/src/Makefile b/src/Makefile
index 3aadb5d..edc299b 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -26,6 +26,7 @@ CMD_SOURCE = \
task.c \
token_manager.c \
watchdog.c \
+ monotime.c \
sanlock_sock.c
LIB_SOURCE = \
@@ -39,6 +40,7 @@ LIB_SOURCE = \
direct.c \
task.c \
direct_lib.c \
+ monotime.c \
sanlock_sock.c
CFLAGS += -D_GNU_SOURCE -g \
diff --git a/src/delta_lease.c b/src/delta_lease.c
index a7f46d1..bf5ad82 100644
--- a/src/delta_lease.c
+++ b/src/delta_lease.c
@@ -286,7 +286,7 @@ int delta_lease_acquire(struct task *task,
}
write_new:
- new_ts = time(NULL);
+ new_ts = monotime();
leader.timestamp = new_ts;
leader.owner_id = host_id;
leader.owner_generation++;
@@ -402,7 +402,7 @@ int delta_lease_renew(struct task *task,
return SANLK_RENEW_DIFF;
}
- new_ts = time(NULL);
+ new_ts = monotime();
if (leader.timestamp >= new_ts) {
log_erros(sp, "delta_renew timestamp too small");
diff --git a/src/direct.c b/src/direct.c
index 03c6a16..8b09fe8 100644
--- a/src/direct.c
+++ b/src/direct.c
@@ -326,7 +326,7 @@ int direct_live_id(struct task *task,
if (rv < 0)
return rv;
- start = time(NULL);
+ start = monotime();
while (1) {
sleep(1);
@@ -350,7 +350,7 @@ int direct_live_id(struct task *task,
break;
}
- if (time(NULL) - start > task->host_dead_seconds) {
+ if (monotime() - start > task->host_dead_seconds) {
*live = 0;
break;
}
diff --git a/src/host_id.c b/src/host_id.c
index 822e8cb..7341d65 100644
--- a/src/host_id.c
+++ b/src/host_id.c
@@ -187,7 +187,7 @@ int host_id_check(struct task *task, struct space *sp)
return 0;
}
- gap = time(NULL) - last_success;
+ gap = monotime() - last_success;
if (gap >= task->id_renewal_fail_seconds) {
log_erros(sp, "host_id_check failed %d", gap);
@@ -245,7 +245,7 @@ static void *lockspace_thread(void *arg_in)
setup_task_aio(&task, main_task.use_aio, HOSTID_AIO_CB_SIZE);
memcpy(task.name, sp->space_name, NAME_ID_SIZE);
- last_attempt = time(NULL);
+ last_attempt = monotime();
rv = open_disk(&sp->host_id_disk);
if (rv < 0) {
@@ -259,7 +259,7 @@ static void *lockspace_thread(void *arg_in)
sp->space_name, our_host_name_global,
sp->host_id, &leader);
delta_result = result;
- delta_length = time(NULL) - last_attempt;
+ delta_length = monotime() - last_attempt;
if (result == SANLK_OK)
last_success = leader.timestamp;
@@ -303,7 +303,7 @@ static void *lockspace_thread(void *arg_in)
if (stop)
break;
- if (time(NULL) - last_success < task.id_renewal_seconds) {
+ if (monotime() - last_success < task.id_renewal_seconds) {
sleep(1);
continue;
} else {
@@ -312,13 +312,13 @@ static void *lockspace_thread(void *arg_in)
usleep(500000);
}
- last_attempt = time(NULL);
+ last_attempt = monotime();
result = delta_lease_renew(&task, sp, &sp->host_id_disk,
sp->space_name, delta_result,
&leader, &leader);
delta_result = result;
- delta_length = time(NULL) - last_attempt;
+ delta_length = monotime() - last_attempt;
if (result == SANLK_OK)
last_success = leader.timestamp;
diff --git a/src/paxos_lease.c b/src/paxos_lease.c
index 1069639..25165f1 100644
--- a/src/paxos_lease.c
+++ b/src/paxos_lease.c
@@ -446,7 +446,7 @@ static int run_ballot(struct task *task, struct token *token, int num_hosts,
/* lver and mbal are already set */
dblock.inp = token->host_id;
dblock.inp2 = token->host_generation;
- dblock.inp3 = time(NULL);
+ dblock.inp3 = monotime();
}
dblock.bal = dblock.mbal;
dblock.checksum = dblock_checksum(&dblock);
@@ -1102,7 +1102,7 @@ int paxos_lease_acquire(struct task *task,
host_id_disk.fd);
}
- start = time(NULL);
+ start = monotime();
while (1) {
error = delta_lease_leader_read(task, &host_id_disk,
@@ -1157,15 +1157,15 @@ int paxos_lease_acquire(struct task *task,
by now
if we trust that the clocks are in sync among hosts, then this
- check could be: if (time(NULL) - host_id_leader.timestamp >
+ check could be: if (time() - host_id_leader.timestamp >
task->host_dead_seconds), but if the clocks are out of sync,
this check would easily give two hosts the lease.
- N.B. we need to be careful about ever comparing local time(NULL)
+ N.B. we need to be careful about ever comparing local time
to a time value we read off disk from another node that may
have different time. */
- if (time(NULL) - start > task->host_dead_seconds) {
+ if (monotime() - start > task->host_dead_seconds) {
log_token(token, "paxos_acquire host_id %llu expired %llu",
(unsigned long long)cur_leader.owner_id,
(unsigned long long)host_id_leader.timestamp);
@@ -1313,7 +1313,7 @@ int paxos_lease_acquire(struct task *task,
new_leader.write_id = token->host_id;
new_leader.write_generation = token->host_generation;
- new_leader.write_timestamp = time(NULL);
+ new_leader.write_timestamp = monotime();
if (new_num_hosts)
new_leader.num_hosts = new_num_hosts;
@@ -1376,7 +1376,7 @@ int paxos_lease_renew(struct task *task,
}
}
- new_leader.timestamp = time(NULL);
+ new_leader.timestamp = monotime();
new_leader.checksum = leader_checksum(&new_leader);
error = write_new_leader(task, token, &new_leader);
@@ -1450,7 +1450,7 @@ int paxos_lease_release(struct task *task,
leader.timestamp = LEASE_FREE;
leader.write_id = token->host_id;
leader.write_generation = token->host_generation;
- leader.write_timestamp = time(NULL);
+ leader.write_timestamp = monotime();
leader.checksum = leader_checksum(&leader);
error = write_new_leader(task, token, &leader, "paxos_release");
diff --git a/src/sanlock_internal.h b/src/sanlock_internal.h
index b25a8eb..833822e 100644
--- a/src/sanlock_internal.h
+++ b/src/sanlock_internal.h
@@ -26,6 +26,7 @@
#include "sanlock_rv.h"
#include "leader.h"
#include "list.h"
+#include "monotime.h"
#include <libaio.h>
diff --git a/wdmd/Makefile b/wdmd/Makefile
index be3f030..8b7330f 100644
--- a/wdmd/Makefile
+++ b/wdmd/Makefile
@@ -42,7 +42,7 @@ CFLAGS += -D_GNU_SOURCE -g \
-fdiagnostics-show-option \
-fPIE -DPIE
-CMD_LDFLAGS += -lwdmd
+CMD_LDFLAGS += -lwdmd -lrt
CMD_LDFLAGS += -Wl,-z,now -Wl,-z,relro -pie
LIB_LDFLAGS += -Wl,-z,relro -pie
diff --git a/wdmd/main.c b/wdmd/main.c
index d8994b4..8977a49 100644
--- a/wdmd/main.c
+++ b/wdmd/main.c
@@ -98,6 +98,14 @@ do { \
} while (0)
+static uint64_t monotime(void)
+{
+ struct timespec ts;
+
+ clock_gettime(CLOCK_MONOTONIC, &ts);
+
+ return ts.tv_sec;
+}
/*
* test clients
@@ -341,7 +349,7 @@ static int test_clients(void)
int fail_count = 0;
int i;
- t = time(NULL);
+ t = monotime();
for (i = 0; i < client_size; i++) {
if (!client[i].used)
@@ -435,7 +443,7 @@ static int test_files(void)
if (rv < 0)
continue;
- t = time(NULL);
+ t = monotime();
if (t >= expire) {
log_error("test failed file %s renewal %llu expire %llu ",
@@ -523,7 +531,7 @@ static int check_script(int i)
if (!scripts[i].pid)
return 0;
- begin = time(NULL);
+ begin = monotime();
while (1) {
rv = waitpid(scripts[i].pid, &status, WNOHANG);
@@ -533,7 +541,7 @@ static int check_script(int i)
} else if (!rv) {
/* pid still running */
- if (time(NULL) - begin >= SCRIPT_WAIT_SECONDS) {
+ if (monotime() - begin >= SCRIPT_WAIT_SECONDS) {
rv = -ETIMEDOUT;
goto out;
}
@@ -549,7 +557,7 @@ static int check_script(int i)
} else {
/* pid state changed but still running */
- if (time(NULL) - begin >= 2) {
+ if (monotime() - begin >= 2) {
rv = -ETIMEDOUT;
goto out;
}
@@ -666,7 +674,7 @@ static void pet_watchdog(void)
rv = ioctl(dev_fd, WDIOC_KEEPALIVE, &unused);
- last_keepalive = time(NULL);
+ last_keepalive = monotime();
log_debug("keepalive %d", rv);
}
@@ -747,8 +755,8 @@ static int test_loop(void)
if (daemon_quit && !active_clients())
break;
- if (time(NULL) - test_time >= test_interval) {
- test_time = time(NULL);
+ if (monotime() - test_time >= test_interval) {
+ test_time = monotime();
log_debug("test_time %llu",
(unsigned long long)test_time);
@@ -761,7 +769,7 @@ static int test_loop(void)
pet_watchdog();
}
- sleep_seconds = test_time + test_interval - time(NULL);
+ sleep_seconds = test_time + test_interval - monotime();
poll_timeout = (sleep_seconds > 0) ? sleep_seconds * 1000 : 1;
log_debug("sleep_seconds %d", sleep_seconds);
}
12 years, 8 months
2 commits - src/Makefile src/client_admin.c src/client_msg.c src/client_msg.h src/client_resource.c src/host_id.c src/main.c src/sanlock_sock.c src/sanlock_sock.h wdmd/Makefile wdmd/client.c wdmd/main.c wdmd/wdmd_sock.c wdmd/wdmd_sock.h
by David Teigland
src/Makefile | 15 ++++++++++----
src/client_admin.c | 1
src/client_msg.c | 52 +-------------------------------------------------
src/client_msg.h | 34 --------------------------------
src/client_resource.c | 1
src/host_id.c | 3 +-
src/main.c | 32 +++++++++++++++++++++++++++---
src/sanlock_sock.c | 20 +++++++++++++++++++
src/sanlock_sock.h | 46 ++++++++++++++++++++++++++++++++++++++++++++
wdmd/Makefile | 26 +++++++++++++++----------
wdmd/client.c | 35 +--------------------------------
wdmd/main.c | 5 +---
wdmd/wdmd_sock.c | 21 ++++++++++++++++++++
wdmd/wdmd_sock.h | 6 +++++
14 files changed, 159 insertions(+), 138 deletions(-)
New commits:
commit 5f218b4aad41c330f5d33ce59cceab445b72b59e
Author: David Teigland <teigland(a)redhat.com>
Date: Wed Aug 3 11:42:20 2011 -0500
sanlock: new build flags
and move some code around to get it to work
diff --git a/src/Makefile b/src/Makefile
index 66d5fcd..3aadb5d 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -25,7 +25,8 @@ CMD_SOURCE = \
paxos_lease.c \
task.c \
token_manager.c \
- watchdog.c
+ watchdog.c \
+ sanlock_sock.c
LIB_SOURCE = \
client_msg.c \
@@ -37,7 +38,8 @@ LIB_SOURCE = \
paxos_lease.c \
direct.c \
task.c \
- direct_lib.c
+ direct_lib.c \
+ sanlock_sock.c
CFLAGS += -D_GNU_SOURCE -g \
-Wall \
@@ -58,10 +60,15 @@ CFLAGS += -D_GNU_SOURCE -g \
-fexceptions \
-fasynchronous-unwind-tables \
-fdiagnostics-show-option
+# -fPIE -DPIE
-CMD_LDFLAGS = -lpthread -luuid -lrt -laio -lblkid -lsanlock -L../wdmd -lwdmd
-LIB_LDFLAGS = -lpthread -lrt -laio -lblkid -L../wdmd -lwdmd
+CMD_LDFLAGS += -lpthread -luuid -lrt -laio -lblkid -lsanlock -L../wdmd -lwdmd
+CMD_LDFLAGS += -Wl,-z,now -Wl,-z,relro
+#CMD_LDFLAGS += -Wl,-z,now -Wl,-z,relro -pie
+LIB_LDFLAGS += -lpthread -lrt -laio -lblkid -L../wdmd -lwdmd
+LIB_LDFLAGS += -Wl,-z,relro
+#LIB_LDFLAGS += -Wl,-z,relro -pie
all: $(SHLIB_TARGET) $(CMD_TARGET)
diff --git a/src/client_admin.c b/src/client_admin.c
index fc3fa4a..94645ec 100644
--- a/src/client_admin.c
+++ b/src/client_admin.c
@@ -28,6 +28,7 @@
#include <sys/un.h>
#include "sanlock_internal.h"
+#include "sanlock_sock.h"
#include "log.h"
#include "client_msg.h"
#include "sanlock_admin.h"
diff --git a/src/client_msg.c b/src/client_msg.c
index 28c0374..21b18db 100644
--- a/src/client_msg.c
+++ b/src/client_msg.c
@@ -28,57 +28,9 @@
#include <sys/un.h>
#include "sanlock_internal.h"
-#include "log.h"
+#include "sanlock_sock.h"
#include "client_msg.h"
-static int get_socket_address(struct sockaddr_un *addr)
-{
- memset(addr, 0, sizeof(struct sockaddr_un));
- addr->sun_family = AF_LOCAL;
- snprintf(addr->sun_path, sizeof(addr->sun_path) - 1, "%s/%s",
- SANLK_RUN_DIR, SANLK_SOCKET_NAME);
- return 0;
-}
-
-int setup_listener_socket(int *listener_socket,
- uid_t owner, gid_t group, mode_t mode)
-{
- int rv, s;
- struct sockaddr_un addr;
-
- s = socket(AF_LOCAL, SOCK_STREAM, 0);
- if (s < 0)
- return -1;
-
- rv = get_socket_address(&addr);
- if (rv < 0)
- return rv;
-
- unlink(addr.sun_path);
- rv = bind(s, (struct sockaddr *) &addr, sizeof(struct sockaddr_un));
- if (rv < 0)
- goto exit_fail;
-
- rv = chmod(addr.sun_path, mode);
- if (rv < 0)
- goto exit_fail;
-
- rv = chown(addr.sun_path, owner, group);
- if (rv < 0)
- goto exit_fail;
-
- rv = listen(s, 5);
- if (rv < 0)
- goto exit_fail;
-
- *listener_socket = s;
- return 0;
-
- exit_fail:
- close(s);
- return -1;
-}
-
int connect_socket(int *sock_fd)
{
int rv, s;
@@ -88,7 +40,7 @@ int connect_socket(int *sock_fd)
if (s < 0)
return -1;
- rv = get_socket_address(&addr);
+ rv = sanlock_socket_address(&addr);
if (rv < 0)
return rv;
diff --git a/src/client_msg.h b/src/client_msg.h
index dcb943a..b7ab2d5 100644
--- a/src/client_msg.h
+++ b/src/client_msg.h
@@ -9,40 +9,6 @@
#ifndef __CLIENT_MSG_H__
#define __CLIENT_MSG_H__
-#define MAX_CLIENT_MSG (1024 * 1024) /* TODO: this is random */
-
-enum {
- SM_CMD_REGISTER = 1,
- SM_CMD_ADD_LOCKSPACE = 2,
- SM_CMD_REM_LOCKSPACE = 3,
- SM_CMD_SHUTDOWN = 4,
- SM_CMD_STATUS = 5,
- SM_CMD_LOG_DUMP = 6,
- SM_CMD_ACQUIRE = 7,
- SM_CMD_RELEASE = 8,
- SM_CMD_INQUIRE = 9,
- SM_CMD_RESTRICT = 10,
-};
-
-#define SANLK_STATE_MAXSTR 4096
-
-#define SANLK_STATE_DAEMON 1
-#define SANLK_STATE_LOCKSPACE 2
-#define SANLK_STATE_CLIENT 3
-#define SANLK_STATE_RESOURCE 4
-
-struct sanlk_state {
- uint32_t type; /* SANLK_STATE_ */
- uint32_t flags;
- uint32_t data32; /* pid (for client) */
- uint64_t data64;
- char name[SANLK_NAME_LEN]; /* client name or resource name */
- uint32_t str_len;
- char str[0]; /* string of internal state */
-};
-
-int setup_listener_socket(int *listener_socket,
- uid_t owner, gid_t group, mode_t mode);
int connect_socket(int *sock_fd);
int send_header(int sock, int cmd, uint32_t cmd_flags, int datalen,
uint32_t data, uint32_t data2);
diff --git a/src/client_resource.c b/src/client_resource.c
index fd3228a..eeffae4 100644
--- a/src/client_resource.c
+++ b/src/client_resource.c
@@ -28,6 +28,7 @@
#include <sys/un.h>
#include "sanlock_internal.h"
+#include "sanlock_sock.h"
#include "client_msg.h"
#include "sanlock_resource.h"
diff --git a/src/host_id.c b/src/host_id.c
index 142f61b..822e8cb 100644
--- a/src/host_id.c
+++ b/src/host_id.c
@@ -22,15 +22,16 @@
#include <sys/types.h>
#include <sys/time.h>
#include <sys/utsname.h>
+#include <sys/un.h>
#include <uuid/uuid.h>
#include "sanlock_internal.h"
+#include "sanlock_sock.h"
#include "diskio.h"
#include "log.h"
#include "delta_lease.h"
#include "host_id.h"
#include "watchdog.h"
-#include "client_msg.h"
#include "task.h"
static unsigned int space_id_counter = 1;
diff --git a/src/main.c b/src/main.c
index 309d242..b58ecd4 100644
--- a/src/main.c
+++ b/src/main.c
@@ -43,7 +43,7 @@
#include "lockfile.h"
#include "watchdog.h"
#include "task.h"
-#include "client_msg.h"
+#include "sanlock_sock.h"
#include "sanlock_resource.h"
#include "sanlock_admin.h"
@@ -2044,18 +2044,44 @@ static void process_listener(int ci GNUC_UNUSED)
static int setup_listener(void)
{
+ struct sockaddr_un addr;
int rv, fd, ci;
- rv = setup_listener_socket(&fd, com.uid, com.gid, DEFAULT_SOCKET_MODE);
+ rv = sanlock_socket_address(&addr);
if (rv < 0)
return rv;
+ fd = socket(AF_LOCAL, SOCK_STREAM, 0);
+ if (fd < 0)
+ return fd;
+
+ unlink(addr.sun_path);
+ rv = bind(fd, (struct sockaddr *) &addr, sizeof(struct sockaddr_un));
+ if (rv < 0)
+ goto exit_fail;
+
+ rv = chmod(addr.sun_path, DEFAULT_SOCKET_MODE);
+ if (rv < 0)
+ goto exit_fail;
+
+ rv = chown(addr.sun_path, com.uid, com.gid);
+ if (rv < 0)
+ goto exit_fail;
+
+ rv = listen(fd, 5);
+ if (rv < 0)
+ goto exit_fail;
+
ci = client_add(fd, process_listener, NULL);
if (ci < 0)
- return -1;
+ goto exit_fail;
strcpy(client[ci].owner_name, "listener");
return 0;
+
+ exit_fail:
+ close(fd);
+ return -1;
}
static void sigterm_handler(int sig GNUC_UNUSED)
diff --git a/src/sanlock_sock.c b/src/sanlock_sock.c
new file mode 100644
index 0000000..dea7c34
--- /dev/null
+++ b/src/sanlock_sock.c
@@ -0,0 +1,20 @@
+#include <inttypes.h>
+#include <unistd.h>
+#include <stdio.h>
+#include <stdint.h>
+#include <stddef.h>
+#include <sys/socket.h>
+#include <sys/un.h>
+
+#include "sanlock_internal.h"
+#include "sanlock_sock.h"
+
+int sanlock_socket_address(struct sockaddr_un *addr)
+{
+ memset(addr, 0, sizeof(struct sockaddr_un));
+ addr->sun_family = AF_LOCAL;
+ snprintf(addr->sun_path, sizeof(addr->sun_path) - 1, "%s/%s",
+ SANLK_RUN_DIR, SANLK_SOCKET_NAME);
+ return 0;
+}
+
diff --git a/src/sanlock_sock.h b/src/sanlock_sock.h
new file mode 100644
index 0000000..154665a
--- /dev/null
+++ b/src/sanlock_sock.h
@@ -0,0 +1,46 @@
+/*
+ * Copyright (C) 2010-2011 Red Hat, Inc. All rights reserved.
+ *
+ * This copyrighted material is made available to anyone wishing to use,
+ * modify, copy, or redistribute it subject to the terms and conditions
+ * of the GNU General Public License v.2.
+ */
+
+#ifndef __SANLOCK_SOCK_H__
+#define __SANLOCK_SOCK_H__
+
+#define MAX_CLIENT_MSG (1024 * 1024) /* TODO: this is random */
+
+enum {
+ SM_CMD_REGISTER = 1,
+ SM_CMD_ADD_LOCKSPACE = 2,
+ SM_CMD_REM_LOCKSPACE = 3,
+ SM_CMD_SHUTDOWN = 4,
+ SM_CMD_STATUS = 5,
+ SM_CMD_LOG_DUMP = 6,
+ SM_CMD_ACQUIRE = 7,
+ SM_CMD_RELEASE = 8,
+ SM_CMD_INQUIRE = 9,
+ SM_CMD_RESTRICT = 10,
+};
+
+#define SANLK_STATE_MAXSTR 4096
+
+#define SANLK_STATE_DAEMON 1
+#define SANLK_STATE_LOCKSPACE 2
+#define SANLK_STATE_CLIENT 3
+#define SANLK_STATE_RESOURCE 4
+
+struct sanlk_state {
+ uint32_t type; /* SANLK_STATE_ */
+ uint32_t flags;
+ uint32_t data32; /* pid (for client) */
+ uint64_t data64;
+ char name[SANLK_NAME_LEN]; /* client name or resource name */
+ uint32_t str_len;
+ char str[0]; /* string of internal state */
+};
+
+int sanlock_socket_address(struct sockaddr_un *addr);
+
+#endif
commit f85b7c11453129fec7819128f6dd340b8e531ace
Author: David Teigland <teigland(a)redhat.com>
Date: Wed Aug 3 10:41:50 2011 -0500
wdmd: new build flags
and move socket address function into a c
file that both lib and cmd can share
diff --git a/wdmd/Makefile b/wdmd/Makefile
index bc30c32..be3f030 100644
--- a/wdmd/Makefile
+++ b/wdmd/Makefile
@@ -15,9 +15,9 @@ SOMINOR=0
SHLIB_TARGET = $(LIB_TARGET).so.$(SOMAJOR).$(SOMINOR)
-CMD_SOURCE = main.c
+CMD_SOURCE = main.c wdmd_sock.c
-LIB_SOURCE = client.c
+LIB_SOURCE = client.c wdmd_sock.c
TEST_SOURCE = wdmd_client.c
@@ -39,16 +39,21 @@ CFLAGS += -D_GNU_SOURCE -g \
-Wp,-D_FORTIFY_SOURCE=2 \
-fexceptions \
-fasynchronous-unwind-tables \
- -fdiagnostics-show-option
+ -fdiagnostics-show-option \
+ -fPIE -DPIE
+
+CMD_LDFLAGS += -lwdmd
+CMD_LDFLAGS += -Wl,-z,now -Wl,-z,relro -pie
+
+LIB_LDFLAGS += -Wl,-z,relro -pie
-CMD_LDFLAGS = -lwdmd
TEST_LDFLAGS = -lwdmd
all: $(SHLIB_TARGET) $(CMD_TARGET) $(TEST_TARGET)
$(SHLIB_TARGET): $(LIB_SOURCE)
- $(CC) -shared -fPIC -o $@ -Wl,-soname=$(LIB_TARGET).so.$(SOMAJOR) $^
+ $(CC) $(CFLAGS) $(LIB_LDFLAGS) -shared -fPIC -o $@ -Wl,-soname=$(LIB_TARGET).so.$(SOMAJOR) $^
ln -sf $(SHLIB_TARGET) $(LIB_TARGET).so
ln -sf $(SHLIB_TARGET) $(LIB_TARGET).so.$(SOMAJOR)
@@ -68,16 +73,17 @@ DESTDIR=
BINDIR=/usr/sbin
LIBDIR=/usr/lib64
HEADIR=/usr/include
-MANDIR=/usr/share/man/
+MANDIR=/usr/share/man
.PHONY: install
install: all
- mkdir -p $(DESTDIR)/$(BINDIR)
- mkdir -p $(DESTDIR)/$(LIBDIR)
- mkdir -p $(DESTDIR)/$(HEADIR)
+ $(INSTALL) -d $(DESTDIR)/$(BINDIR)
+ $(INSTALL) -d $(DESTDIR)/$(LIBDIR)
+ $(INSTALL) -d $(DESTDIR)/$(HEADIR)
+ $(INSTALL) -d $(DESTDIR)/$(MANDIR)
$(INSTALL) -c -m 755 $(CMD_TARGET) $(DESTDIR)/$(BINDIR)
$(INSTALL) -c -m 755 $(SHLIB_TARGET) $(DESTDIR)/$(LIBDIR)
cp -a $(LIB_TARGET).so $(DESTDIR)/$(LIBDIR)
cp -a $(LIB_TARGET).so.$(SOMAJOR) $(DESTDIR)/$(LIBDIR)
$(INSTALL) -c -m 644 $(HEADER_TARGET) $(DESTDIR)/$(HEADIR)
- install $(MAN_TARGET) ${DESTDIR}/${MANDIR}/man8
+ $(INSTALL) $(MAN_TARGET) ${DESTDIR}/${MANDIR}/man8
diff --git a/wdmd/client.c b/wdmd/client.c
index 6a459ae..95b60de 100644
--- a/wdmd/client.c
+++ b/wdmd/client.c
@@ -21,38 +21,7 @@
#include "wdmd.h"
#include "wdmd_internal.h"
-
-int get_socket_address(struct sockaddr_un *addr)
-{
- memset(addr, 0, sizeof(struct sockaddr_un));
- addr->sun_family = AF_LOCAL;
- snprintf(addr->sun_path, sizeof(addr->sun_path) - 1, "%s/%s",
- WDMD_RUN_DIR, WDMD_SOCKET_NAME);
- return 0;
-}
-
-static int connect_socket(int *sock_fd)
-{
- int rv, s;
- struct sockaddr_un addr;
-
- s = socket(AF_LOCAL, SOCK_STREAM, 0);
- if (s < 0)
- return -errno;
-
- rv = get_socket_address(&addr);
- if (rv < 0)
- return rv;
-
- rv = connect(s, (struct sockaddr *) &addr, sizeof(struct sockaddr_un));
- if (rv < 0) {
- rv = -errno;
- close(s);
- return rv;
- }
- *sock_fd = s;
- return 0;
-}
+#include "wdmd_sock.h"
int wdmd_connect(void)
{
@@ -63,7 +32,7 @@ int wdmd_connect(void)
if (s < 0)
return -errno;
- rv = get_socket_address(&addr);
+ rv = wdmd_socket_address(&addr);
if (rv < 0)
return rv;
diff --git a/wdmd/main.c b/wdmd/main.c
index 25c6781..d8994b4 100644
--- a/wdmd/main.c
+++ b/wdmd/main.c
@@ -34,6 +34,7 @@
#include "wdmd.h"
#include "wdmd_internal.h"
+#include "wdmd_sock.h"
#define DEFAULT_TEST_INTERVAL 10
#define DEFAULT_FIRE_TIMEOUT 60
@@ -284,8 +285,6 @@ static void close_clients(void)
{
}
-extern int get_socket_address(struct sockaddr_un *addr);
-
static int setup_listener_socket(int *listener_socket)
{
int rv, s;
@@ -295,7 +294,7 @@ static int setup_listener_socket(int *listener_socket)
if (s < 0)
return -errno;
- rv = get_socket_address(&addr);
+ rv = wdmd_socket_address(&addr);
if (rv < 0)
return rv;
diff --git a/wdmd/wdmd_sock.c b/wdmd/wdmd_sock.c
new file mode 100644
index 0000000..ef2923e
--- /dev/null
+++ b/wdmd/wdmd_sock.c
@@ -0,0 +1,21 @@
+#include <inttypes.h>
+#include <unistd.h>
+#include <stdio.h>
+#include <stdint.h>
+#include <stddef.h>
+#include <sys/socket.h>
+#include <sys/un.h>
+
+#include "wdmd.h"
+#include "wdmd_internal.h"
+#include "wdmd_sock.h"
+
+int wdmd_socket_address(struct sockaddr_un *addr)
+{
+ memset(addr, 0, sizeof(struct sockaddr_un));
+ addr->sun_family = AF_LOCAL;
+ snprintf(addr->sun_path, sizeof(addr->sun_path) - 1, "%s/%s",
+ WDMD_RUN_DIR, WDMD_SOCKET_NAME);
+ return 0;
+}
+
diff --git a/wdmd/wdmd_sock.h b/wdmd/wdmd_sock.h
new file mode 100644
index 0000000..cf2f4b1
--- /dev/null
+++ b/wdmd/wdmd_sock.h
@@ -0,0 +1,6 @@
+#ifndef __WDMD_SOCK_H__
+#define __WDMD_SOCK_H__
+
+int wdmd_socket_address(struct sockaddr_un *addr);
+
+#endif
12 years, 8 months
12 commits - init.d/sanlock init.d/wdmd src/Makefile src/client_msg.c src/lockfile.c src/main.c src/sanlock.8 src/sanlock_internal.h wdmd/Makefile wdmd/client.c wdmd/main.c wdmd/wdmd.8 wdmd/wdmd_internal.h
by David Teigland
init.d/sanlock | 8 +++
init.d/wdmd | 8 +++
src/Makefile | 30 +++++++------
src/client_msg.c | 7 ---
src/lockfile.c | 10 ++++
src/main.c | 22 ---------
src/sanlock.8 | 112 +++++++++++++++++++++++++++++++++++++++++++++++++
src/sanlock_internal.h | 2
wdmd/Makefile | 34 +++++++-------
wdmd/client.c | 7 ---
wdmd/main.c | 61 ++++++++++++++++++++++----
wdmd/wdmd.8 | 74 ++++++++++++++++++++++++++++++++
wdmd/wdmd_internal.h | 2
13 files changed, 301 insertions(+), 76 deletions(-)
New commits:
commit aebd65aa012ab07234e83a784902927852ddb741
Author: David Teigland <teigland(a)redhat.com>
Date: Tue Aug 2 16:24:56 2011 -0500
sanlock/wdmd: improve mkdir of run dir
suggested by Dan Walsh
diff --git a/init.d/sanlock b/init.d/sanlock
index e245e98..74df2c9 100644
--- a/init.d/sanlock
+++ b/init.d/sanlock
@@ -29,8 +29,13 @@ exec="/usr/sbin/$prog"
start() {
[ -x $exec ] || exit 5
+
+ if [ ! -d /var/run/$prog ]; then
+ mkdir -p /var/run/$prog
+ [ -x /usr/sbin/restorecon ] && restorecon /var/run/$prog
+ fi
+
echo -n $"Starting $prog: "
- mkdir -p /var/run/$prog
daemon $prog daemon $SANLOCKOPTS
retval=$?
echo
diff --git a/init.d/wdmd b/init.d/wdmd
index 9b7c2b7..1adb03a 100644
--- a/init.d/wdmd
+++ b/init.d/wdmd
@@ -27,8 +27,13 @@ exec="/usr/sbin/$prog"
start() {
[ -x $exec ] || exit 5
+
+ if [ ! -d /var/run/$prog ]; then
+ mkdir -p /var/run/$prog
+ [ -x /usr/sbin/restorecon ] && restorecon /var/run/$prog
+ fi
+
echo -n $"Starting $prog: "
- mkdir -p /var/run/$prog
daemon $prog
retval=$?
echo
commit 0195f36b4069e0e4bcf158cfd4b0190b4f73a40f
Author: David Teigland <teigland(a)redhat.com>
Date: Tue Aug 2 16:16:42 2011 -0500
sanlock: add man page
diff --git a/src/Makefile b/src/Makefile
index 7f703ac..66d5fcd 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -5,14 +5,12 @@
# of the GNU General Public License v.2.
CMD_TARGET = sanlock
-
LIB_TARGET = libsanlock
-
HEADER_TARGET = sanlock.h sanlock_rv.h sanlock_resource.h sanlock_admin.h sanlock_direct.h
+MAN_TARGET = sanlock.8
SOMAJOR=1
SOMINOR=0
-
SHLIB_TARGET = $(LIB_TARGET).so.$(SOMAJOR).$(SOMINOR)
CMD_SOURCE = \
@@ -81,16 +79,20 @@ clean:
INSTALL=$(shell which install)
-CMD_BINDIR=/usr/sbin
-LIB_LIBDIR=/usr/lib64
-HEADER_DIR=/usr/include
+DESTDIR=
+BINDIR=/usr/sbin
+LIBDIR=/usr/lib64
+HEADIR=/usr/include
+MANDIR=/usr/share/man
.PHONY: install
install: all
- mkdir -p $(DESTDIR)/$(CMD_BINDIR)
- mkdir -p $(DESTDIR)/$(LIB_LIBDIR)
- mkdir -p $(DESTDIR)/$(HEADER_DIR)
- $(INSTALL) -c -m 755 $(CMD_TARGET) $(DESTDIR)/$(CMD_BINDIR)
- $(INSTALL) -c -m 755 $(SHLIB_TARGET) $(DESTDIR)/$(LIB_LIBDIR)
- cp -a $(LIB_TARGET).so $(DESTDIR)/$(LIB_LIBDIR)
- $(INSTALL) -c -m 644 $(HEADER_TARGET) $(DESTDIR)/$(HEADER_DIR)
+ $(INSTALL) -d $(DESTDIR)/$(BINDIR)
+ $(INSTALL) -d $(DESTDIR)/$(LIBDIR)
+ $(INSTALL) -d $(DESTDIR)/$(HEADIR)
+ $(INSTALL) -d $(DESTDIR)/$(MANDIR)
+ $(INSTALL) -c -m 755 $(CMD_TARGET) $(DESTDIR)/$(BINDIR)
+ $(INSTALL) -c -m 755 $(SHLIB_TARGET) $(DESTDIR)/$(LIBDIR)
+ cp -a $(LIB_TARGET).so $(DESTDIR)/$(LIBDIR)
+ $(INSTALL) -c -m 644 $(HEADER_TARGET) $(DESTDIR)/$(HEADIR)
+ $(INSTALL) $(MAN_TARGET) ${DESTDIR}/${MANDIR}/man8
diff --git a/src/sanlock.8 b/src/sanlock.8
new file mode 100644
index 0000000..b7405cf
--- /dev/null
+++ b/src/sanlock.8
@@ -0,0 +1,112 @@
+.TH SANLOCK 8 2011-08-02
+
+.SH NAME
+sanlock \- shared storage lock manager
+
+.SH SYNOPSIS
+.B sanlock
+[COMMAND] [ACTION] [OPTIONS]
+
+.SH DESCRIPTION
+
+The sanlock daemon manages leases for applications running on a cluster of
+hosts with shared storage. All lease management and coordination is done
+through reading and writing blocks on the shared storage. Two basic types
+of leases are used, each based on a different algorithm:
+
+"delta leases" are slow to acquire and demand regular i/o to shared
+storage. sanlock only uses them internally to hold a lease on its
+"host_id" (an integer host identifier from 1-2000). They simply prevent
+two hosts from using the same host identifier. They also serve in a
+limited way as host "liveness" indicators since they are renewed.
+
+"paxos leases" are generally fast to acquire and sanlock makes them
+available to applications as "general purpose" resource leases. The disk
+paxos algorithm uses host_id's internally, and does not work if different
+hosts use the same host_id. So the delta leases provide unique host_id's
+for implementing paxos leases.
+
+Before sanlock can be used, the user must assign each host an identifier,
+or "host_id", which is an integer value between 1 and 2000. Two hosts
+should not be given the same host_id.
+
+sanlock views a pool of storage as a "lockspace". Each distinct pool of
+storage, e.g. from different sources, would typically be defined as a
+separate lockspace, with a separate lockspace name.
+
+A part of this storage space must be reserved and initialized for sanlock
+to store delta leases. Each host that wants to use the lockspace must
+first acquire a delta lease on its host_id number within the lockspace.
+(See the add_lockspace action/api.) These delta leases prevent two hosts
+from using the same host_id, and allow hosts to detect which other hosts
+are currently using the lockspace. The space required for 2000 delta
+leases in the lockspace (for all possible host_id's) is 1GB for storage
+with 512 byte sectors, or 8GB for storage with 4k sectors.
+
+Example:
+
+1. Create storage pools and reserve and initialize host_id leases
+.br
+two different LUNs on a SAN: /dev/sdb, /dev/sdc
+.br
+vgcreate pool1 /dev/sdb
+.br
+vgcreate pool2 /dev/sdc
+.br
+lvcreate -n hostid_leases -L 1GB pool1
+.br
+lvcreate -n hostid_leases -L 1GB pool2
+.br
+sanlock direct init -s lockspace1:0:/dev/pool1/hostid_leases:0
+.br
+sanlock direct init -s lockspace2:0:/dev/pool2/hostid_leases:0
+.br
+
+2. Start the sanlock daemon on each host
+.br
+sanlock daemon
+.br
+
+3. Add each lockspace to be used
+.br
+host1:
+.br
+sanlock client add_lockspace -s lockspace1:1:/dev/pool1/hostid_leases:0
+.br
+sanlock client add_lockspace -s lockspace2:1:/dev/pool2/hostid_leases:0
+.br
+host2:
+.br
+sanlock client add_lockspace -s lockspace1:2:/dev/pool1/hostid_leases:0
+.br
+sanlock client add_lockspace -s lockspace2:2:/dev/pool2/hostid_leases:0
+.br
+
+4. Applications can now reserve/initialize space for resource leases, and
+then acquire the leases as they need to access the resources.
+
+
+
+sanlock is a multi purpose binary, with three top level commands:
+
+.TP
+.B daemon
+start daemon
+
+.TP
+.B client
+send request to daemon (default command if none given)
+
+.TP
+.B direct
+access storage directly (no coordination with daemon)
+
+.SS client actions
+
+.SS direct actions
+
+.SH OPTIONS
+
+.SH SEE ALSO
+.BR wdmd (8)
+
commit 77817f3c61b136ac972e041b4adfe7efc28cfa04
Author: David Teigland <teigland(a)redhat.com>
Date: Tue Aug 2 14:39:38 2011 -0500
sanlock: pid and sock file changes
/var/run/sanlock/sanlock.pid
/var/run/sanlock/sanlock.sock (. instead of _)
and mkdir /var/run/sanlock from init script also
diff --git a/init.d/sanlock b/init.d/sanlock
index d46dcef..e245e98 100644
--- a/init.d/sanlock
+++ b/init.d/sanlock
@@ -22,7 +22,7 @@
. /etc/rc.d/init.d/functions
prog="sanlock"
-lockfile="/var/run/sanlock/$prog.pid"
+lockfile="/var/run/$prog/$prog.pid"
exec="/usr/sbin/$prog"
[ -f /etc/sysconfig/$prog ] && . /etc/sysconfig/$prog
@@ -30,6 +30,7 @@ exec="/usr/sbin/$prog"
start() {
[ -x $exec ] || exit 5
echo -n $"Starting $prog: "
+ mkdir -p /var/run/$prog
daemon $prog daemon $SANLOCKOPTS
retval=$?
echo
diff --git a/src/lockfile.c b/src/lockfile.c
index 034fc45..860c4de 100644
--- a/src/lockfile.c
+++ b/src/lockfile.c
@@ -20,6 +20,7 @@
#include <syslog.h>
#include <sys/types.h>
#include <sys/time.h>
+#include <sys/stat.h>
#include "sanlock_internal.h"
#include "log.h"
@@ -30,8 +31,17 @@ int lockfile(const char *dir, const char *name)
char path[PATH_MAX];
char buf[16];
struct flock lock;
+ mode_t old_umask;
int fd, rv;
+ old_umask = umask(0022);
+ rv = mkdir(SANLK_RUN_DIR, 0777);
+ if (rv < 0 && errno != EEXIST) {
+ umask(old_umask);
+ return rv;
+ }
+ umask(old_umask);
+
snprintf(path, PATH_MAX, "%s/%s", dir, name);
fd = open(path, O_CREAT|O_WRONLY|O_CLOEXEC, 0666);
diff --git a/src/main.c b/src/main.c
index 0754cba..309d242 100644
--- a/src/main.c
+++ b/src/main.c
@@ -2063,22 +2063,6 @@ static void sigterm_handler(int sig GNUC_UNUSED)
external_shutdown = 1;
}
-static int make_dirs(void)
-{
- mode_t old_umask;
- int rv;
-
- old_umask = umask(0022);
- rv = mkdir(SANLK_RUN_DIR, 0777);
- if (rv < 0 && errno != EEXIST)
- goto out;
-
- rv = 0;
- out:
- umask(old_umask);
- return rv;
-}
-
static void setup_priority(void)
{
struct sched_param sched_param;
@@ -2139,10 +2123,6 @@ static int do_daemon(void)
if (rv < 0)
return rv;
- rv = make_dirs();
- if (rv < 0)
- return rv;
-
fd = lockfile(SANLK_RUN_DIR, SANLK_LOCKFILE_NAME);
if (fd < 0)
return fd;
diff --git a/src/sanlock_internal.h b/src/sanlock_internal.h
index cb7cf60..b25a8eb 100644
--- a/src/sanlock_internal.h
+++ b/src/sanlock_internal.h
@@ -47,7 +47,7 @@
#define SANLK_RUN_DIR "/var/run/sanlock"
#define SANLK_LOG_DIR "/var/log"
-#define SANLK_SOCKET_NAME "sanlock_sock"
+#define SANLK_SOCKET_NAME "sanlock.sock"
#define SANLK_LOGFILE_NAME "sanlock.log"
#define SANLK_LOCKFILE_NAME "sanlock.pid"
commit a784a2a1cc19290454c6ee27dfe5a9cac1545c19
Author: David Teigland <teigland(a)redhat.com>
Date: Tue Aug 2 14:28:43 2011 -0500
sanlock: add explicit -luuid
diff --git a/src/Makefile b/src/Makefile
index 9491ce1..7f703ac 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -61,7 +61,7 @@ CFLAGS += -D_GNU_SOURCE -g \
-fasynchronous-unwind-tables \
-fdiagnostics-show-option
-CMD_LDFLAGS = -lpthread -lrt -laio -lblkid -lsanlock -L../wdmd -lwdmd
+CMD_LDFLAGS = -lpthread -luuid -lrt -laio -lblkid -lsanlock -L../wdmd -lwdmd
LIB_LDFLAGS = -lpthread -lrt -laio -lblkid -L../wdmd -lwdmd
commit d5c963a4e71714fa17ad7e216df50aff92258dfe
Author: David Teigland <teigland(a)redhat.com>
Date: Tue Aug 2 14:21:44 2011 -0500
sanlock: tidy sun_addr snprintf
diff --git a/src/client_msg.c b/src/client_msg.c
index 11d5ab5..28c0374 100644
--- a/src/client_msg.c
+++ b/src/client_msg.c
@@ -33,13 +33,10 @@
static int get_socket_address(struct sockaddr_un *addr)
{
- char path[PATH_MAX];
-
- snprintf(path, PATH_MAX, "%s/%s", SANLK_RUN_DIR, SANLK_SOCKET_NAME);
-
memset(addr, 0, sizeof(struct sockaddr_un));
addr->sun_family = AF_LOCAL;
- strncpy(addr->sun_path, path, sizeof(addr->sun_path) - 1);
+ snprintf(addr->sun_path, sizeof(addr->sun_path) - 1, "%s/%s",
+ SANLK_RUN_DIR, SANLK_SOCKET_NAME);
return 0;
}
commit 3b42a648ac1a1e1e041bb33995c893a7210c9d19
Author: David Teigland <teigland(a)redhat.com>
Date: Tue Aug 2 14:19:40 2011 -0500
sanlock: use accept4 with SOCK_NONBLOCK
to avoid theoretical cases where accept could block
diff --git a/src/main.c b/src/main.c
index 667149b..0754cba 100644
--- a/src/main.c
+++ b/src/main.c
@@ -2033,7 +2033,7 @@ static void process_listener(int ci GNUC_UNUSED)
int fd;
int on = 1;
- fd = accept(client[ci].fd, NULL, NULL);
+ fd = accept4(client[ci].fd, NULL, NULL, SOCK_NONBLOCK);
if (fd < 0)
return;
commit f37fa70ff24c5aa87012ce0f4d4a54bd086c58e6
Author: David Teigland <teigland(a)redhat.com>
Date: Tue Aug 2 14:15:53 2011 -0500
wdmd: disable test scripts
until we're ready to really test and use it
diff --git a/wdmd/Makefile b/wdmd/Makefile
index a35bcfb..bc30c32 100644
--- a/wdmd/Makefile
+++ b/wdmd/Makefile
@@ -39,8 +39,7 @@ CFLAGS += -D_GNU_SOURCE -g \
-Wp,-D_FORTIFY_SOURCE=2 \
-fexceptions \
-fasynchronous-unwind-tables \
- -fdiagnostics-show-option \
- -DTEST_SCRIPTS
+ -fdiagnostics-show-option
CMD_LDFLAGS = -lwdmd
TEST_LDFLAGS = -lwdmd
commit f6a95179d9f73e11070bd8667226944c2b6f567b
Author: David Teigland <teigland(a)redhat.com>
Date: Tue Aug 2 14:02:34 2011 -0500
wdmd: add man page
diff --git a/wdmd/Makefile b/wdmd/Makefile
index 65d767f..a35bcfb 100644
--- a/wdmd/Makefile
+++ b/wdmd/Makefile
@@ -5,17 +5,15 @@
# of the GNU General Public License v.2.
CMD_TARGET = wdmd
-
LIB_TARGET = libwdmd
-
HEADER_TARGET = wdmd.h
+MAN_TARGET = wdmd.8
+TEST_TARGET = wdmd_client
SOMAJOR=1
SOMINOR=0
-
SHLIB_TARGET = $(LIB_TARGET).so.$(SOMAJOR).$(SOMINOR)
-TEST_TARGET = wdmd_client
CMD_SOURCE = main.c
@@ -67,17 +65,20 @@ clean:
INSTALL=$(shell which install)
-CMD_BINDIR=/usr/sbin
-LIB_LIBDIR=/usr/lib64
-HEADER_DIR=/usr/include
+DESTDIR=
+BINDIR=/usr/sbin
+LIBDIR=/usr/lib64
+HEADIR=/usr/include
+MANDIR=/usr/share/man/
.PHONY: install
install: all
- mkdir -p $(DESTDIR)/$(CMD_BINDIR)
- mkdir -p $(DESTDIR)/$(LIB_LIBDIR)
- mkdir -p $(DESTDIR)/$(HEADER_DIR)
- $(INSTALL) -c -m 755 $(CMD_TARGET) $(DESTDIR)/$(CMD_BINDIR)
- $(INSTALL) -c -m 755 $(SHLIB_TARGET) $(DESTDIR)/$(LIB_LIBDIR)
- cp -a $(LIB_TARGET).so $(DESTDIR)/$(LIB_LIBDIR)
- cp -a $(LIB_TARGET).so.$(SOMAJOR) $(DESTDIR)/$(LIB_LIBDIR)
- $(INSTALL) -c -m 644 $(HEADER_TARGET) $(DESTDIR)/$(HEADER_DIR)
+ mkdir -p $(DESTDIR)/$(BINDIR)
+ mkdir -p $(DESTDIR)/$(LIBDIR)
+ mkdir -p $(DESTDIR)/$(HEADIR)
+ $(INSTALL) -c -m 755 $(CMD_TARGET) $(DESTDIR)/$(BINDIR)
+ $(INSTALL) -c -m 755 $(SHLIB_TARGET) $(DESTDIR)/$(LIBDIR)
+ cp -a $(LIB_TARGET).so $(DESTDIR)/$(LIBDIR)
+ cp -a $(LIB_TARGET).so.$(SOMAJOR) $(DESTDIR)/$(LIBDIR)
+ $(INSTALL) -c -m 644 $(HEADER_TARGET) $(DESTDIR)/$(HEADIR)
+ install $(MAN_TARGET) ${DESTDIR}/${MANDIR}/man8
diff --git a/wdmd/wdmd.8 b/wdmd/wdmd.8
new file mode 100644
index 0000000..2df7297
--- /dev/null
+++ b/wdmd/wdmd.8
@@ -0,0 +1,74 @@
+.TH WDMD 8 2011-08-01
+
+.SH NAME
+wdmd \- watchdog multiplexing daemon
+
+.SH SYNOPSIS
+.B wdmd
+[OPTIONS]
+
+.SH DESCRIPTION
+
+This daemon opens /dev/watchdog and allows multiple independent sources to
+detmermine whether each KEEPALIVE is done. Every test interval (10
+seconds), the daemon tests each source. If any test fails, the KEEPALIVE
+is not done. In a standard configuration, the watchdog timer will reset
+the system if no KEEPALIVE is done for 60 seconds ("fire timeout"). This
+means that if single test fails 5-6 times in row, the watchdog will fire
+and reset the system. With multiple test sources, fewer separate failures
+back to back can also cause a reset, e.g.
+
+T seconds, P pass, F fail
+.br
+T00: test1 P, test2 P, test3 P: KEEPALIVE done
+.br
+T10: test1 F, test2 F, test3 P: KEEPALIVE skipped
+.br
+T20: test1 F, test2 P, test3 P: KEEPALIVE skipped
+.br
+T30: test1 P, test2 F, test3 P: KEEPALIVE skipped
+.br
+T40: test1 P, test2 P, test3 F: KEEPALIVE skipped
+.br
+T50: test1 F, test2 F, test3 P: KEEPALIVE skipped
+.br
+T60: test1 P, test2 F, test3 P: KEEPALIVE skipped
+.br
+T60: watchdog fires, system resets
+
+(Depending on timings, the system may be reset sometime shortly before
+T60, and the tests at T60 would not be run.)
+
+A crucial aspect to the design and function of wdmd is that if any single
+source does not pass tests for the fire timeout, the watchdog is
+guaranteed to fire, regardless of whether other sources on the system have
+passed or failed. A spurious reset due to the combined effects of
+multiple failing tests as shown above, is an accepted side effect.
+
+wdmd will exit if a watchdog driver is not loaded.
+
+wdmd cannot be used on the system with any other program that needs to
+open /dev/watchdog, e.g. watchdog(8).
+
+.SS Test Source: clients
+
+Using libwdmd, programs connect to wdmd via a unix socket, and send
+regular messages to wdmd to update an expiry time for their connection.
+Every test interval, wdmd will check if the expiry time for a connection
+has been reached. If so, the test for that client fails.
+
+(Other test sources such as scripts executed each test interval may be
+added in the future.)
+
+.SH OPTIONS
+.TP
+.B \-D
+Enable debugging to stderr and don't fork.
+
+.TP
+.BI \-H " num"
+Enable (1) or disable (0) high priority features such as realtime
+scheduling priority and mlockall.
+.br
+Default 1.
+
commit f27d35af2e2e065c214b19d1aca3244ea203be00
Author: David Teigland <teigland(a)redhat.com>
Date: Tue Aug 2 13:01:45 2011 -0500
wdmd: pid and sock file changes
/var/run/wdmd/wdmd.pid
/var/run/wdmd/wdmd.sock (. instead of _)
and mkdir /var/run/wdmd from init script also
diff --git a/init.d/wdmd b/init.d/wdmd
index c6902ac..9b7c2b7 100644
--- a/init.d/wdmd
+++ b/init.d/wdmd
@@ -22,12 +22,13 @@
. /etc/rc.d/init.d/functions
prog="wdmd"
-lockfile="/var/run/wdmd/$prog.pid"
+lockfile="/var/run/$prog/$prog.pid"
exec="/usr/sbin/$prog"
start() {
[ -x $exec ] || exit 5
echo -n $"Starting $prog: "
+ mkdir -p /var/run/$prog
daemon $prog
retval=$?
echo
diff --git a/wdmd/main.c b/wdmd/main.c
index 412be83..25c6781 100644
--- a/wdmd/main.c
+++ b/wdmd/main.c
@@ -48,10 +48,6 @@ static time_t last_keepalive;
static char lockfile_path[PATH_MAX];
static int dev_fd;
-#define WDMD_RUN_DIR "/var/run/wdmd"
-#define FILES_DIR "/var/run/wdmd/test_files"
-#define SCRIPTS_DIR "/etc/wdmd/test_scripts"
-
struct script_status {
int pid;
char path[PATH_MAX];
@@ -379,6 +375,7 @@ static int active_clients(void)
#ifdef TEST_FILES
+#define FILES_DIR "/var/run/wdmd/test_files"
const char *files_built = " files";
static DIR *files_dir;
@@ -464,6 +461,7 @@ static int test_files(void) { return 0; }
#ifdef TEST_SCRIPTS
+#define SCRIPTS_DIR "/etc/wdmd/test_scripts"
static DIR *scripts_dir;
const char *scripts_built = " scripts";
@@ -781,8 +779,10 @@ static int lockfile(void)
old_umask = umask(0022);
rv = mkdir(WDMD_RUN_DIR, 0777);
- if (rv < 0 && errno != EEXIST)
+ if (rv < 0 && errno != EEXIST) {
+ umask(old_umask);
return rv;
+ }
umask(old_umask);
sprintf(lockfile_path, "%s/wdmd.pid", WDMD_RUN_DIR);
diff --git a/wdmd/wdmd_internal.h b/wdmd/wdmd_internal.h
index 4ace3cf..584b833 100644
--- a/wdmd/wdmd_internal.h
+++ b/wdmd/wdmd_internal.h
@@ -14,7 +14,7 @@
#endif
#define WDMD_RUN_DIR "/var/run/wdmd"
-#define WDMD_SOCKET_NAME "wdmd_sock"
+#define WDMD_SOCKET_NAME "wdmd.sock"
enum {
CMD_REGISTER = 1,
commit 516c0384627466c1454affdfe997e53fad7faeaf
Author: David Teigland <teigland(a)redhat.com>
Date: Tue Aug 2 12:40:43 2011 -0500
wdmd: tidy sun_addr snprintf
diff --git a/wdmd/client.c b/wdmd/client.c
index b46da96..6a459ae 100644
--- a/wdmd/client.c
+++ b/wdmd/client.c
@@ -24,13 +24,10 @@
int get_socket_address(struct sockaddr_un *addr)
{
- char path[PATH_MAX];
-
- snprintf(path, PATH_MAX, "%s/%s", WDMD_RUN_DIR, WDMD_SOCKET_NAME);
-
memset(addr, 0, sizeof(struct sockaddr_un));
addr->sun_family = AF_LOCAL;
- strncpy(addr->sun_path, path, sizeof(addr->sun_path) - 1);
+ snprintf(addr->sun_path, sizeof(addr->sun_path) - 1, "%s/%s",
+ WDMD_RUN_DIR, WDMD_SOCKET_NAME);
return 0;
}
commit 273387bded88dd2c78d7777cf1d023c3150d7e4e
Author: David Teigland <teigland(a)redhat.com>
Date: Tue Aug 2 12:32:45 2011 -0500
wdmd: use accept4 with SOCK_NONBLOCK
to avoid theoretical cases where accept could block
diff --git a/wdmd/main.c b/wdmd/main.c
index 82c170d..412be83 100644
--- a/wdmd/main.c
+++ b/wdmd/main.c
@@ -275,9 +275,9 @@ static void process_listener(int ci)
int fd;
int on = 1;
- fd = accept(client[ci].fd, NULL, NULL);
+ fd = accept4(client[ci].fd, NULL, NULL, SOCK_NONBLOCK);
if (fd < 0)
- return;
+ return;
setsockopt(fd, SOL_SOCKET, SO_PASSCRED, &on, sizeof(on));
commit 2f05b4fbedfa04e0cdaed004eec74375dd178548
Author: David Teigland <teigland(a)redhat.com>
Date: Mon Aug 1 12:56:31 2011 -0500
wdmd: add option for high priority
-H 0 to disable high priority features
diff --git a/wdmd/main.c b/wdmd/main.c
index e024c82..82c170d 100644
--- a/wdmd/main.c
+++ b/wdmd/main.c
@@ -863,6 +863,8 @@ static void print_usage(void)
printf("version print version\n");
printf("help print usage\n");
printf("-D debug: no fork and print all logging to stderr\n");
+ printf("-H <num> use high priority features (1 yes, 0 no, default %d)\n",
+ DEFAULT_HIGH_PRIORITY);
}
/* If wdmd exits abnormally, /dev/watchdog will eventually fire, and clients
@@ -877,11 +879,13 @@ static void print_usage(void)
int main(int argc, char *argv[])
{
- int rv;
+ char optchar;
+ char *optionarg;
+ char *p;
+ int i, rv;
/*
- * TODO: real option parsing, including
- * -h <num> use high priority features (1 yes, 0 no, default 1)
+ * TODO:
* -c <num> enable test clients (1 yes, 0 no, default ...)
* -s <num> enable test scripts (1 yes, 0 no, default ...)
* -f <num> enable test files (1 yes, 0 no, default ...)
@@ -902,6 +906,43 @@ int main(int argc, char *argv[])
return 0;
}
+ for (i = 1; i < argc; ) {
+ p = argv[i];
+
+ if ((p[0] != '-') || (strlen(p) != 2)) {
+ fprintf(stderr, "unknown option %s\n", p);
+ fprintf(stderr, "space required before option value\n");
+ exit(EXIT_FAILURE);
+ }
+
+ optchar = p[1];
+ i++;
+
+ /* the only option that does not have optionarg */
+ if (optchar == 'D') {
+ daemon_debug = 1;
+ continue;
+ }
+
+ if (i >= argc) {
+ fprintf(stderr, "option '%c' requires arg\n", optchar);
+ exit(EXIT_FAILURE);
+ }
+
+ optionarg = argv[i];
+
+ switch (optchar) {
+ case 'H':
+ high_priority = atoi(optionarg);
+ break;
+ default:
+ fprintf(stderr, "unknown option: %c\n", optchar);
+ exit(EXIT_FAILURE);
+ }
+
+ i++;
+ }
+
if ((argc > 1) &&
!strcmp(argv[1], "-D")) {
daemon_debug = 1;
12 years, 8 months