28 files changed, 1691 insertions, 445 deletions

diff --git a/sbin/hastd/Makefile b/sbin/hastd/Makefile
index 0b0721e5b63b..588e1fa0bd36 100644
--- a/sbin/hastd/Makefile
+++ b/sbin/hastd/Makefile
@@ -5,7 +5,7 @@
 PROG=	hastd
 SRCS=	activemap.c
 SRCS+=	control.c
-SRCS+=	ebuf.c
+SRCS+=	ebuf.c event.c
 SRCS+=	hast_proto.c hastd.c hooks.c
 SRCS+=	metadata.c
 SRCS+=	nv.c
diff --git a/sbin/hastd/control.c b/sbin/hastd/control.c
index 0ad39b40d001..b1cb10b31633 100644
--- a/sbin/hastd/control.c
+++ b/sbin/hastd/control.c
@@ -32,17 +32,19 @@ __FBSDID("$FreeBSD$");
 
 #include <sys/types.h>
 #include <sys/wait.h>
-#include <signal.h>
 
 #include <assert.h>
 #include <errno.h>
 #include <pthread.h>
+#include <signal.h>
 #include <stdio.h>
 #include <string.h>
+#include <unistd.h>
 
 #include "hast.h"
 #include "hastd.h"
 #include "hast_proto.h"
+#include "hooks.h"
 #include "nv.h"
 #include "pjdlog.h"
 #include "proto.h"
@@ -50,19 +52,31 @@ __FBSDID("$FreeBSD$");
 
 #include "control.h"
 
-static void
-control_set_role(struct hastd_config *cfg, struct nv *nvout, uint8_t role,
-    struct hast_resource *res, const char *name, unsigned int no)
+void
+child_cleanup(struct hast_resource *res)
 {
 
-	assert(cfg != NULL);
-	assert(nvout != NULL);
-	assert(name != NULL);
+	proto_close(res->hr_ctrl);
+	res->hr_ctrl = NULL;
+	proto_close(res->hr_event);
+	res->hr_event = NULL;
+	res->hr_workerpid = 0;
+}
+
+static void
+control_set_role_common(struct hastd_config *cfg, struct nv *nvout,
+    uint8_t role, struct hast_resource *res, const char *name, unsigned int no)
+{
+	int oldrole;
 
 	/* Name is always needed. */
-	nv_add_string(nvout, name, "resource%u", no);
+	if (name != NULL)
+		nv_add_string(nvout, name, "resource%u", no);
 
 	if (res == NULL) {
+		assert(cfg != NULL);
+		assert(name != NULL);
+
 		TAILQ_FOREACH(res, &cfg->hc_resources, hr_next) {
 			if (strcmp(res->hr_name, name) == 0)
 				break;
@@ -85,6 +99,7 @@ control_set_role(struct hastd_config *cfg, struct nv *nvout, uint8_t role,
 	pjdlog_info("Role changed to %s.", role2str(role));
 
 	/* Change role to the new one. */
+	oldrole = res->hr_role;
 	res->hr_role = role;
 	pjdlog_prefix_set("[%s] (%s) ", res->hr_name, role2str(res->hr_role));
 
@@ -106,13 +121,22 @@ control_set_role(struct hastd_config *cfg, struct nv *nvout, uint8_t role,
 			pjdlog_debug(1, "Worker process %u stopped.",
 			    (unsigned int)res->hr_workerpid);
 		}
-		res->hr_workerpid = 0;
+		child_cleanup(res);
 	}
 
 	/* Start worker process if we are changing to primary. */
 	if (role == HAST_ROLE_PRIMARY)
 		hastd_primary(res);
 	pjdlog_prefix_set("%s", "");
+	hook_exec(res->hr_exec, "role", res->hr_name, role2str(oldrole),
+	    role2str(res->hr_role), NULL);
+}
+
+void
+control_set_role(struct hast_resource *res, uint8_t role)
+{
+
+	control_set_role_common(NULL, NULL, role, res, NULL, 0);
 }
 
 static void
@@ -306,7 +330,7 @@ control_handle(struct hastd_config *cfg)
 		TAILQ_FOREACH(res, &cfg->hc_resources, hr_next) {
 			switch (cmd) {
 			case HASTCTL_SET_ROLE:
-				control_set_role(cfg, nvout, role, res,
+				control_set_role_common(cfg, nvout, role, res,
 				    res->hr_name, ii++);
 				break;
 			case HASTCTL_STATUS:
@@ -329,8 +353,8 @@ control_handle(struct hastd_config *cfg)
 				break;
 			switch (cmd) {
 			case HASTCTL_SET_ROLE:
-				control_set_role(cfg, nvout, role, NULL, str,
-				    ii);
+				control_set_role_common(cfg, nvout, role, NULL,
+				    str, ii);
 				break;
 			case HASTCTL_STATUS:
 				control_status(cfg, nvout, NULL, str, ii);
@@ -375,7 +399,8 @@ ctrl_thread(void *arg)
 				pthread_exit(NULL);
 			pjdlog_errno(LOG_ERR,
 			    "Unable to receive control message");
-			continue;
+			kill(getpid(), SIGTERM);
+			pthread_exit(NULL);
 		}
 		cmd = nv_get_uint8(nvin, "cmd");
 		if (cmd == 0) {
diff --git a/sbin/hastd/control.h b/sbin/hastd/control.h
index 15ea290ea557..582617d21e18 100644
--- a/sbin/hastd/control.h
+++ b/sbin/hastd/control.h
@@ -36,6 +36,11 @@
 #define	HASTCTL_STATUS		2
 
 struct hastd_config;
+struct hast_resource;
+
+void child_cleanup(struct hast_resource *res);
+
+void control_set_role(struct hast_resource *res, uint8_t role);
 
 void control_handle(struct hastd_config *cfg);
 
diff --git a/sbin/hastd/event.c b/sbin/hastd/event.c
new file mode 100644
index 000000000000..d772f692ba2e
--- /dev/null
+++ b/sbin/hastd/event.c
@@ -0,0 +1,162 @@
+/*-
+ * Copyright (c) 2010 Pawel Jakub Dawidek <pjd@FreeBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <assert.h>
+#include <errno.h>
+
+#include "hast.h"
+#include "hast_proto.h"
+#include "hooks.h"
+#include "nv.h"
+#include "pjdlog.h"
+#include "proto.h"
+#include "subr.h"
+
+#include "event.h"
+
+void
+event_send(const struct hast_resource *res, int event)
+{
+	struct nv *nvin, *nvout;
+	int error;
+
+	assert(res != NULL);
+	assert(event >= EVENT_MIN && event <= EVENT_MAX);
+
+	nvin = nvout = NULL;
+
+	/*
+	 * Prepare and send event to parent process.
+	 */
+	nvout = nv_alloc();
+	nv_add_uint8(nvout, (uint8_t)event, "event");
+	error = nv_error(nvout);
+	if (error != 0) {
+		pjdlog_common(LOG_ERR, 0, error,
+		    "Unable to prepare event header");
+		goto done;
+	}
+	if (hast_proto_send(res, res->hr_event, nvout, NULL, 0) < 0) {
+		pjdlog_errno(LOG_ERR, "Unable to send event header");
+		goto done;
+	}
+	if (hast_proto_recv_hdr(res->hr_event, &nvin) < 0) {
+		pjdlog_errno(LOG_ERR, "Unable to receive event header");
+		goto done;
+	}
+	/*
+	 * Do nothing with the answer. We only wait for it to be sure not
+	 * to exit too quickly after sending an event and exiting immediately.
+	 */
+done:
+	if (nvin != NULL)
+		nv_free(nvin);
+	if (nvout != NULL)
+		nv_free(nvout);
+}
+
+int
+event_recv(const struct hast_resource *res)
+{
+	struct nv *nvin, *nvout;
+	const char *evstr;
+	uint8_t event;
+	int error;
+
+	assert(res != NULL);
+
+	nvin = nvout = NULL;
+
+	if (hast_proto_recv_hdr(res->hr_event, &nvin) < 0) {
+		/*
+		 * First error log as debug. This is because worker process
+		 * most likely exited.
+		 */
+		pjdlog_common(LOG_DEBUG, 1, errno,
+		    "Unable to receive event header");
+		goto fail;
+	}
+
+	event = nv_get_uint8(nvin, "event");
+	if (event == EVENT_NONE) {
+		pjdlog_error("Event header is missing 'event' field.");
+		goto fail;
+	}
+
+	switch (event) {
+	case EVENT_CONNECT:
+		evstr = "connect";
+		break;
+	case EVENT_DISCONNECT:
+		evstr = "disconnect";
+		break;
+	case EVENT_SYNCSTART:
+		evstr = "syncstart";
+		break;
+	case EVENT_SYNCDONE:
+		evstr = "syncdone";
+		break;
+	case EVENT_SYNCINTR:
+		evstr = "syncintr";
+		break;
+	case EVENT_SPLITBRAIN:
+		evstr = "split-brain";
+		break;
+	default:
+		pjdlog_error("Event header contain invalid event number (%hhu).",
+		    event);
+		goto fail;
+	}
+
+	pjdlog_prefix_set("[%s] (%s) ", res->hr_name, role2str(res->hr_role));
+	hook_exec(res->hr_exec, evstr, res->hr_name, NULL);
+	pjdlog_prefix_set("%s", "");
+
+	nvout = nv_alloc();
+	nv_add_int16(nvout, 0, "error");
+	error = nv_error(nvout);
+	if (error != 0) {
+		pjdlog_common(LOG_ERR, 0, error,
+		    "Unable to prepare event header");
+		goto fail;
+	}
+	if (hast_proto_send(res, res->hr_event, nvout, NULL, 0) < 0) {
+		pjdlog_errno(LOG_ERR, "Unable to send event header");
+		goto fail;
+	}
+	nv_free(nvin);
+	nv_free(nvout);
+	return (0);
+fail:
+	if (nvin != NULL)
+		nv_free(nvin);
+	if (nvout != NULL)
+		nv_free(nvout);
+	return (-1);
+}
diff --git a/sbin/hastd/event.h b/sbin/hastd/event.h
new file mode 100644
index 000000000000..1614bf199163
--- /dev/null
+++ b/sbin/hastd/event.h
@@ -0,0 +1,46 @@
+/*-
+ * Copyright (c) 2010 Pawel Jakub Dawidek <pjd@FreeBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef	_EVENT_H_
+#define	_EVENT_H_
+
+#define	EVENT_NONE		0
+#define	EVENT_CONNECT		1
+#define	EVENT_DISCONNECT	2
+#define	EVENT_SYNCSTART		3
+#define	EVENT_SYNCDONE		4
+#define	EVENT_SYNCINTR		5
+#define	EVENT_SPLITBRAIN	6
+
+#define	EVENT_MIN		EVENT_CONNECT
+#define	EVENT_MAX		EVENT_SPLITBRAIN
+
+void event_send(const struct hast_resource *res, int event);
+int event_recv(const struct hast_resource *res);
+
+#endif	/* !_EVENT_H_ */
diff --git a/sbin/hastd/hast.conf.5 b/sbin/hastd/hast.conf.5
index 1ccd47941c66..87a7e35f3b10 100644
--- a/sbin/hastd/hast.conf.5
+++ b/sbin/hastd/hast.conf.5
@@ -1,4 +1,5 @@
 .\" Copyright (c) 2010 The FreeBSD Foundation
+.\" Copyright (c) 2010 Pawel Jakub Dawidek <pjd@FreeBSD.org>
 .\" All rights reserved.
 .\"
 .\" This software was developed by Pawel Jakub Dawidek under sponsorship from
@@ -27,14 +28,14 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd February 1, 2010
+.Dd August 30, 2010
 .Dt HAST.CONF 5
 .Os
 .Sh NAME
 .Nm hast.conf
 .Nd configuration file for the
 .Xr hastd 8
-deamon and the
+daemon and the
 .Xr hastctl 8
 utility.
 .Sh DESCRIPTION
@@ -59,6 +60,7 @@ control <addr>
 listen <addr>
 replication <mode>
 timeout <seconds>
+exec <path>
 
 on <node> {
 	# Node section
@@ -78,6 +80,7 @@ resource <name> {
 	name <name>
 	local <path>
 	timeout <seconds>
+	exec <path>
 
 	on <node> {
 		# Resource-node section
@@ -169,14 +172,16 @@ The only situation where some small amount of data could be lost is when
 the data is stored on primary node and sent to the secondary.
 Secondary node then acknowledges data receipt and primary reports
 success to an application.
-However, it may happen that the seconderay goes down before the received
+However, it may happen that the secondary goes down before the received
 data is really stored locally.
 Before secondary node returns, primary node dies entirely.
 When the secondary node comes back to life it becomes the new primary.
 Unfortunately some small amount of data which was confirmed to be stored
 to the application was lost.
-The risk of such a situation is very small, which is the reason for this
-mode to be the default.
+The risk of such a situation is very small.
+The
+.Ic memsync
+replication mode is currently not implemented.
 .It Ic fullsync
 .Pp
 Mark the write operation as completed when local as well as remote
@@ -184,7 +189,7 @@ write completes.
 This is the safest and the slowest replication mode.
 The
 .Ic fullsync
-replication mode is currently not implemented.
+replication mode is the default.
 .It Ic async
 .Pp
 The write operation is reported as complete right after the local write
@@ -201,6 +206,76 @@ replication mode is currently not implemented.
 Connection timeout in seconds.
 The default value is
 .Va 5 .
+.It Ic exec Aq path
+.Pp
+Execute the given program on various HAST events.
+Below is the list of currently implemented events and arguments the given
+program is executed with:
+.Bl -tag -width ".Ic xxxx"
+.It Ic "<path> role <resource> <oldrole> <newrole>"
+.Pp
+Executed on both primary and secondary nodes when resource role is changed.
+.Pp
+.It Ic "<path> connect <resource>"
+.Pp
+Executed on both primary and secondary nodes when connection for the given
+resource between the nodes is established.
+.Pp
+.It Ic "<path> disconnect <resource>"
+.Pp
+Executed on both primary and secondary nodes when connection for the given
+resource between the nodes is lost.
+.Pp
+.It Ic "<path> syncstart <resource>"
+.Pp
+Executed on primary node when synchronization process of secondary node is
+started.
+.Pp
+.It Ic "<path> syncdone <resource>"
+.Pp
+Executed on primary node when synchronization process of secondary node is
+completed successfully.
+.Pp
+.It Ic "<path> syncintr <resource>"
+.Pp
+Executed on primary node when synchronization process of secondary node is
+interrupted, most likely due to secondary node outage or connection failure
+between the nodes.
+.Pp
+.It Ic "<path> split-brain <resource>"
+.Pp
+Executed on both primary and secondary nodes when split-brain condition is
+detected.
+.Pp
+.El
+The
+.Aq path
+argument should contain full path to executable program.
+If the given program exits with code different than
+.Va 0 ,
+.Nm hastd
+will log it as an error.
+.Pp
+The
+.Aq resource
+argument is resource name from the configuration file.
+.Pp
+The
+.Aq oldrole
+argument is previous resource role (before the change).
+It can be one of:
+.Ar init ,
+.Ar secondary ,
+.Ar primary .
+.Pp
+The
+.Aq newrole
+argument is current resource role (after the change).
+It can be one of:
+.Ar init ,
+.Ar secondary ,
+.Ar primary .
+.Pp
 .It Ic name Aq name
 .Pp
 GEOM provider name that will appear as
@@ -223,6 +298,24 @@ When operating as a primary node this address will be used to connect to
 the secondary node.
 When operating as a secondary node only connections from this address
 will be accepted.
+.Pp
+A special value of
+.Va none
+can be used when the remote address is not yet known (eg. the other node is not
+set up yet).
+.El
+.Sh FILES
+.Bl -tag -width ".Pa /var/run/hastctl" -compact
+.It Pa /etc/hast.conf
+The default
+.Nm
+configuration file.
+.It Pa /var/run/hastctl
+Control socket used by the
+.Xr hastctl 8
+control utility to communicate with the
+.Xr hastd 8
+daemon.
 .El
 .Sh EXAMPLES
 The example configuration file can look as follows:
@@ -248,19 +341,6 @@ resource tank {
 	}
 }
 .Ed
-.Sh FILES
-.Bl -tag -width ".Pa /var/run/hastctl" -compact
-.It Pa /etc/hast.conf
-The default
-.Nm
-configuration file.
-.It Pa /var/run/hastctl
-Control socket used by the
-.Xr hastctl 8
-control utility to communicate with the
-.Xr hastd 8
-daemon.
-.El
 .Sh SEE ALSO
 .Xr gethostname 3 ,
 .Xr geom 4 ,
diff --git a/sbin/hastd/hast.h b/sbin/hastd/hast.h
index 2230afb20d60..a42865f0d061 100644
--- a/sbin/hastd/hast.h
+++ b/sbin/hastd/hast.h
@@ -48,7 +48,12 @@
 
 #include "proto.h"
 
-#define	HAST_PROTO_VERSION	0
+/*
+ * Version history:
+ * 0 - initial version
+ * 1 - HIO_KEEPALIVE added
+ */
+#define	HAST_PROTO_VERSION	1
 
 #define	EHAST_OK		0
 #define	EHAST_NOENTRY		1
@@ -74,6 +79,7 @@
 #define	HIO_WRITE		2
 #define	HIO_DELETE		3
 #define	HIO_FLUSH		4
+#define	HIO_KEEPALIVE		5
 
 #define	HAST_TIMEOUT	5
 #define	HAST_CONFIG	"/etc/hast.conf"
@@ -121,6 +127,8 @@ struct hast_resource {
 	int	hr_extentsize;
 	/* Maximum number of extents that are kept dirty. */
 	int	hr_keepdirty;
+	/* Path to a program to execute on various events. */
+	char	hr_exec[PATH_MAX];
 
 	/* Path to local component. */
 	char	hr_localpath[PATH_MAX];
@@ -173,6 +181,8 @@ struct hast_resource {
 	pid_t	hr_workerpid;
 	/* Control connection between parent and child. */
 	struct proto_conn *hr_ctrl;
+	/* Events from child to parent. */
+	struct proto_conn *hr_event;
 
 	/* Activemap structure. */
 	struct activemap *hr_amp;
@@ -183,7 +193,7 @@ struct hast_resource {
 	TAILQ_ENTRY(hast_resource) hr_next;
 };
 
-struct hastd_config *yy_config_parse(const char *config);
+struct hastd_config *yy_config_parse(const char *config, bool exitonerror);
 void yy_config_free(struct hastd_config *config);
 
 void yyerror(const char *);
diff --git a/sbin/hastd/hast_proto.c b/sbin/hastd/hast_proto.c
index 4e898ca12e09..101a508cc86c 100644
--- a/sbin/hastd/hast_proto.c
+++ b/sbin/hastd/hast_proto.c
@@ -56,8 +56,10 @@ struct hast_main_header {
 	uint32_t	size;
 } __packed;
 
-typedef int hps_send_t(struct hast_resource *, struct nv *nv, void **, size_t *, bool *);
-typedef int hps_recv_t(struct hast_resource *, struct nv *nv, void **, size_t *, bool *);
+typedef int hps_send_t(const struct hast_resource *, struct nv *nv, void **,
+    size_t *, bool *);
+typedef int hps_recv_t(const struct hast_resource *, struct nv *nv, void **,
+    size_t *, bool *);
 
 struct hast_pipe_stage {
 	const char	*hps_name;
@@ -65,14 +67,14 @@ struct hast_pipe_stage {
 	hps_recv_t	*hps_recv;
 };
 
-static int compression_send(struct hast_resource *res, struct nv *nv,
+static int compression_send(const struct hast_resource *res, struct nv *nv,
     void **datap, size_t *sizep, bool *freedatap);
-static int compression_recv(struct hast_resource *res, struct nv *nv,
+static int compression_recv(const struct hast_resource *res, struct nv *nv,
     void **datap, size_t *sizep, bool *freedatap);
 #ifdef HAVE_CRYPTO
-static int checksum_send(struct hast_resource *res, struct nv *nv,
+static int checksum_send(const struct hast_resource *res, struct nv *nv,
     void **datap, size_t *sizep, bool *freedatap);
-static int checksum_recv(struct hast_resource *res, struct nv *nv,
+static int checksum_recv(const struct hast_resource *res, struct nv *nv,
     void **datap, size_t *sizep, bool *freedatap);
 #endif
 
@@ -84,7 +86,7 @@ static struct hast_pipe_stage pipeline[] = {
 };
 
 static int
-compression_send(struct hast_resource *res, struct nv *nv, void **datap,
+compression_send(const struct hast_resource *res, struct nv *nv, void **datap,
     size_t *sizep, bool *freedatap)
 {
 	unsigned char *newbuf;
@@ -132,7 +134,7 @@ compression_send(struct hast_resource *res, struct nv *nv, void **datap,
 }
 
 static int
-compression_recv(struct hast_resource *res, struct nv *nv, void **datap,
+compression_recv(const struct hast_resource *res, struct nv *nv, void **datap,
     size_t *sizep, bool *freedatap)
 {
 	unsigned char *newbuf;
@@ -169,7 +171,7 @@ compression_recv(struct hast_resource *res, struct nv *nv, void **datap,
 
 #ifdef HAVE_CRYPTO
 static int
-checksum_send(struct hast_resource *res, struct nv *nv, void **datap,
+checksum_send(const struct hast_resource *res, struct nv *nv, void **datap,
     size_t *sizep, bool *freedatap __unused)
 {
 	unsigned char hash[SHA256_DIGEST_LENGTH];
@@ -188,7 +190,7 @@ checksum_send(struct hast_resource *res, struct nv *nv, void **datap,
 }
 
 static int
-checksum_recv(struct hast_resource *res, struct nv *nv, void **datap,
+checksum_recv(const struct hast_resource *res, struct nv *nv, void **datap,
     size_t *sizep, bool *freedatap __unused)
 {
 	unsigned char chash[SHA256_DIGEST_LENGTH];
@@ -236,7 +238,7 @@ checksum_recv(struct hast_resource *res, struct nv *nv, void **datap,
  * There can be no data at all (data is NULL then).
  */
 int
-hast_proto_send(struct hast_resource *res, struct proto_conn *conn,
+hast_proto_send(const struct hast_resource *res, struct proto_conn *conn,
     struct nv *nv, const void *data, size_t size)
 {
 	struct hast_main_header hdr;
@@ -293,7 +295,7 @@ end:
 }
 
 int
-hast_proto_recv_hdr(struct proto_conn *conn, struct nv **nvp)
+hast_proto_recv_hdr(const struct proto_conn *conn, struct nv **nvp)
 {
 	struct hast_main_header hdr;
 	struct nv *nv;
@@ -335,7 +337,7 @@ fail:
 }
 
 int
-hast_proto_recv_data(struct hast_resource *res, struct proto_conn *conn,
+hast_proto_recv_data(const struct hast_resource *res, struct proto_conn *conn,
     struct nv *nv, void *data, size_t size)
 {
 	unsigned int ii;
@@ -384,7 +386,7 @@ if (ret < 0) printf("%s:%u %s\n", __func__, __LINE__, strerror(errno));
 }
 
 int
-hast_proto_recv(struct hast_resource *res, struct proto_conn *conn,
+hast_proto_recv(const struct hast_resource *res, struct proto_conn *conn,
     struct nv **nvp, void *data, size_t size)
 {
 	struct nv *nv;
diff --git a/sbin/hastd/hast_proto.h b/sbin/hastd/hast_proto.h
index 3894e3838089..b48c3ca12146 100644
--- a/sbin/hastd/hast_proto.h
+++ b/sbin/hastd/hast_proto.h
@@ -37,12 +37,12 @@
 #include <nv.h>
 #include <proto.h>
 
-int hast_proto_send(struct hast_resource *res, struct proto_conn *conn,
+int hast_proto_send(const struct hast_resource *res, struct proto_conn *conn,
     struct nv *nv, const void *data, size_t size);
-int hast_proto_recv(struct hast_resource *res, struct proto_conn *conn,
+int hast_proto_recv(const struct hast_resource *res, struct proto_conn *conn,
     struct nv **nvp, void *data, size_t size);
-int hast_proto_recv_hdr(struct proto_conn *conn, struct nv **nvp);
-int hast_proto_recv_data(struct hast_resource *res, struct proto_conn *conn,
-    struct nv *nv, void *data, size_t size);
+int hast_proto_recv_hdr(const struct proto_conn *conn, struct nv **nvp);
+int hast_proto_recv_data(const struct hast_resource *res,
+    struct proto_conn *conn, struct nv *nv, void *data, size_t size);
 
 #endif	/* !_HAST_PROTO_H_ */
diff --git a/sbin/hastd/hastd.8 b/sbin/hastd/hastd.8
index 276b3d301ab4..84896560b681 100644
--- a/sbin/hastd/hastd.8
+++ b/sbin/hastd/hastd.8
@@ -170,6 +170,23 @@ stored.
 The default location is
 .Pa /var/run/hastd.pid .
 .El
+.Sh FILES
+.Bl -tag -width ".Pa /var/run/hastctl" -compact
+.It Pa /etc/hast.conf
+The configuration file for
+.Nm
+and
+.Xr hastctl 8 .
+.It Pa /var/run/hastctl
+Control socket used by the
+.Xr hastctl 8
+control utility to communicate with
+.Nm .
+.It Pa /var/run/hastd.pid
+The default location of the
+.Nm
+PID file.
+.El
 .Sh EXIT STATUS
 Exit status is 0 on success, or one of the values described in
 .Xr sysexits 3
@@ -196,23 +213,6 @@ nodeA# hastctl role primary shared
 nodeA# newfs -U /dev/hast/shared
 nodeA# mount -o noatime /dev/hast/shared /shared
 .Ed
-.Sh FILES
-.Bl -tag -width ".Pa /var/run/hastctl" -compact
-.It Pa /etc/hast.conf
-The configuration file for
-.Nm
-and
-.Xr hastctl 8 .
-.It Pa /var/run/hastctl
-Control socket used by the
-.Xr hastctl 8
-control utility to communicate with
-.Nm .
-.It Pa /var/run/hastd.pid
-The default location of the
-.Nm
-PID file.
-.El
 .Sh SEE ALSO
 .Xr sysexits 3 ,
 .Xr geom 4 ,
diff --git a/sbin/hastd/hastd.c b/sbin/hastd/hastd.c
index 1d9d1b4e3af6..e47b19d49246 100644
--- a/sbin/hastd/hastd.c
+++ b/sbin/hastd/hastd.c
@@ -1,5 +1,6 @@
 /*-
  * Copyright (c) 2009-2010 The FreeBSD Foundation
+ * Copyright (c) 2010 Pawel Jakub Dawidek <pjd@FreeBSD.org>
  * All rights reserved.
  *
  * This software was developed by Pawel Jakub Dawidek under sponsorship from
@@ -51,24 +52,25 @@ __FBSDID("$FreeBSD$");
 #include <pjdlog.h>
 
 #include "control.h"
+#include "event.h"
 #include "hast.h"
 #include "hast_proto.h"
 #include "hastd.h"
+#include "hooks.h"
 #include "subr.h"
 
 /* Path to configuration file. */
-static const char *cfgpath = HAST_CONFIG;
+const char *cfgpath = HAST_CONFIG;
 /* Hastd configuration. */
 static struct hastd_config *cfg;
-/* Was SIGCHLD signal received? */
-static bool sigchld_received = false;
-/* Was SIGHUP signal received? */
-static bool sighup_received = false;
 /* Was SIGINT or SIGTERM signal received? */
 bool sigexit_received = false;
 /* PID file handle. */
 struct pidfh *pfh;
 
+/* How often check for hooks running for too long. */
+#define	REPORT_INTERVAL	10
+
 static void
 usage(void)
 {
@@ -77,22 +79,6 @@ usage(void)
 }
 
 static void
-sighandler(int sig)
-{
-
-	switch (sig) {
-	case SIGCHLD:
-		sigchld_received = true;
-		break;
-	case SIGHUP:
-		sighup_received = true;
-		break;
-	default:
-		assert(!"invalid condition");
-	}
-}
-
-static void
 g_gate_load(void)
 {
 
@@ -139,15 +125,16 @@ child_exit(void)
 		if (res == NULL) {
 			/*
 			 * This can happen when new connection arrives and we
-			 * cancel child responsible for the old one.
+			 * cancel child responsible for the old one or if this
+			 * was hook which we executed.
 			 */
+			hook_check_one(pid, status);
 			continue;
 		}
 		pjdlog_prefix_set("[%s] (%s) ", res->hr_name,
 		    role2str(res->hr_role));
 		child_exit_log(pid, status);
-		proto_close(res->hr_ctrl);
-		res->hr_workerpid = 0;
+		child_cleanup(res);
 		if (res->hr_role == HAST_ROLE_PRIMARY) {
 			/*
 			 * Restart child process if it was killed by signal
@@ -169,12 +156,226 @@ child_exit(void)
 	}
 }
 
+static bool
+resource_needs_restart(const struct hast_resource *res0,
+    const struct hast_resource *res1)
+{
+
+	assert(strcmp(res0->hr_name, res1->hr_name) == 0);
+
+	if (strcmp(res0->hr_provname, res1->hr_provname) != 0)
+		return (true);
+	if (strcmp(res0->hr_localpath, res1->hr_localpath) != 0)
+		return (true);
+	if (res0->hr_role == HAST_ROLE_INIT ||
+	    res0->hr_role == HAST_ROLE_SECONDARY) {
+		if (strcmp(res0->hr_remoteaddr, res1->hr_remoteaddr) != 0)
+			return (true);
+		if (res0->hr_replication != res1->hr_replication)
+			return (true);
+		if (res0->hr_timeout != res1->hr_timeout)
+			return (true);
+		if (strcmp(res0->hr_exec, res1->hr_exec) != 0)
+			return (true);
+	}
+	return (false);
+}
+
+static bool
+resource_needs_reload(const struct hast_resource *res0,
+    const struct hast_resource *res1)
+{
+
+	assert(strcmp(res0->hr_name, res1->hr_name) == 0);
+	assert(strcmp(res0->hr_provname, res1->hr_provname) == 0);
+	assert(strcmp(res0->hr_localpath, res1->hr_localpath) == 0);
+
+	if (res0->hr_role != HAST_ROLE_PRIMARY)
+		return (false);
+
+	if (strcmp(res0->hr_remoteaddr, res1->hr_remoteaddr) != 0)
+		return (true);
+	if (res0->hr_replication != res1->hr_replication)
+		return (true);
+	if (res0->hr_timeout != res1->hr_timeout)
+		return (true);
+	if (strcmp(res0->hr_exec, res1->hr_exec) != 0)
+		return (true);
+	return (false);
+}
+
 static void
 hastd_reload(void)
 {
+	struct hastd_config *newcfg;
+	struct hast_resource *nres, *cres, *tres;
+	uint8_t role;
+
+	pjdlog_info("Reloading configuration...");
+
+	newcfg = yy_config_parse(cfgpath, false);
+	if (newcfg == NULL)
+		goto failed;
 
-	/* TODO */
-	pjdlog_warning("Configuration reload is not implemented.");
+	/*
+	 * Check if control address has changed.
+	 */
+	if (strcmp(cfg->hc_controladdr, newcfg->hc_controladdr) != 0) {
+		if (proto_server(newcfg->hc_controladdr,
+		    &newcfg->hc_controlconn) < 0) {
+			pjdlog_errno(LOG_ERR,
+			    "Unable to listen on control address %s",
+			    newcfg->hc_controladdr);
+			goto failed;
+		}
+	}
+	/*
+	 * Check if listen address has changed.
+	 */
+	if (strcmp(cfg->hc_listenaddr, newcfg->hc_listenaddr) != 0) {
+		if (proto_server(newcfg->hc_listenaddr,
+		    &newcfg->hc_listenconn) < 0) {
+			pjdlog_errno(LOG_ERR, "Unable to listen on address %s",
+			    newcfg->hc_listenaddr);
+			goto failed;
+		}
+	}
+	/*
+	 * Only when both control and listen sockets are successfully
+	 * initialized switch them to new configuration.
+	 */
+	if (newcfg->hc_controlconn != NULL) {
+		pjdlog_info("Control socket changed from %s to %s.",
+		    cfg->hc_controladdr, newcfg->hc_controladdr);
+		proto_close(cfg->hc_controlconn);
+		cfg->hc_controlconn = newcfg->hc_controlconn;
+		newcfg->hc_controlconn = NULL;
+		strlcpy(cfg->hc_controladdr, newcfg->hc_controladdr,
+		    sizeof(cfg->hc_controladdr));
+	}
+	if (newcfg->hc_listenconn != NULL) {
+		pjdlog_info("Listen socket changed from %s to %s.",
+		    cfg->hc_listenaddr, newcfg->hc_listenaddr);
+		proto_close(cfg->hc_listenconn);
+		cfg->hc_listenconn = newcfg->hc_listenconn;
+		newcfg->hc_listenconn = NULL;
+		strlcpy(cfg->hc_listenaddr, newcfg->hc_listenaddr,
+		    sizeof(cfg->hc_listenaddr));
+	}
+
+	/*
+	 * Stop and remove resources that were removed from the configuration.
+	 */
+	TAILQ_FOREACH_SAFE(cres, &cfg->hc_resources, hr_next, tres) {
+		TAILQ_FOREACH(nres, &newcfg->hc_resources, hr_next) {
+			if (strcmp(cres->hr_name, nres->hr_name) == 0)
+				break;
+		}
+		if (nres == NULL) {
+			control_set_role(cres, HAST_ROLE_INIT);
+			TAILQ_REMOVE(&cfg->hc_resources, cres, hr_next);
+			pjdlog_info("Resource %s removed.", cres->hr_name);
+			free(cres);
+		}
+	}
+	/*
+	 * Move new resources to the current configuration.
+	 */
+	TAILQ_FOREACH_SAFE(nres, &newcfg->hc_resources, hr_next, tres) {
+		TAILQ_FOREACH(cres, &cfg->hc_resources, hr_next) {
+			if (strcmp(cres->hr_name, nres->hr_name) == 0)
+				break;
+		}
+		if (cres == NULL) {
+			TAILQ_REMOVE(&newcfg->hc_resources, nres, hr_next);
+			TAILQ_INSERT_TAIL(&cfg->hc_resources, nres, hr_next);
+			pjdlog_info("Resource %s added.", nres->hr_name);
+		}
+	}
+	/*
+	 * Deal with modified resources.
+	 * Depending on what has changed exactly we might want to perform
+	 * different actions.
+	 *
+	 * We do full resource restart in the following situations:
+	 * Resource role is INIT or SECONDARY.
+	 * Resource role is PRIMARY and path to local component or provider
+	 * name has changed.
+	 * In case of PRIMARY, the worker process will be killed and restarted,
+	 * which also means removing /dev/hast/<name> provider and
+	 * recreating it.
+	 *
+	 * We do just reload (send SIGHUP to worker process) if we act as
+	 * PRIMARY, but only remote address, replication mode and timeout
+	 * has changed. For those, there is no need to restart worker process.
+	 * If PRIMARY receives SIGHUP, it will reconnect if remote address or
+	 * replication mode has changed or simply set new timeout if only
+	 * timeout has changed.
+	 */
+	TAILQ_FOREACH_SAFE(nres, &newcfg->hc_resources, hr_next, tres) {
+		TAILQ_FOREACH(cres, &cfg->hc_resources, hr_next) {
+			if (strcmp(cres->hr_name, nres->hr_name) == 0)
+				break;
+		}
+		assert(cres != NULL);
+		if (resource_needs_restart(cres, nres)) {
+			pjdlog_info("Resource %s configuration was modified, restarting it.",
+			    cres->hr_name);
+			role = cres->hr_role;
+			control_set_role(cres, HAST_ROLE_INIT);
+			TAILQ_REMOVE(&cfg->hc_resources, cres, hr_next);
+			free(cres);
+			TAILQ_REMOVE(&newcfg->hc_resources, nres, hr_next);
+			TAILQ_INSERT_TAIL(&cfg->hc_resources, nres, hr_next);
+			control_set_role(nres, role);
+		} else if (resource_needs_reload(cres, nres)) {
+			pjdlog_info("Resource %s configuration was modified, reloading it.",
+			    cres->hr_name);
+			strlcpy(cres->hr_remoteaddr, nres->hr_remoteaddr,
+			    sizeof(cres->hr_remoteaddr));
+			cres->hr_replication = nres->hr_replication;
+			cres->hr_timeout = nres->hr_timeout;
+			if (cres->hr_workerpid != 0) {
+				if (kill(cres->hr_workerpid, SIGHUP) < 0) {
+					pjdlog_errno(LOG_WARNING,
+					    "Unable to send SIGHUP to worker process %u",
+					    (unsigned int)cres->hr_workerpid);
+				}
+			}
+		}
+	}
+
+	yy_config_free(newcfg);
+	pjdlog_info("Configuration reloaded successfully.");
+	return;
+failed:
+	if (newcfg != NULL) {
+		if (newcfg->hc_controlconn != NULL)
+			proto_close(newcfg->hc_controlconn);
+		if (newcfg->hc_listenconn != NULL)
+			proto_close(newcfg->hc_listenconn);
+		yy_config_free(newcfg);
+	}
+	pjdlog_warning("Configuration not reloaded.");
+}
+
+static void
+terminate_workers(void)
+{
+	struct hast_resource *res;
+
+	pjdlog_info("Termination signal received, exiting.");
+	TAILQ_FOREACH(res, &cfg->hc_resources, hr_next) {
+		if (res->hr_workerpid == 0)
+			continue;
+		pjdlog_info("Terminating worker process (resource=%s, role=%s, pid=%u).",
+		    res->hr_name, role2str(res->hr_role), res->hr_workerpid);
+		if (kill(res->hr_workerpid, SIGTERM) == 0)
+			continue;
+		pjdlog_errno(LOG_WARNING,
+		    "Unable to send signal to worker process (resource=%s, role=%s, pid=%u).",
+		    res->hr_name, role2str(res->hr_role), res->hr_workerpid);
+	}
 }
 
 static void
@@ -322,7 +523,7 @@ listen_accept(void)
 			} else {
 				child_exit_log(res->hr_workerpid, status);
 			}
-			res->hr_workerpid = 0;
+			child_cleanup(res);
 		} else if (res->hr_remotein != NULL) {
 			char oaddr[256];
 
@@ -399,43 +600,88 @@ close:
 static void
 main_loop(void)
 {
-	fd_set rfds, wfds;
-	int cfd, lfd, maxfd, ret;
+	struct hast_resource *res;
+	struct timeval seltimeout;
+	struct timespec sigtimeout;
+	int fd, maxfd, ret, signo;
+	sigset_t mask;
+	fd_set rfds;
+
+	seltimeout.tv_sec = REPORT_INTERVAL;
+	seltimeout.tv_usec = 0;
+	sigtimeout.tv_sec = 0;
+	sigtimeout.tv_nsec = 0;
 
-	cfd = proto_descriptor(cfg->hc_controlconn);
-	lfd = proto_descriptor(cfg->hc_listenconn);
-	maxfd = cfd > lfd ? cfd : lfd;
+	PJDLOG_VERIFY(sigemptyset(&mask) == 0);
+	PJDLOG_VERIFY(sigaddset(&mask, SIGHUP) == 0);
+	PJDLOG_VERIFY(sigaddset(&mask, SIGINT) == 0);
+	PJDLOG_VERIFY(sigaddset(&mask, SIGTERM) == 0);
+	PJDLOG_VERIFY(sigaddset(&mask, SIGCHLD) == 0);
 
 	for (;;) {
-		if (sigchld_received) {
-			sigchld_received = false;
-			child_exit();
-		}
-		if (sighup_received) {
-			sighup_received = false;
-			hastd_reload();
+		while ((signo = sigtimedwait(&mask, NULL, &sigtimeout)) != -1) {
+			switch (signo) {
+			case SIGINT:
+			case SIGTERM:
+				sigexit_received = true;
+				terminate_workers();
+				exit(EX_OK);
+				break;
+			case SIGCHLD:
+				child_exit();
+				break;
+			case SIGHUP:
+				hastd_reload();
+				break;
+			default:
+				assert(!"invalid condition");
+			}
 		}
 
 		/* Setup descriptors for select(2). */
 		FD_ZERO(&rfds);
-		FD_SET(cfd, &rfds);
-		FD_SET(lfd, &rfds);
-		FD_ZERO(&wfds);
-		FD_SET(cfd, &wfds);
-		FD_SET(lfd, &wfds);
+		maxfd = fd = proto_descriptor(cfg->hc_controlconn);
+		assert(fd >= 0);
+		FD_SET(fd, &rfds);
+		fd = proto_descriptor(cfg->hc_listenconn);
+		assert(fd >= 0);
+		FD_SET(fd, &rfds);
+		maxfd = fd > maxfd ? fd : maxfd;
+		TAILQ_FOREACH(res, &cfg->hc_resources, hr_next) {
+			if (res->hr_event == NULL)
+				continue;
+			fd = proto_descriptor(res->hr_event);
+			assert(fd >= 0);
+			FD_SET(fd, &rfds);
+			maxfd = fd > maxfd ? fd : maxfd;
+		}
 
-		ret = select(maxfd + 1, &rfds, &wfds, NULL, NULL);
-		if (ret == -1) {
+		assert(maxfd + 1 <= (int)FD_SETSIZE);
+		ret = select(maxfd + 1, &rfds, NULL, NULL, &seltimeout);
+		if (ret == 0)
+			hook_check(false);
+		else if (ret == -1) {
 			if (errno == EINTR)
 				continue;
 			KEEP_ERRNO((void)pidfile_remove(pfh));
 			pjdlog_exit(EX_OSERR, "select() failed");
 		}
 
-		if (FD_ISSET(cfd, &rfds) || FD_ISSET(cfd, &wfds))
+		if (FD_ISSET(proto_descriptor(cfg->hc_controlconn), &rfds))
 			control_handle(cfg);
-		if (FD_ISSET(lfd, &rfds) || FD_ISSET(lfd, &wfds))
+		if (FD_ISSET(proto_descriptor(cfg->hc_listenconn), &rfds))
 			listen_accept();
+		TAILQ_FOREACH(res, &cfg->hc_resources, hr_next) {
+			if (res->hr_event == NULL)
+				continue;
+			if (FD_ISSET(proto_descriptor(res->hr_event), &rfds)) {
+				if (event_recv(res) == 0)
+					continue;
+				/* The worker process exited? */
+				proto_close(res->hr_event);
+				res->hr_event = NULL;
+			}
+		}
 	}
 }
 
@@ -446,6 +692,7 @@ main(int argc, char *argv[])
 	pid_t otherpid;
 	bool foreground;
 	int debuglevel;
+	sigset_t mask;
 
 	g_gate_load();
 
@@ -490,14 +737,18 @@ main(int argc, char *argv[])
 			    (intmax_t)otherpid);
 		}
 		/* If we cannot create pidfile from other reasons, only warn. */
-		pjdlog_errno(LOG_WARNING, "Cannot open or create pidfile");
+		pjdlog_errno(LOG_WARNING, "Unable to open or create pidfile");
 	}
 
-	cfg = yy_config_parse(cfgpath);
+	cfg = yy_config_parse(cfgpath, true);
 	assert(cfg != NULL);
 
-	signal(SIGHUP, sighandler);
-	signal(SIGCHLD, sighandler);
+	PJDLOG_VERIFY(sigemptyset(&mask) == 0);
+	PJDLOG_VERIFY(sigaddset(&mask, SIGHUP) == 0);
+	PJDLOG_VERIFY(sigaddset(&mask, SIGINT) == 0);
+	PJDLOG_VERIFY(sigaddset(&mask, SIGTERM) == 0);
+	PJDLOG_VERIFY(sigaddset(&mask, SIGCHLD) == 0);
+	PJDLOG_VERIFY(sigprocmask(SIG_SETMASK, &mask, NULL) == 0);
 
 	/* Listen on control address. */
 	if (proto_server(cfg->hc_controladdr, &cfg->hc_controlconn) < 0) {
@@ -528,6 +779,8 @@ main(int argc, char *argv[])
 		}
 	}
 
+	hook_init();
+
 	main_loop();
 
 	exit(0);
diff --git a/sbin/hastd/hastd.h b/sbin/hastd/hastd.h
index 199de8c94d5c..0186e81e0e8e 100644
--- a/sbin/hastd/hastd.h
+++ b/sbin/hastd/hastd.h
@@ -39,6 +39,7 @@
 
 #include "hast.h"
 
+extern const char *cfgpath;
 extern bool sigexit_received;
 extern struct pidfh *pfh;
 
diff --git a/sbin/hastd/hooks.c b/sbin/hastd/hooks.c
index 1fdeb7500859..eff2f6f95319 100644
--- a/sbin/hastd/hooks.c
+++ b/sbin/hastd/hooks.c
@@ -1,5 +1,6 @@
 /*-
  * Copyright (c) 2010 The FreeBSD Foundation
+ * Copyright (c) 2010 Pawel Jakub Dawidek <pjd@FreeBSD.org>
  * All rights reserved.
  *
  * This software was developed by Pawel Jakub Dawidek under sponsorship from
@@ -31,21 +32,58 @@
 __FBSDID("$FreeBSD$");
 
 #include <sys/types.h>
+#include <sys/sysctl.h>
 #include <sys/wait.h>
 
 #include <assert.h>
+#include <errno.h>
 #include <fcntl.h>
+#include <libgen.h>
+#include <paths.h>
+#include <signal.h>
+#include <stdbool.h>
+#include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include <unistd.h>
 #include <string.h>
 #include <syslog.h>
-#include <libgen.h>
-#include <paths.h>
+#include <unistd.h>
 
 #include <pjdlog.h>
 
 #include "hooks.h"
+#include "synch.h"
+
+/* Report processes that are running for too long not often than this value. */
+#define	REPORT_INTERVAL	60
+
+/* Are we initialized? */
+static bool hooks_initialized = false;
+
+/*
+ * Keep all processes we forked on a global queue, so we can report nicely
+ * when they finish or report that they are running for a long time.
+ */
+#define	HOOKPROC_MAGIC_ALLOCATED	0x80090ca
+#define	HOOKPROC_MAGIC_ONLIST		0x80090c0
+struct hookproc {
+	/* Magic. */
+	int	hp_magic;
+	/* PID of a forked child. */
+	pid_t	hp_pid;
+	/* When process were forked? */
+	time_t	hp_birthtime;
+	/* When we logged previous reported? */
+	time_t	hp_lastreport;
+	/* Path to executable and all the arguments we passed. */
+	char	hp_comm[PATH_MAX];
+	TAILQ_ENTRY(hookproc) hp_next;
+};
+static TAILQ_HEAD(, hookproc) hookprocs;
+static pthread_mutex_t hookprocs_lock;
+
+static void hook_remove(struct hookproc *hp);
+static void hook_free(struct hookproc *hp);
 
 static void
 descriptors(void)
@@ -61,8 +99,21 @@ descriptors(void)
 		pjdlog_errno(LOG_WARNING, "sysconf(_SC_OPEN_MAX) failed");
 		maxfd = 1024;
 	}
-	for (fd = 0; fd <= maxfd; fd++)
-		close(fd);
+	for (fd = 0; fd <= maxfd; fd++) {
+		switch (fd) {
+		case STDIN_FILENO:
+		case STDOUT_FILENO:
+		case STDERR_FILENO:
+			if (pjdlog_mode_get() == PJDLOG_MODE_STD)
+				break;
+			/* FALLTHROUGH */
+		default:
+			close(fd);
+			break;
+		}
+	}
+	if (pjdlog_mode_get() == PJDLOG_MODE_STD)
+		return;
 	/*
 	 * Redirect stdin, stdout and stderr to /dev/null.
 	 */
@@ -95,28 +146,230 @@ descriptors(void)
 	}
 }
 
-int
+void
+hook_init(void)
+{
+
+	assert(!hooks_initialized);
+
+	mtx_init(&hookprocs_lock);
+	TAILQ_INIT(&hookprocs);
+	hooks_initialized = true;
+}
+
+void
+hook_fini(void)
+{
+	struct hookproc *hp;
+
+	assert(hooks_initialized);
+
+	mtx_lock(&hookprocs_lock);
+	while ((hp = TAILQ_FIRST(&hookprocs)) != NULL) {
+		assert(hp->hp_magic == HOOKPROC_MAGIC_ONLIST);
+		assert(hp->hp_pid > 0);
+
+		hook_remove(hp);
+		hook_free(hp);
+	}
+	mtx_unlock(&hookprocs_lock);
+
+	mtx_destroy(&hookprocs_lock);
+	TAILQ_INIT(&hookprocs);
+	hooks_initialized = false;
+}
+
+static struct hookproc *
+hook_alloc(const char *path, char **args)
+{
+	struct hookproc *hp;
+	unsigned int ii;
+
+	hp = malloc(sizeof(*hp));
+	if (hp == NULL) {
+		pjdlog_error("Unable to allocate %zu bytes of memory for a hook.",
+		    sizeof(*hp));
+		return (NULL);
+	}
+
+	hp->hp_pid = 0;
+	hp->hp_birthtime = hp->hp_lastreport = time(NULL);
+	(void)strlcpy(hp->hp_comm, path, sizeof(hp->hp_comm));
+	/* We start at 2nd argument as we don't want to have exec name twice. */
+	for (ii = 1; args[ii] != NULL; ii++) {
+		(void)strlcat(hp->hp_comm, " ", sizeof(hp->hp_comm));
+		(void)strlcat(hp->hp_comm, args[ii], sizeof(hp->hp_comm));
+	}
+	if (strlen(hp->hp_comm) >= sizeof(hp->hp_comm) - 1) {
+		pjdlog_error("Exec path too long, correct configuration file.");
+		free(hp);
+		return (NULL);
+	}
+	hp->hp_magic = HOOKPROC_MAGIC_ALLOCATED;
+	return (hp);
+}
+
+static void
+hook_add(struct hookproc *hp, pid_t pid)
+{
+
+	assert(hp->hp_magic == HOOKPROC_MAGIC_ALLOCATED);
+	assert(hp->hp_pid == 0);
+
+	hp->hp_pid = pid;
+	mtx_lock(&hookprocs_lock);
+	hp->hp_magic = HOOKPROC_MAGIC_ONLIST;
+	TAILQ_INSERT_TAIL(&hookprocs, hp, hp_next);
+	mtx_unlock(&hookprocs_lock);
+}
+
+static void
+hook_remove(struct hookproc *hp)
+{
+
+	assert(hp->hp_magic == HOOKPROC_MAGIC_ONLIST);
+	assert(hp->hp_pid > 0);
+	assert(mtx_owned(&hookprocs_lock));
+
+	TAILQ_REMOVE(&hookprocs, hp, hp_next);
+	hp->hp_magic = HOOKPROC_MAGIC_ALLOCATED;
+}
+
+static void
+hook_free(struct hookproc *hp)
+{
+
+	assert(hp->hp_magic == HOOKPROC_MAGIC_ALLOCATED);
+	assert(hp->hp_pid > 0);
+
+	hp->hp_magic = 0;
+	free(hp);
+}
+
+static struct hookproc *
+hook_find(pid_t pid)
+{
+	struct hookproc *hp;
+
+	assert(mtx_owned(&hookprocs_lock));
+
+	TAILQ_FOREACH(hp, &hookprocs, hp_next) {
+		assert(hp->hp_magic == HOOKPROC_MAGIC_ONLIST);
+		assert(hp->hp_pid > 0);
+
+		if (hp->hp_pid == pid)
+			break;
+	}
+
+	return (hp);
+}
+
+void
+hook_check_one(pid_t pid, int status)
+{
+	struct hookproc *hp;
+
+	mtx_lock(&hookprocs_lock);
+	hp = hook_find(pid);
+	if (hp == NULL) {
+		mtx_unlock(&hookprocs_lock);
+		pjdlog_debug(1, "Unknown process pid=%u", pid);
+		return;
+	}
+	hook_remove(hp);
+	mtx_unlock(&hookprocs_lock);
+	if (WIFEXITED(status) && WEXITSTATUS(status) == 0) {
+		pjdlog_debug(1, "Hook exited gracefully (pid=%u, cmd=[%s]).",
+		    pid, hp->hp_comm);
+	} else if (WIFSIGNALED(status)) {
+		pjdlog_error("Hook was killed (pid=%u, signal=%d, cmd=[%s]).",
+		    pid, WTERMSIG(status), hp->hp_comm);
+	} else {
+		pjdlog_error("Hook exited ungracefully (pid=%u, exitcode=%d, cmd=[%s]).",
+		    pid, WIFEXITED(status) ? WEXITSTATUS(status) : -1,
+		    hp->hp_comm);
+	}
+	hook_free(hp);
+}
+
+void
+hook_check(bool sigchld)
+{
+	struct hookproc *hp, *hp2;
+	int status;
+	time_t now;
+	pid_t pid;
+
+	assert(hooks_initialized);
+
+	/*
+	 * If SIGCHLD was received, garbage collect finished processes.
+	 */
+	if (sigchld) {
+		while ((pid = wait3(&status, WNOHANG, NULL)) > 0)
+			hook_check_one(pid, status);
+	}
+
+	/*
+	 * Report about processes that are running for a long time.
+	 */
+	now = time(NULL);
+	mtx_lock(&hookprocs_lock);
+	TAILQ_FOREACH_SAFE(hp, &hookprocs, hp_next, hp2) {
+		assert(hp->hp_magic == HOOKPROC_MAGIC_ONLIST);
+		assert(hp->hp_pid > 0);
+
+		/*
+		 * If process doesn't exists we somehow missed it.
+		 * Not much can be done expect for logging this situation.
+		 */
+		if (kill(hp->hp_pid, 0) == -1 && errno == ESRCH) {
+			pjdlog_warning("Hook disappeared (pid=%u, cmd=[%s]).",
+			    hp->hp_pid, hp->hp_comm);
+			hook_remove(hp);
+			hook_free(hp);
+			continue;
+		}
+
+		/*
+		 * Skip proccesses younger than 1 minute.
+		 */
+		if (now - hp->hp_lastreport < REPORT_INTERVAL)
+			continue;
+
+		/*
+		 * Hook is running for too long, report it.
+		 */
+		pjdlog_warning("Hook is running for %ju seconds (pid=%u, cmd=[%s]).",
+		    (uintmax_t)(now - hp->hp_birthtime), hp->hp_pid,
+		    hp->hp_comm);
+		hp->hp_lastreport = now;
+	}
+	mtx_unlock(&hookprocs_lock);
+}
+
+void
 hook_exec(const char *path, ...)
 {
 	va_list ap;
-	int ret;
 
 	va_start(ap, path);
-	ret = hook_execv(path, ap);
+	hook_execv(path, ap);
 	va_end(ap);
-	return (ret);
 }
 
-int
+void
 hook_execv(const char *path, va_list ap)
 {
+	struct hookproc *hp;
 	char *args[64];
 	unsigned int ii;
-	pid_t pid, wpid;
-	int status;
+	pid_t pid;
+
+	assert(hooks_initialized);
 
 	if (path == NULL || path[0] == '\0')
-		return (0);
+		return;
 
 	memset(args, 0, sizeof(args));
 	args[0] = basename(path);
@@ -127,22 +380,22 @@ hook_execv(const char *path, va_list ap)
 	}
 	assert(ii < sizeof(args) / sizeof(args[0]));
 
+	hp = hook_alloc(path, args);
+	if (hp == NULL)
+		return;
+
 	pid = fork();
 	switch (pid) {
 	case -1:	/* Error. */
-		pjdlog_errno(LOG_ERR, "Unable to fork %s", path);
-		return (-1);
+		pjdlog_errno(LOG_ERR, "Unable to fork to execute %s", path);
+		return;
 	case 0:		/* Child. */
 		descriptors();
 		execv(path, args);
 		pjdlog_errno(LOG_ERR, "Unable to execute %s", path);
 		exit(EX_SOFTWARE);
 	default:	/* Parent. */
+		hook_add(hp, pid);
 		break;
 	}
-
-	wpid = waitpid(pid, &status, 0);
-	assert(wpid == pid);
-
-	return (WEXITSTATUS(status));
 }
diff --git a/sbin/hastd/hooks.h b/sbin/hastd/hooks.h
index 799b781d319f..5cc57df5e2bc 100644
--- a/sbin/hastd/hooks.h
+++ b/sbin/hastd/hooks.h
@@ -1,5 +1,6 @@
 /*-
  * Copyright (c) 2010 The FreeBSD Foundation
+ * Copyright (c) 2010 Pawel Jakub Dawidek <pjd@FreeBSD.org>
  * All rights reserved.
  *
  * This software was developed by Pawel Jakub Dawidek under sponsorship from
@@ -32,9 +33,16 @@
 #ifndef	_HOOKS_H_
 #define	_HOOKS_H_
 
+#include <sys/types.h>
+
 #include <stdarg.h>
+#include <stdbool.h>
 
-int hook_exec(const char *path, ...);
-int hook_execv(const char *path, va_list ap);
+void hook_init(void);
+void hook_fini(void);
+void hook_check_one(pid_t pid, int status);
+void hook_check(bool sigchld);
+void hook_exec(const char *path, ...);
+void hook_execv(const char *path, va_list ap);
 
 #endif	/* !_HOOKS_H_ */
diff --git a/sbin/hastd/parse.y b/sbin/hastd/parse.y
index 840a84489513..ca575cf1238b 100644
--- a/sbin/hastd/parse.y
+++ b/sbin/hastd/parse.y
@@ -43,6 +43,8 @@
 #include <sysexits.h>
 #include <unistd.h>
 
+#include <pjdlog.h>
+
 #include "hast.h"
 
 extern int depth;
@@ -51,7 +53,7 @@ extern int lineno;
 extern FILE *yyin;
 extern char *yytext;
 
-static struct hastd_config lconfig;
+static struct hastd_config *lconfig;
 static struct hast_resource *curres;
 static bool mynode;
 
@@ -59,11 +61,14 @@ static char depth0_control[HAST_ADDRSIZE];
 static char depth0_listen[HAST_ADDRSIZE];
 static int depth0_replication;
 static int depth0_timeout;
+static char depth0_exec[PATH_MAX];
 
 static char depth1_provname[PATH_MAX];
 static char depth1_localpath[PATH_MAX];
 
-static bool
+extern void yyrestart(FILE *);
+
+static int
 isitme(const char *name)
 {
 	char buf[MAXHOSTNAMELEN];
@@ -73,78 +78,102 @@ isitme(const char *name)
 	/*
 	 * First check if the give name matches our full hostname.
 	 */
-	if (gethostname(buf, sizeof(buf)) < 0)
-		err(EX_OSERR, "gethostname() failed");
+	if (gethostname(buf, sizeof(buf)) < 0) {
+		pjdlog_errno(LOG_ERR, "gethostname() failed");
+		return (-1);
+	}
 	if (strcmp(buf, name) == 0)
-		return (true);
+		return (1);
 
 	/*
 	 * Now check if it matches first part of the host name.
 	 */
 	pos = strchr(buf, '.');
 	if (pos != NULL && pos != buf && strncmp(buf, name, pos - buf) == 0)
-		return (true);
+		return (1);
 
 	/*
 	 * At the end check if name is equal to our host's UUID.
 	 */
 	bufsize = sizeof(buf);
-	if (sysctlbyname("kern.hostuuid", buf, &bufsize, NULL, 0) < 0)
-		err(EX_OSERR, "sysctlbyname(kern.hostuuid) failed");
+	if (sysctlbyname("kern.hostuuid", buf, &bufsize, NULL, 0) < 0) {
+		pjdlog_errno(LOG_ERR, "sysctlbyname(kern.hostuuid) failed");
+		return (-1);
+	}
 	if (strcasecmp(buf, name) == 0)
-		return (true);
+		return (1);
 
 	/*
 	 * Looks like this isn't about us.
 	 */
-	return (false);
+	return (0);
 }
 
 void
 yyerror(const char *str)
 {
 
-	fprintf(stderr, "error at line %d near '%s': %s\n",
+	pjdlog_error("Unable to parse configuration file at line %d near '%s': %s",
 	    lineno, yytext, str);
 }
 
 struct hastd_config *
-yy_config_parse(const char *config)
+yy_config_parse(const char *config, bool exitonerror)
 {
 	int ret;
 
 	curres = NULL;
 	mynode = false;
+	depth = 0;
+	lineno = 0;
 
 	depth0_timeout = HAST_TIMEOUT;
 	depth0_replication = HAST_REPLICATION_MEMSYNC;
 	strlcpy(depth0_control, HAST_CONTROL, sizeof(depth0_control));
 	strlcpy(depth0_listen, HASTD_LISTEN, sizeof(depth0_listen));
+	depth0_exec[0] = '\0';
+
+	lconfig = calloc(1, sizeof(*lconfig));
+	if (lconfig == NULL) {
+		pjdlog_error("Unable to allocate memory for configuration.");
+		if (exitonerror)
+			exit(EX_TEMPFAIL);
+		return (NULL);
+	}
 
-	TAILQ_INIT(&lconfig.hc_resources);
+	TAILQ_INIT(&lconfig->hc_resources);
 
 	yyin = fopen(config, "r");
-	if (yyin == NULL)
-		err(EX_OSFILE, "cannot open configuration file %s", config);
+	if (yyin == NULL) {
+		pjdlog_errno(LOG_ERR, "Unable to open configuration file %s",
+		    config);
+		yy_config_free(lconfig);
+		if (exitonerror)
+			exit(EX_OSFILE);
+		return (NULL);
+	}
+	yyrestart(yyin);
 	ret = yyparse();
 	fclose(yyin);
 	if (ret != 0) {
-		yy_config_free(&lconfig);
-		exit(EX_CONFIG);
+		yy_config_free(lconfig);
+		if (exitonerror)
+			exit(EX_CONFIG);
+		return (NULL);
 	}
 
 	/*
 	 * Let's see if everything is set up.
 	 */
-	if (lconfig.hc_controladdr[0] == '\0') {
-		strlcpy(lconfig.hc_controladdr, depth0_control,
-		    sizeof(lconfig.hc_controladdr));
+	if (lconfig->hc_controladdr[0] == '\0') {
+		strlcpy(lconfig->hc_controladdr, depth0_control,
+		    sizeof(lconfig->hc_controladdr));
 	}
-	if (lconfig.hc_listenaddr[0] == '\0') {
-		strlcpy(lconfig.hc_listenaddr, depth0_listen,
-		    sizeof(lconfig.hc_listenaddr));
+	if (lconfig->hc_listenaddr[0] == '\0') {
+		strlcpy(lconfig->hc_listenaddr, depth0_listen,
+		    sizeof(lconfig->hc_listenaddr));
 	}
-	TAILQ_FOREACH(curres, &lconfig.hc_resources, hr_next) {
+	TAILQ_FOREACH(curres, &lconfig->hc_resources, hr_next) {
 		assert(curres->hr_provname[0] != '\0');
 		assert(curres->hr_localpath[0] != '\0');
 		assert(curres->hr_remoteaddr[0] != '\0');
@@ -163,9 +192,17 @@ yy_config_parse(const char *config)
 			 */
 			curres->hr_timeout = depth0_timeout;
 		}
+		if (curres->hr_exec[0] == '\0') {
+			/*
+			 * Exec is not set at resource-level.
+			 * Use global or default setting.
+			 */
+			strlcpy(curres->hr_exec, depth0_exec,
+			    sizeof(curres->hr_exec));
+		}
 	}
 
-	return (&lconfig);
+	return (lconfig);
 }
 
 void
@@ -177,10 +214,11 @@ yy_config_free(struct hastd_config *config)
 		TAILQ_REMOVE(&config->hc_resources, res, hr_next);
 		free(res);
 	}
+	free(config);
 }
 %}
 
-%token CONTROL LISTEN PORT REPLICATION TIMEOUT EXTENTSIZE RESOURCE NAME LOCAL REMOTE ON
+%token CONTROL LISTEN PORT REPLICATION TIMEOUT EXEC EXTENTSIZE RESOURCE NAME LOCAL REMOTE ON
 %token FULLSYNC MEMSYNC ASYNC
 %token NUM STR OB CB
 
@@ -211,6 +249,8 @@ statement:
 	|
 	timeout_statement
 	|
+	exec_statement
+	|
 	node_statement
 	|
 	resource_statement
@@ -223,17 +263,18 @@ control_statement:	CONTROL STR
 			if (strlcpy(depth0_control, $2,
 			    sizeof(depth0_control)) >=
 			    sizeof(depth0_control)) {
-				errx(EX_CONFIG, "control argument too long");
+				pjdlog_error("control argument is too long.");
+				return (1);
 			}
 			break;
 		case 1:
-			if (mynode) {
-				if (strlcpy(lconfig.hc_controladdr, $2,
-				    sizeof(lconfig.hc_controladdr)) >=
-				    sizeof(lconfig.hc_controladdr)) {
-					errx(EX_CONFIG,
-					    "control argument too long");
-				}
+			if (!mynode)
+				break;
+			if (strlcpy(lconfig->hc_controladdr, $2,
+			    sizeof(lconfig->hc_controladdr)) >=
+			    sizeof(lconfig->hc_controladdr)) {
+				pjdlog_error("control argument is too long.");
+				return (1);
 			}
 			break;
 		default:
@@ -249,17 +290,18 @@ listen_statement:	LISTEN STR
 			if (strlcpy(depth0_listen, $2,
 			    sizeof(depth0_listen)) >=
 			    sizeof(depth0_listen)) {
-				errx(EX_CONFIG, "listen argument too long");
+				pjdlog_error("listen argument is too long.");
+				return (1);
 			}
 			break;
 		case 1:
-			if (mynode) {
-				if (strlcpy(lconfig.hc_listenaddr, $2,
-				    sizeof(lconfig.hc_listenaddr)) >=
-				    sizeof(lconfig.hc_listenaddr)) {
-					errx(EX_CONFIG,
-					    "listen argument too long");
-				}
+			if (!mynode)
+				break;
+			if (strlcpy(lconfig->hc_listenaddr, $2,
+			    sizeof(lconfig->hc_listenaddr)) >=
+			    sizeof(lconfig->hc_listenaddr)) {
+				pjdlog_error("listen argument is too long.");
+				return (1);
 			}
 			break;
 		default:
@@ -308,6 +350,32 @@ timeout_statement:	TIMEOUT NUM
 	}
 	;
 
+exec_statement:		EXEC STR
+	{
+		switch (depth) {
+		case 0:
+			if (strlcpy(depth0_exec, $2, sizeof(depth0_exec)) >=
+			    sizeof(depth0_exec)) {
+				pjdlog_error("Exec path is too long.");
+				return (1);
+			}
+			break;
+		case 1:
+			if (curres == NULL)
+				break;
+			if (strlcpy(curres->hr_exec, $2,
+			    sizeof(curres->hr_exec)) >=
+			    sizeof(curres->hr_exec)) {
+				pjdlog_error("Exec path is too long.");
+				return (1);
+			}
+			break;
+		default:
+			assert(!"exec at wrong depth level");
+		}
+	}
+	;
+
 node_statement:		ON node_start OB node_entries CB
 	{
 		mynode = false;
@@ -316,8 +384,17 @@ node_statement:		ON node_start OB node_entries CB
 
 node_start:	STR
 	{
-		if (isitme($1))
+		switch (isitme($1)) {
+		case -1:
+			return (1);
+		case 0:
+			break;
+		case 1:
 			mynode = true;
+			break;
+		default:
+			assert(!"invalid isitme() return value");
+		}
 	}
 	;
 
@@ -372,22 +449,22 @@ resource_statement:	RESOURCE resource_start OB resource_entries CB
 			 * Remote address has to be configured at this point.
 			 */
 			if (curres->hr_remoteaddr[0] == '\0') {
-				errx(EX_CONFIG,
-				    "remote address not configured for resource %s",
+				pjdlog_error("Remote address not configured for resource %s.",
 				    curres->hr_name);
+				return (1);
 			}
 			/*
 			 * Path to local provider has to be configured at this
 			 * point.
 			 */
 			if (curres->hr_localpath[0] == '\0') {
-				errx(EX_CONFIG,
-				    "path local component not configured for resource %s",
+				pjdlog_error("Path to local component not configured for resource %s.",
 				    curres->hr_name);
+				return (1);
 			}
 
 			/* Put it onto resource list. */
-			TAILQ_INSERT_TAIL(&lconfig.hc_resources, curres, hr_next);
+			TAILQ_INSERT_TAIL(&lconfig->hc_resources, curres, hr_next);
 			curres = NULL;
 		}
 	}
@@ -404,19 +481,20 @@ resource_start:	STR
 
 		curres = calloc(1, sizeof(*curres));
 		if (curres == NULL) {
-			errx(EX_TEMPFAIL,
-			    "cannot allocate memory for resource");
+			pjdlog_error("Unable to allocate memory for resource.");
+			return (1);
 		}
 		if (strlcpy(curres->hr_name, $1,
 		    sizeof(curres->hr_name)) >=
 		    sizeof(curres->hr_name)) {
-			errx(EX_CONFIG,
-			    "resource name (%s) too long", $1);
+			pjdlog_error("Resource name is too long.");
+			return (1);
 		}
 		curres->hr_role = HAST_ROLE_INIT;
 		curres->hr_previous_role = HAST_ROLE_INIT;
 		curres->hr_replication = -1;
 		curres->hr_timeout = -1;
+		curres->hr_exec[0] = '\0';
 		curres->hr_provname[0] = '\0';
 		curres->hr_localpath[0] = '\0';
 		curres->hr_localfd = -1;
@@ -435,6 +513,8 @@ resource_entry:
 	|
 	timeout_statement
 	|
+	exec_statement
+	|
 	name_statement
 	|
 	local_statement
@@ -449,18 +529,19 @@ name_statement:		NAME STR
 			if (strlcpy(depth1_provname, $2,
 			    sizeof(depth1_provname)) >=
 			    sizeof(depth1_provname)) {
-				errx(EX_CONFIG, "name argument too long");
+				pjdlog_error("name argument is too long.");
+				return (1);
 			}
 			break;
 		case 2:
-			if (mynode) {
-				assert(curres != NULL);
-				if (strlcpy(curres->hr_provname, $2,
-				    sizeof(curres->hr_provname)) >=
-				    sizeof(curres->hr_provname)) {
-					errx(EX_CONFIG,
-					    "name argument too long");
-				}
+			if (!mynode)
+				break;
+			assert(curres != NULL);
+			if (strlcpy(curres->hr_provname, $2,
+			    sizeof(curres->hr_provname)) >=
+			    sizeof(curres->hr_provname)) {
+				pjdlog_error("name argument is too long.");
+				return (1);
 			}
 			break;
 		default:
@@ -476,18 +557,19 @@ local_statement:	LOCAL STR
 			if (strlcpy(depth1_localpath, $2,
 			    sizeof(depth1_localpath)) >=
 			    sizeof(depth1_localpath)) {
-				errx(EX_CONFIG, "local argument too long");
+				pjdlog_error("local argument is too long.");
+				return (1);
 			}
 			break;
 		case 2:
-			if (mynode) {
-				assert(curres != NULL);
-				if (strlcpy(curres->hr_localpath, $2,
-				    sizeof(curres->hr_localpath)) >=
-				    sizeof(curres->hr_localpath)) {
-					errx(EX_CONFIG,
-					    "local argument too long");
-				}
+			if (!mynode)
+				break;
+			assert(curres != NULL);
+			if (strlcpy(curres->hr_localpath, $2,
+			    sizeof(curres->hr_localpath)) >=
+			    sizeof(curres->hr_localpath)) {
+				pjdlog_error("local argument is too long.");
+				return (1);
 			}
 			break;
 		default:
@@ -504,8 +586,19 @@ resource_node_statement:ON resource_node_start OB resource_node_entries CB
 
 resource_node_start:	STR
 	{
-		if (curres != NULL && isitme($1))
-			mynode = true;
+		if (curres != NULL) {
+			switch (isitme($1)) {
+			case -1:
+				return (1);
+			case 0:
+				break;
+			case 1:
+				mynode = true;
+				break;
+			default:
+				assert(!"invalid isitme() return value");
+			}
+		}
 	}
 	;
 
@@ -530,7 +623,8 @@ remote_statement:	REMOTE STR
 			if (strlcpy(curres->hr_remoteaddr, $2,
 			    sizeof(curres->hr_remoteaddr)) >=
 			    sizeof(curres->hr_remoteaddr)) {
-				errx(EX_CONFIG, "remote argument too long");
+				pjdlog_error("remote argument is too long.");
+				return (1);
 			}
 		}
 	}
diff --git a/sbin/hastd/pjdlog.c b/sbin/hastd/pjdlog.c
index 9f8b3f499de7..891210586291 100644
--- a/sbin/hastd/pjdlog.c
+++ b/sbin/hastd/pjdlog.c
@@ -25,8 +25,6 @@
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
- *
- * $FreeBSD$
  */
 
 #include <sys/cdefs.h>
@@ -59,6 +57,9 @@ pjdlog_mode_set(int mode)
 	assert(mode == PJDLOG_MODE_STD || mode == PJDLOG_MODE_SYSLOG);
 
 	pjdlog_mode = mode;
+
+	if (mode == PJDLOG_MODE_SYSLOG)
+		openlog(NULL, LOG_PID, LOG_DAEMON);
 }
 
 /*
@@ -219,6 +220,7 @@ pjdlogv_common(int loglevel, int debuglevel, int error, const char *fmt,
 		if (error != -1)
 			fprintf(out, ": %s.", strerror(error));
 		fprintf(out, "\n");
+		fflush(out);
 		break;
 	    }
 	case PJDLOG_MODE_SYSLOG:
@@ -325,6 +327,7 @@ pjdlogv_exit(int exitcode, const char *fmt, va_list ap)
 
 	pjdlogv_errno(LOG_ERR, fmt, ap);
 	exit(exitcode);
+	/* NOTREACHED */
 }
 
 /*
@@ -350,6 +353,7 @@ pjdlogv_exitx(int exitcode, const char *fmt, va_list ap)
 
 	pjdlogv(LOG_ERR, fmt, ap);
 	exit(exitcode);
+	/* NOTREACHED */
 }
 
 /*
@@ -365,3 +369,23 @@ pjdlog_exitx(int exitcode, const char *fmt, ...)
 	/* NOTREACHED */
 	va_end(ap);
 }
+
+/*
+ * Log assertion and exit.
+ */
+void
+pjdlog_verify(const char *func, const char *file, int line,
+    const char *failedexpr)
+{
+
+	if (func == NULL) {
+		pjdlog_critical("Assertion failed: (%s), file %s, line %d.",
+		    failedexpr, file, line);
+	} else {
+		pjdlog_critical("Assertion failed: (%s), function %s, file %s, line %d.",
+		    failedexpr, func, file, line);
+	}
+	abort();
+        /* NOTREACHED */
+}
+
diff --git a/sbin/hastd/pjdlog.h b/sbin/hastd/pjdlog.h
index 2136b12f42ae..28b49dec7d78 100644
--- a/sbin/hastd/pjdlog.h
+++ b/sbin/hastd/pjdlog.h
@@ -85,4 +85,17 @@ void pjdlogv_exit(int exitcode, const char *fmt, va_list ap) __printflike(2, 0)
 void pjdlog_exitx(int exitcode, const char *fmt, ...) __printflike(2, 3) __dead2;
 void pjdlogv_exitx(int exitcode, const char *fmt, va_list ap) __printflike(2, 0) __dead2;
 
+void pjdlog_verify(const char *func, const char *file, int line,
+    const char *failedexpr) __dead2;
+
+#define	PJDLOG_VERIFY(expr)	do {					\
+	if (!(expr))							\
+		pjdlog_verify(__func__, __FILE__, __LINE__, #expr);	\
+} while (0)
+#ifdef NDEBUG
+#define	PJDLOG_ASSERT(expr)	do { } while (0)
+#else
+#define	PJDLOG_ASSERT(expr)	PJDLOG_VERIFY(expr)
+#endif
+
 #endif	/* !_PJDLOG_H_ */
diff --git a/sbin/hastd/primary.c b/sbin/hastd/primary.c
index ae7422044480..d99bfd7ae775 100644
--- a/sbin/hastd/primary.c
+++ b/sbin/hastd/primary.c
@@ -1,5 +1,6 @@
 /*-
  * Copyright (c) 2009 The FreeBSD Foundation
+ * Copyright (c) 2010 Pawel Jakub Dawidek <pjd@FreeBSD.org>
  * All rights reserved.
  *
  * This software was developed by Pawel Jakub Dawidek under sponsorship from
@@ -45,6 +46,7 @@ __FBSDID("$FreeBSD$");
 #include <fcntl.h>
 #include <libgeom.h>
 #include <pthread.h>
+#include <signal.h>
 #include <stdint.h>
 #include <stdio.h>
 #include <string.h>
@@ -56,15 +58,20 @@ __FBSDID("$FreeBSD$");
 #include <rangelock.h>
 
 #include "control.h"
+#include "event.h"
 #include "hast.h"
 #include "hast_proto.h"
 #include "hastd.h"
+#include "hooks.h"
 #include "metadata.h"
 #include "proto.h"
 #include "pjdlog.h"
 #include "subr.h"
 #include "synch.h"
 
+/* The is only one remote component for now. */
+#define	ISREMOTE(no)	((no) == 1)
+
 struct hio {
 	/*
 	 * Number of components we are still waiting for.
@@ -127,8 +134,6 @@ static pthread_cond_t sync_cond;
  * The lock below allows to synchornize access to remote connections.
  */
 static pthread_rwlock_t *hio_remote_lock;
-static pthread_mutex_t hio_guard_lock;
-static pthread_cond_t hio_guard_cond;
 
 /*
  * Lock to synchronize metadata updates. Also synchronize access to
@@ -147,9 +152,9 @@ static pthread_mutex_t metadata_lock;
  */
 #define	HAST_NCOMPONENTS	2
 /*
- * Number of seconds to sleep before next reconnect try.
+ * Number of seconds to sleep between reconnect retries or keepalive packets.
  */
-#define	RECONNECT_SLEEP		5
+#define	RETRY_SLEEP		10
 
 #define	ISCONNECTED(res, no)	\
 	((res)->hr_remotein != NULL && (res)->hr_remoteout != NULL)
@@ -221,8 +226,6 @@ static void *ggate_send_thread(void *arg);
 static void *sync_thread(void *arg);
 static void *guard_thread(void *arg);
 
-static void sighandler(int sig);
-
 static void
 cleanup(struct hast_resource *res)
 {
@@ -255,7 +258,7 @@ cleanup(struct hast_resource *res)
 	errno = rerrno;
 }
 
-static void
+static __dead2 void
 primary_exit(int exitcode, const char *fmt, ...)
 {
 	va_list ap;
@@ -268,7 +271,7 @@ primary_exit(int exitcode, const char *fmt, ...)
 	exit(exitcode);
 }
 
-static void
+static __dead2 void
 primary_exitx(int exitcode, const char *fmt, ...)
 {
 	va_list ap;
@@ -298,6 +301,13 @@ hast_activemap_flush(struct hast_resource *res)
 	return (0);
 }
 
+static bool
+real_remote(const struct hast_resource *res)
+{
+
+	return (strcmp(res->hr_remoteaddr, "none") != 0);
+}
+
 static void
 init_environment(struct hast_resource *res __unused)
 {
@@ -373,8 +383,6 @@ init_environment(struct hast_resource *res __unused)
 	TAILQ_INIT(&hio_done_list);
 	mtx_init(&hio_done_list_lock);
 	cv_init(&hio_done_list_cond);
-	mtx_init(&hio_guard_lock);
-	cv_init(&hio_guard_cond);
 	mtx_init(&metadata_lock);
 
 	/*
@@ -411,12 +419,6 @@ init_environment(struct hast_resource *res __unused)
 		hio->hio_ggio.gctl_error = 0;
 		TAILQ_INSERT_HEAD(&hio_free_list, hio, hio_free_next);
 	}
-
-	/*
-	 * Turn on signals handling.
-	 */
-	signal(SIGINT, sighandler);
-	signal(SIGTERM, sighandler);
 }
 
 static void
@@ -479,8 +481,10 @@ init_remote(struct hast_resource *res, struct proto_conn **inp,
 	size_t size;
 
 	assert((inp == NULL && outp == NULL) || (inp != NULL && outp != NULL));
+	assert(real_remote(res));
 
 	in = out = NULL;
+	errmsg = NULL;
 
 	/* Prepare outgoing connection with remote node. */
 	if (proto_client(res->hr_remoteaddr, &out) < 0) {
@@ -654,8 +658,11 @@ init_remote(struct hast_resource *res, struct proto_conn **inp,
 		res->hr_remotein = in;
 		res->hr_remoteout = out;
 	}
+	event_send(res, EVENT_CONNECT);
 	return (true);
 close:
+	if (errmsg != NULL && strcmp(errmsg, "Split-brain condition!") == 0)
+		event_send(res, EVENT_SPLITBRAIN);
 	proto_close(out);
 	if (in != NULL)
 		proto_close(in);
@@ -673,6 +680,16 @@ sync_start(void)
 }
 
 static void
+sync_stop(void)
+{
+
+	mtx_lock(&sync_lock);
+	if (sync_inprogress)
+		sync_inprogress = false;
+	mtx_unlock(&sync_lock);
+}
+
+static void
 init_ggate(struct hast_resource *res)
 {
 	struct g_gate_ctl_create ggiocreate;
@@ -735,37 +752,62 @@ hastd_primary(struct hast_resource *res)
 	pid_t pid;
 	int error;
 
-	gres = res;
-
 	/*
 	 * Create communication channel between parent and child.
 	 */
 	if (proto_client("socketpair://", &res->hr_ctrl) < 0) {
 		KEEP_ERRNO((void)pidfile_remove(pfh));
-		primary_exit(EX_OSERR,
+		pjdlog_exit(EX_OSERR,
 		    "Unable to create control sockets between parent and child");
 	}
+	/*
+	 * Create communication channel between child and parent.
+	 */
+	if (proto_client("socketpair://", &res->hr_event) < 0) {
+		KEEP_ERRNO((void)pidfile_remove(pfh));
+		pjdlog_exit(EX_OSERR,
+		    "Unable to create event sockets between child and parent");
+	}
 
 	pid = fork();
 	if (pid < 0) {
 		KEEP_ERRNO((void)pidfile_remove(pfh));
-		primary_exit(EX_TEMPFAIL, "Unable to fork");
+		pjdlog_exit(EX_TEMPFAIL, "Unable to fork");
 	}
 
 	if (pid > 0) {
 		/* This is parent. */
+		/* Declare that we are receiver. */
+		proto_recv(res->hr_event, NULL, 0);
 		res->hr_workerpid = pid;
 		return;
 	}
+
+	gres = res;
+
 	(void)pidfile_close(pfh);
+	hook_fini();
 
 	setproctitle("%s (primary)", res->hr_name);
 
+	/* Declare that we are sender. */
+	proto_send(res->hr_event, NULL, 0);
+
 	init_local(res);
-	if (init_remote(res, NULL, NULL))
-		sync_start();
 	init_ggate(res);
 	init_environment(res);
+	/*
+	 * Create the control thread before sending any event to the parent,
+	 * as we can deadlock when parent sends control request to worker,
+	 * but worker has no control thread started yet, so parent waits.
+	 * In the meantime worker sends an event to the parent, but parent
+	 * is unable to handle the event, because it waits for control
+	 * request response.
+	 */
+	error = pthread_create(&td, NULL, ctrl_thread, res);
+	assert(error == 0);
+	if (real_remote(res) && init_remote(res, NULL, NULL))
+		sync_start();
 	error = pthread_create(&td, NULL, ggate_recv_thread, res);
 	assert(error == 0);
 	error = pthread_create(&td, NULL, local_send_thread, res);
@@ -778,8 +820,6 @@ hastd_primary(struct hast_resource *res)
 	assert(error == 0);
 	error = pthread_create(&td, NULL, sync_thread, res);
 	assert(error == 0);
-	error = pthread_create(&td, NULL, ctrl_thread, res);
-	assert(error == 0);
 	(void)guard_thread(res);
 }
 
@@ -841,31 +881,25 @@ remote_close(struct hast_resource *res, int ncomp)
 	assert(res->hr_remotein != NULL);
 	assert(res->hr_remoteout != NULL);
 
-	pjdlog_debug(2, "Closing old incoming connection to %s.",
+	pjdlog_debug(2, "Closing incoming connection to %s.",
 	    res->hr_remoteaddr);
 	proto_close(res->hr_remotein);
 	res->hr_remotein = NULL;
-	pjdlog_debug(2, "Closing old outgoing connection to %s.",
+	pjdlog_debug(2, "Closing outgoing connection to %s.",
 	    res->hr_remoteaddr);
 	proto_close(res->hr_remoteout);
 	res->hr_remoteout = NULL;
 
 	rw_unlock(&hio_remote_lock[ncomp]);
 
+	pjdlog_warning("Disconnected from %s.", res->hr_remoteaddr);
+
 	/*
 	 * Stop synchronization if in-progress.
 	 */
-	mtx_lock(&sync_lock);
-	if (sync_inprogress)
-		sync_inprogress = false;
-	mtx_unlock(&sync_lock);
+	sync_stop();
 
-	/*
-	 * Wake up guard thread, so it can immediately start reconnect.
-	 */
-	mtx_lock(&hio_guard_lock);
-	cv_signal(&hio_guard_cond);
-	mtx_unlock(&hio_guard_lock);
+	event_send(res, EVENT_DISCONNECT);
 }
 
 /*
@@ -1197,12 +1231,12 @@ remote_send_thread(void *arg)
 		    data != NULL ? length : 0) < 0) {
 			hio->hio_errors[ncomp] = errno;
 			rw_unlock(&hio_remote_lock[ncomp]);
-			remote_close(res, ncomp);
 			pjdlog_debug(2,
 			    "remote_send: (%p) Unable to send request.", hio);
 			reqlog(LOG_ERR, 0, ggio,
 			    "Unable to send request (%s): ",
 			    strerror(hio->hio_errors[ncomp]));
+			remote_close(res, ncomp);
 			/*
 			 * Take request back from the receive queue and move
 			 * it immediately to the done queue.
@@ -1473,9 +1507,16 @@ sync_thread(void *arg __unused)
 	ncomps = HAST_NCOMPONENTS;
 	dorewind = true;
 	synced = 0;
+	offset = -1;
 
 	for (;;) {
 		mtx_lock(&sync_lock);
+		if (offset >= 0 && !sync_inprogress) {
+			pjdlog_info("Synchronization interrupted. "
+			    "%jd bytes synchronized so far.",
+			    (intmax_t)synced);
+			event_send(res, EVENT_SYNCINTR);
+		}
 		while (!sync_inprogress) {
 			dorewind = true;
 			synced = 0;
@@ -1507,12 +1548,11 @@ sync_thread(void *arg __unused)
 				pjdlog_info("Synchronization started. %ju bytes to go.",
 				    (uintmax_t)(res->hr_extentsize *
 				    activemap_ndirty(res->hr_amp)));
+				event_send(res, EVENT_SYNCSTART);
 			}
 		}
 		if (offset < 0) {
-			mtx_lock(&sync_lock);
-			sync_inprogress = false;
-			mtx_unlock(&sync_lock);
+			sync_stop();
 			pjdlog_debug(1, "Nothing to synchronize.");
 			/*
 			 * Synchronization complete, make both localcnt and
@@ -1525,6 +1565,7 @@ sync_thread(void *arg __unused)
 					pjdlog_info("Synchronization complete. "
 					    "%jd bytes synchronized.",
 					    (intmax_t)synced);
+					event_send(res, EVENT_SYNCDONE);
 				}
 				mtx_lock(&metadata_lock);
 				res->hr_syncsrc = HAST_SYNCSRC_UNDEF;
@@ -1538,10 +1579,6 @@ sync_thread(void *arg __unused)
 				    (uintmax_t)res->hr_secondary_localcnt);
 				(void)metadata_write(res);
 				mtx_unlock(&metadata_lock);
-			} else if (synced > 0) {
-				pjdlog_info("Synchronization interrupted. "
-				    "%jd bytes synchronized so far.",
-				    (intmax_t)synced);
 			}
 			rw_unlock(&hio_remote_lock[ncomp]);
 			continue;
@@ -1675,15 +1712,14 @@ sync_thread(void *arg __unused)
 			    strerror(hio->hio_errors[ncomp]));
 			goto free_queue;
 		}
+
+		synced += length;
 free_queue:
 		mtx_lock(&range_lock);
 		rangelock_del(range_sync, offset, length);
 		if (range_regular_wait)
 			cv_signal(&range_regular_cond);
 		mtx_unlock(&range_lock);
-
-		synced += length;
-
 		pjdlog_debug(2, "sync: (%p) Moving request to the free queue.",
 		    hio);
 		QUEUE_INSERT2(hio, free);
@@ -1693,26 +1729,205 @@ free_queue:
 }
 
 static void
-sighandler(int sig)
+config_reload(void)
+{
+	struct hastd_config *newcfg;
+	struct hast_resource *res;
+	unsigned int ii, ncomps;
+	int modified;
+
+	pjdlog_info("Reloading configuration...");
+
+	ncomps = HAST_NCOMPONENTS;
+
+	newcfg = yy_config_parse(cfgpath, false);
+	if (newcfg == NULL)
+		goto failed;
+
+	TAILQ_FOREACH(res, &newcfg->hc_resources, hr_next) {
+		if (strcmp(res->hr_name, gres->hr_name) == 0)
+			break;
+	}
+	/*
+	 * If resource was removed from the configuration file, resource
+	 * name, provider name or path to local component was modified we
+	 * shouldn't be here. This means that someone modified configuration
+	 * file and send SIGHUP to us instead of main hastd process.
+	 * Log advice and ignore the signal.
+	 */
+	if (res == NULL || strcmp(gres->hr_name, res->hr_name) != 0 ||
+	    strcmp(gres->hr_provname, res->hr_provname) != 0 ||
+	    strcmp(gres->hr_localpath, res->hr_localpath) != 0) {
+		pjdlog_warning("To reload configuration send SIGHUP to the main hastd process (pid %u).",
+		    (unsigned int)getppid());
+		goto failed;
+	}
+
+#define MODIFIED_REMOTEADDR	0x1
+#define MODIFIED_REPLICATION	0x2
+#define MODIFIED_TIMEOUT	0x4
+#define MODIFIED_EXEC		0x8
+	modified = 0;
+	if (strcmp(gres->hr_remoteaddr, res->hr_remoteaddr) != 0) {
+		/*
+		 * Don't copy res->hr_remoteaddr to gres just yet.
+		 * We want remote_close() to log disconnect from the old
+		 * addresses, not from the new ones.
+		 */
+		modified |= MODIFIED_REMOTEADDR;
+	}
+	if (gres->hr_replication != res->hr_replication) {
+		gres->hr_replication = res->hr_replication;
+		modified |= MODIFIED_REPLICATION;
+	}
+	if (gres->hr_timeout != res->hr_timeout) {
+		gres->hr_timeout = res->hr_timeout;
+		modified |= MODIFIED_TIMEOUT;
+	}
+	if (strcmp(gres->hr_exec, res->hr_exec) != 0) {
+		strlcpy(gres->hr_exec, res->hr_exec, sizeof(gres->hr_exec));
+		modified |= MODIFIED_EXEC;
+	}
+	/*
+	 * If only timeout was modified we only need to change it without
+	 * reconnecting.
+	 */
+	if (modified == MODIFIED_TIMEOUT) {
+		for (ii = 0; ii < ncomps; ii++) {
+			if (!ISREMOTE(ii))
+				continue;
+			rw_rlock(&hio_remote_lock[ii]);
+			if (!ISCONNECTED(gres, ii)) {
+				rw_unlock(&hio_remote_lock[ii]);
+				continue;
+			}
+			rw_unlock(&hio_remote_lock[ii]);
+			if (proto_timeout(gres->hr_remotein,
+			    gres->hr_timeout) < 0) {
+				pjdlog_errno(LOG_WARNING,
+				    "Unable to set connection timeout");
+			}
+			if (proto_timeout(gres->hr_remoteout,
+			    gres->hr_timeout) < 0) {
+				pjdlog_errno(LOG_WARNING,
+				    "Unable to set connection timeout");
+			}
+		}
+	} else if ((modified &
+	    (MODIFIED_REMOTEADDR | MODIFIED_REPLICATION)) != 0) {
+		for (ii = 0; ii < ncomps; ii++) {
+			if (!ISREMOTE(ii))
+				continue;
+			remote_close(gres, ii);
+		}
+		if (modified & MODIFIED_REMOTEADDR) {
+			strlcpy(gres->hr_remoteaddr, res->hr_remoteaddr,
+			    sizeof(gres->hr_remoteaddr));
+		}
+	}
+#undef	MODIFIED_REMOTEADDR
+#undef	MODIFIED_REPLICATION
+#undef	MODIFIED_TIMEOUT
+#undef	MODIFIED_EXEC
+
+	pjdlog_info("Configuration reloaded successfully.");
+	return;
+failed:
+	if (newcfg != NULL) {
+		if (newcfg->hc_controlconn != NULL)
+			proto_close(newcfg->hc_controlconn);
+		if (newcfg->hc_listenconn != NULL)
+			proto_close(newcfg->hc_listenconn);
+		yy_config_free(newcfg);
+	}
+	pjdlog_warning("Configuration not reloaded.");
+}
+
+static void
+keepalive_send(struct hast_resource *res, unsigned int ncomp)
+{
+	struct nv *nv;
+
+	nv = nv_alloc();
+	nv_add_uint8(nv, HIO_KEEPALIVE, "cmd");
+	if (nv_error(nv) != 0) {
+		nv_free(nv);
+		pjdlog_debug(1,
+		    "keepalive_send: Unable to prepare header to send.");
+		return;
+	}
+	if (hast_proto_send(res, res->hr_remoteout, nv, NULL, 0) < 0) {
+		pjdlog_common(LOG_DEBUG, 1, errno,
+		    "keepalive_send: Unable to send request");
+		nv_free(nv);
+		rw_unlock(&hio_remote_lock[ncomp]);
+		remote_close(res, ncomp);
+		rw_rlock(&hio_remote_lock[ncomp]);
+		return;
+	}
+	nv_free(nv);
+	pjdlog_debug(2, "keepalive_send: Request sent.");
+}
+
+static void
+guard_one(struct hast_resource *res, unsigned int ncomp)
 {
-	bool unlock;
+	struct proto_conn *in, *out;
+
+	if (!ISREMOTE(ncomp))
+		return;
 
-	switch (sig) {
-	case SIGINT:
-	case SIGTERM:
-		sigexit_received = true;
-		break;
-	default:
-		assert(!"invalid condition");
+	rw_rlock(&hio_remote_lock[ncomp]);
+
+	if (!real_remote(res)) {
+		rw_unlock(&hio_remote_lock[ncomp]);
+		return;
+	}
+
+	if (ISCONNECTED(res, ncomp)) {
+		assert(res->hr_remotein != NULL);
+		assert(res->hr_remoteout != NULL);
+		keepalive_send(res, ncomp);
+	}
+
+	if (ISCONNECTED(res, ncomp)) {
+		assert(res->hr_remotein != NULL);
+		assert(res->hr_remoteout != NULL);
+		rw_unlock(&hio_remote_lock[ncomp]);
+		pjdlog_debug(2, "remote_guard: Connection to %s is ok.",
+		    res->hr_remoteaddr);
+		return;
 	}
+
+	assert(res->hr_remotein == NULL);
+	assert(res->hr_remoteout == NULL);
 	/*
-	 * XXX: Racy, but if we cannot obtain hio_guard_lock here, we don't
-	 * want to risk deadlock.
+	 * Upgrade the lock. It doesn't have to be atomic as no other thread
+	 * can change connection status from disconnected to connected.
 	 */
-	unlock = mtx_trylock(&hio_guard_lock);
-	cv_signal(&hio_guard_cond);
-	if (unlock)
-		mtx_unlock(&hio_guard_lock);
+	rw_unlock(&hio_remote_lock[ncomp]);
+	pjdlog_debug(2, "remote_guard: Reconnecting to %s.",
+	    res->hr_remoteaddr);
+	in = out = NULL;
+	if (init_remote(res, &in, &out)) {
+		rw_wlock(&hio_remote_lock[ncomp]);
+		assert(res->hr_remotein == NULL);
+		assert(res->hr_remoteout == NULL);
+		assert(in != NULL && out != NULL);
+		res->hr_remotein = in;
+		res->hr_remoteout = out;
+		rw_unlock(&hio_remote_lock[ncomp]);
+		pjdlog_info("Successfully reconnected to %s.",
+		    res->hr_remoteaddr);
+		sync_start();
+	} else {
+		/* Both connections should be NULL. */
+		assert(res->hr_remotein == NULL);
+		assert(res->hr_remoteout == NULL);
+		assert(in == NULL && out == NULL);
+		pjdlog_debug(2, "remote_guard: Reconnect to %s failed.",
+		    res->hr_remoteaddr);
+	}
 }
 
 /*
@@ -1723,81 +1938,48 @@ static void *
 guard_thread(void *arg)
 {
 	struct hast_resource *res = arg;
-	struct proto_conn *in, *out;
 	unsigned int ii, ncomps;
-	int timeout;
+	struct timespec timeout;
+	time_t lastcheck, now;
+	sigset_t mask;
+	int signo;
 
 	ncomps = HAST_NCOMPONENTS;
-	/* The is only one remote component for now. */
-#define	ISREMOTE(no)	((no) == 1)
+	lastcheck = time(NULL);
+
+	PJDLOG_VERIFY(sigemptyset(&mask) == 0);
+	PJDLOG_VERIFY(sigaddset(&mask, SIGHUP) == 0);
+	PJDLOG_VERIFY(sigaddset(&mask, SIGINT) == 0);
+	PJDLOG_VERIFY(sigaddset(&mask, SIGTERM) == 0);
+
+	timeout.tv_nsec = 0;
+	signo = -1;
 
 	for (;;) {
-		if (sigexit_received) {
+		switch (signo) {
+		case SIGHUP:
+			config_reload();
+			break;
+		case SIGINT:
+		case SIGTERM:
+			sigexit_received = true;
 			primary_exitx(EX_OK,
 			    "Termination signal received, exiting.");
+			break;
+		default:
+			break;
 		}
-		/*
-		 * If all the connection will be fine, we will sleep until
-		 * someone wakes us up.
-		 * If any of the connections will be broken and we won't be
-		 * able to connect, we will sleep only for RECONNECT_SLEEP
-		 * seconds so we can retry soon.
-		 */
-		timeout = 0;
+
 		pjdlog_debug(2, "remote_guard: Checking connections.");
-		mtx_lock(&hio_guard_lock);
-		for (ii = 0; ii < ncomps; ii++) {
-			if (!ISREMOTE(ii))
-				continue;
-			rw_rlock(&hio_remote_lock[ii]);
-			if (ISCONNECTED(res, ii)) {
-				assert(res->hr_remotein != NULL);
-				assert(res->hr_remoteout != NULL);
-				rw_unlock(&hio_remote_lock[ii]);
-				pjdlog_debug(2,
-				    "remote_guard: Connection to %s is ok.",
-				    res->hr_remoteaddr);
-			} else {
-				assert(res->hr_remotein == NULL);
-				assert(res->hr_remoteout == NULL);
-				/*
-				 * Upgrade the lock. It doesn't have to be
-				 * atomic as no other thread can change
-				 * connection status from disconnected to
-				 * connected.
-				 */
-				rw_unlock(&hio_remote_lock[ii]);
-				pjdlog_debug(2,
-				    "remote_guard: Reconnecting to %s.",
-				    res->hr_remoteaddr);
-				in = out = NULL;
-				if (init_remote(res, &in, &out)) {
-					rw_wlock(&hio_remote_lock[ii]);
-					assert(res->hr_remotein == NULL);
-					assert(res->hr_remoteout == NULL);
-					assert(in != NULL && out != NULL);
-					res->hr_remotein = in;
-					res->hr_remoteout = out;
-					rw_unlock(&hio_remote_lock[ii]);
-					pjdlog_info("Successfully reconnected to %s.",
-					    res->hr_remoteaddr);
-					sync_start();
-				} else {
-					/* Both connections should be NULL. */
-					assert(res->hr_remotein == NULL);
-					assert(res->hr_remoteout == NULL);
-					assert(in == NULL && out == NULL);
-					pjdlog_debug(2,
-					    "remote_guard: Reconnect to %s failed.",
-					    res->hr_remoteaddr);
-					timeout = RECONNECT_SLEEP;
-				}
-			}
+		now = time(NULL);
+		if (lastcheck + RETRY_SLEEP <= now) {
+			for (ii = 0; ii < ncomps; ii++)
+				guard_one(res, ii);
+			lastcheck = now;
 		}
-		(void)cv_timedwait(&hio_guard_cond, &hio_guard_lock, timeout);
-		mtx_unlock(&hio_guard_lock);
+		timeout.tv_sec = RETRY_SLEEP;
+		signo = sigtimedwait(&mask, NULL, &timeout);
 	}
-#undef	ISREMOTE
 	/* NOTREACHED */
 	return (NULL);
 }
diff --git a/sbin/hastd/proto.c b/sbin/hastd/proto.c
index 531e7e5fc059..db2a2ef4338f 100644
--- a/sbin/hastd/proto.c
+++ b/sbin/hastd/proto.c
@@ -52,13 +52,20 @@ struct proto_conn {
 #define	PROTO_SIDE_SERVER_WORK		2
 };
 
-static LIST_HEAD(, hast_proto) protos = LIST_HEAD_INITIALIZER(protos);
+static TAILQ_HEAD(, hast_proto) protos = TAILQ_HEAD_INITIALIZER(protos);
 
 void
-proto_register(struct hast_proto *proto)
+proto_register(struct hast_proto *proto, bool isdefault)
 {
+	static bool seen_default = false;
 
-	LIST_INSERT_HEAD(&protos, proto, hp_next);
+	if (!isdefault)
+		TAILQ_INSERT_HEAD(&protos, proto, hp_next);
+	else {
+		assert(!seen_default);
+		seen_default = true;
+		TAILQ_INSERT_TAIL(&protos, proto, hp_next);
+	}
 }
 
 static int
@@ -75,7 +82,7 @@ proto_common_setup(const char *addr, struct proto_conn **connp, int side)
 	if (conn == NULL)
 		return (-1);
 
-	LIST_FOREACH(proto, &protos, hp_next) {
+	TAILQ_FOREACH(proto, &protos, hp_next) {
 		if (side == PROTO_SIDE_CLIENT)
 			ret = proto->hp_client(addr, &ctx);
 		else /* if (side == PROTO_SIDE_SERVER_LISTEN) */
@@ -172,7 +179,7 @@ proto_accept(struct proto_conn *conn, struct proto_conn **newconnp)
 }
 
 int
-proto_send(struct proto_conn *conn, const void *data, size_t size)
+proto_send(const struct proto_conn *conn, const void *data, size_t size)
 {
 	int ret;
 
@@ -189,7 +196,7 @@ proto_send(struct proto_conn *conn, const void *data, size_t size)
 }
 
 int
-proto_recv(struct proto_conn *conn, void *data, size_t size)
+proto_recv(const struct proto_conn *conn, void *data, size_t size)
 {
 	int ret;
 
diff --git a/sbin/hastd/proto.h b/sbin/hastd/proto.h
index 13248bad649f..8d1046caed52 100644
--- a/sbin/hastd/proto.h
+++ b/sbin/hastd/proto.h
@@ -41,8 +41,8 @@ int proto_client(const char *addr, struct proto_conn **connp);
 int proto_connect(struct proto_conn *conn);
 int proto_server(const char *addr, struct proto_conn **connp);
 int proto_accept(struct proto_conn *conn, struct proto_conn **newconnp);
-int proto_send(struct proto_conn *conn, const void *data, size_t size);
-int proto_recv(struct proto_conn *conn, void *data, size_t size);
+int proto_send(const struct proto_conn *conn, const void *data, size_t size);
+int proto_recv(const struct proto_conn *conn, void *data, size_t size);
 int proto_descriptor(const struct proto_conn *conn);
 bool proto_address_match(const struct proto_conn *conn, const char *addr);
 void proto_local_address(const struct proto_conn *conn, char *addr,
diff --git a/sbin/hastd/proto_common.c b/sbin/hastd/proto_common.c
index 131d30e19ef2..84680c2bfdb6 100644
--- a/sbin/hastd/proto_common.c
+++ b/sbin/hastd/proto_common.c
@@ -42,8 +42,7 @@ __FBSDID("$FreeBSD$");
 
 /* Maximum size of packet we want to use when sending data. */
 #ifndef MAX_SEND_SIZE
-//#define	MAX_SEND_SIZE	32768
-#define	MAX_SEND_SIZE	131072
+#define	MAX_SEND_SIZE	32768
 #endif
 
 int
diff --git a/sbin/hastd/proto_impl.h b/sbin/hastd/proto_impl.h
index ea6548d58042..f0dfadd5836b 100644
--- a/sbin/hastd/proto_impl.h
+++ b/sbin/hastd/proto_impl.h
@@ -64,10 +64,10 @@ struct hast_proto {
 	hp_local_address_t *hp_local_address;
 	hp_remote_address_t *hp_remote_address;
 	hp_close_t	*hp_close;
-	LIST_ENTRY(hast_proto) hp_next;
+	TAILQ_ENTRY(hast_proto) hp_next;
 };
 
-void proto_register(struct hast_proto *proto);
+void proto_register(struct hast_proto *proto, bool isdefault);
 
 int proto_common_send(int fd, const unsigned char *data, size_t size);
 int proto_common_recv(int fd, unsigned char *data, size_t size);
diff --git a/sbin/hastd/proto_socketpair.c b/sbin/hastd/proto_socketpair.c
index 08d0c667a069..c0b41a46da3e 100644
--- a/sbin/hastd/proto_socketpair.c
+++ b/sbin/hastd/proto_socketpair.c
@@ -140,6 +140,10 @@ sp_send(void *ctx, const unsigned char *data, size_t size)
 		abort();
 	}
 
+	/* Someone is just trying to decide about side. */
+	if (data == NULL)
+		return (0);
+
 	return (proto_common_send(fd, data, size));
 }
 
@@ -174,6 +178,10 @@ sp_recv(void *ctx, unsigned char *data, size_t size)
 		abort();
 	}
 
+	/* Someone is just trying to decide about side. */
+	if (data == NULL)
+		return (0);
+
 	return (proto_common_recv(fd, data, size));
 }
 
@@ -271,5 +279,5 @@ static __constructor void
 sp_ctor(void)
 {
 
-	proto_register(&sp_proto);
+	proto_register(&sp_proto, false);
 }
diff --git a/sbin/hastd/proto_tcp4.c b/sbin/hastd/proto_tcp4.c
index 5af82d531853..9bb5c62375bd 100644
--- a/sbin/hastd/proto_tcp4.c
+++ b/sbin/hastd/proto_tcp4.c
@@ -126,11 +126,12 @@ tcp4_addr(const char *addr, struct sockaddr_in *sinp)
 		addr += 7;
 	else if (strncasecmp(addr, "tcp://", 6) == 0)
 		addr += 6;
-	else if (addr[0] != '/' &&	/* If this is not path... */
-	    strstr(addr, "://") == NULL)/* ...and has no prefix... */
-		;			/* ...tcp4 is the default. */
-	else
-		return (-1);
+	else {
+		/*
+		 * Because TCP4 is the default assume IP or host is given without
+		 * prefix.
+		 */
+	}
 
 	sinp->sin_family = AF_INET;
 	sinp->sin_len = sizeof(*sinp);
@@ -155,7 +156,7 @@ tcp4_addr(const char *addr, struct sockaddr_in *sinp)
 		size = (size_t)(pp - addr + 1);
 		if (size > sizeof(iporhost))
 			return (ENAMETOOLONG);
-		strlcpy(iporhost, addr, size);
+		(void)strlcpy(iporhost, addr, size);
 	}
 	/* Convert string (IP address or host name) to in_addr_t. */
 	ip = str2ip(iporhost);
@@ -241,8 +242,8 @@ tcp4_connect(void *ctx)
 		return (errno);
 	}
 	/*
-	 * We make socket non-blocking so we have decided about connection
-	 * timeout.
+	 * We make socket non-blocking so we can handle connection timeout
+	 * manually.
 	 */
 	flags |= O_NONBLOCK;
 	if (fcntl(tctx->tc_fd, F_SETFL, flags) == -1) {
@@ -419,8 +420,9 @@ sin2str(struct sockaddr_in *sinp, char *addr, size_t size)
 
 	ip = ntohl(sinp->sin_addr.s_addr);
 	port = ntohs(sinp->sin_port);
-	snprintf(addr, size, "tcp4://%u.%u.%u.%u:%u", ((ip >> 24) & 0xff),
-	    ((ip >> 16) & 0xff), ((ip >> 8) & 0xff), (ip & 0xff), port);
+	PJDLOG_VERIFY(snprintf(addr, size, "tcp4://%u.%u.%u.%u:%u",
+	    ((ip >> 24) & 0xff), ((ip >> 16) & 0xff), ((ip >> 8) & 0xff),
+	    (ip & 0xff), port) < (ssize_t)size);
 }
 
 static bool
@@ -458,7 +460,7 @@ tcp4_local_address(const void *ctx, char *addr, size_t size)
 
 	sinlen = sizeof(sin);
 	if (getsockname(tctx->tc_fd, (struct sockaddr *)&sin, &sinlen) < 0) {
-		strlcpy(addr, "N/A", size);
+		PJDLOG_VERIFY(strlcpy(addr, "N/A", size) < size);
 		return;
 	}
 	sin2str(&sin, addr, size);
@@ -476,7 +478,7 @@ tcp4_remote_address(const void *ctx, char *addr, size_t size)
 
 	sinlen = sizeof(sin);
 	if (getpeername(tctx->tc_fd, (struct sockaddr *)&sin, &sinlen) < 0) {
-		strlcpy(addr, "N/A", size);
+		PJDLOG_VERIFY(strlcpy(addr, "N/A", size) < size);
 		return;
 	}
 	sin2str(&sin, addr, size);
@@ -515,5 +517,5 @@ static __constructor void
 tcp4_ctor(void)
 {
 
-	proto_register(&tcp4_proto);
+	proto_register(&tcp4_proto, true);
 }
diff --git a/sbin/hastd/proto_uds.c b/sbin/hastd/proto_uds.c
index 0fac82f24d06..9c900e711d07 100644
--- a/sbin/hastd/proto_uds.c
+++ b/sbin/hastd/proto_uds.c
@@ -43,6 +43,7 @@ __FBSDID("$FreeBSD$");
 #include <unistd.h>
 
 #include "hast.h"
+#include "pjdlog.h"
 #include "proto_impl.h"
 
 #define	UDS_CTX_MAGIC	0xd541c
@@ -257,15 +258,15 @@ uds_local_address(const void *ctx, char *addr, size_t size)
 
 	sunlen = sizeof(sun);
 	if (getsockname(uctx->uc_fd, (struct sockaddr *)&sun, &sunlen) < 0) {
-		strlcpy(addr, "N/A", size);
+		PJDLOG_VERIFY(strlcpy(addr, "N/A", size) < size);
 		return;
 	}
 	assert(sun.sun_family == AF_UNIX);
 	if (sun.sun_path[0] == '\0') {
-		strlcpy(addr, "N/A", size);
+		PJDLOG_VERIFY(strlcpy(addr, "N/A", size) < size);
 		return;
 	}
-	snprintf(addr, size, "uds://%s", sun.sun_path);
+	PJDLOG_VERIFY(snprintf(addr, size, "uds://%s", sun.sun_path) < (ssize_t)size);
 }
 
 static void
@@ -281,12 +282,12 @@ uds_remote_address(const void *ctx, char *addr, size_t size)
 
 	sunlen = sizeof(sun);
 	if (getpeername(uctx->uc_fd, (struct sockaddr *)&sun, &sunlen) < 0) {
-		strlcpy(addr, "N/A", size);
+		PJDLOG_VERIFY(strlcpy(addr, "N/A", size) < size);
 		return;
 	}
 	assert(sun.sun_family == AF_UNIX);
 	if (sun.sun_path[0] == '\0') {
-		strlcpy(addr, "N/A", size);
+		PJDLOG_VERIFY(strlcpy(addr, "N/A", size) < size);
 		return;
 	}
 	snprintf(addr, size, "uds://%s", sun.sun_path);
@@ -326,5 +327,5 @@ static __constructor void
 uds_ctor(void)
 {
 
-	proto_register(&uds_proto);
+	proto_register(&uds_proto, false);
 }
diff --git a/sbin/hastd/secondary.c b/sbin/hastd/secondary.c
index b487e9dc2bee..403c5b214e67 100644
--- a/sbin/hastd/secondary.c
+++ b/sbin/hastd/secondary.c
@@ -1,5 +1,6 @@
 /*-
  * Copyright (c) 2009-2010 The FreeBSD Foundation
+ * Copyright (c) 2010 Pawel Jakub Dawidek <pjd@FreeBSD.org>
  * All rights reserved.
  *
  * This software was developed by Pawel Jakub Dawidek under sponsorship from
@@ -42,6 +43,7 @@ __FBSDID("$FreeBSD$");
 #include <fcntl.h>
 #include <libgeom.h>
 #include <pthread.h>
+#include <signal.h>
 #include <stdint.h>
 #include <stdio.h>
 #include <string.h>
@@ -53,9 +55,11 @@ __FBSDID("$FreeBSD$");
 #include <pjdlog.h>
 
 #include "control.h"
+#include "event.h"
 #include "hast.h"
 #include "hast_proto.h"
 #include "hastd.h"
+#include "hooks.h"
 #include "metadata.h"
 #include "proto.h"
 #include "subr.h"
@@ -72,6 +76,8 @@ struct hio {
 	TAILQ_ENTRY(hio) hio_next;
 };
 
+static struct hast_resource *gres;
+
 /*
  * Free list holds unused structures. When free list is empty, we have to wait
  * until some in-progress requests are freed.
@@ -102,6 +108,26 @@ static void *recv_thread(void *arg);
 static void *disk_thread(void *arg);
 static void *send_thread(void *arg);
 
+#define	QUEUE_INSERT(name, hio)	do {					\
+	bool _wakeup;							\
+									\
+	mtx_lock(&hio_##name##_list_lock);				\
+	_wakeup = TAILQ_EMPTY(&hio_##name##_list);			\
+	TAILQ_INSERT_TAIL(&hio_##name##_list, (hio), hio_next);		\
+	mtx_unlock(&hio_##name##_list_lock);				\
+	if (_wakeup)							\
+		cv_signal(&hio_##name##_list_cond);			\
+} while (0)
+#define	QUEUE_TAKE(name, hio)	do {					\
+	mtx_lock(&hio_##name##_list_lock);				\
+	while (((hio) = TAILQ_FIRST(&hio_##name##_list)) == NULL) {	\
+		cv_wait(&hio_##name##_list_cond,			\
+		    &hio_##name##_list_lock);				\
+	}								\
+	TAILQ_REMOVE(&hio_##name##_list, (hio), hio_next);		\
+	mtx_unlock(&hio_##name##_list_lock);				\
+} while (0)
+
 static void
 init_environment(void)
 {
@@ -127,14 +153,16 @@ init_environment(void)
 	for (ii = 0; ii < HAST_HIO_MAX; ii++) {
 		hio = malloc(sizeof(*hio));
 		if (hio == NULL) {
-			errx(EX_TEMPFAIL, "cannot allocate %zu bytes of memory "
-			    "for hio request", sizeof(*hio));
+			pjdlog_exitx(EX_TEMPFAIL,
+			    "Unable to allocate memory (%zu bytes) for hio request.",
+			    sizeof(*hio));
 		}
 		hio->hio_error = 0;
 		hio->hio_data = malloc(MAXPHYS);
 		if (hio->hio_data == NULL) {
-			errx(EX_TEMPFAIL, "cannot allocate %zu bytes of memory "
-			    "for gctl_data", (size_t)MAXPHYS);
+			pjdlog_exitx(EX_TEMPFAIL,
+			    "Unable to allocate memory (%zu bytes) for gctl_data.",
+			    (size_t)MAXPHYS);
 		}
 		TAILQ_INSERT_HEAD(&hio_free_list, hio, hio_next);
 	}
@@ -299,6 +327,7 @@ init_remote(struct hast_resource *res, struct nv *nvin)
 	if (res->hr_secondary_localcnt > res->hr_primary_remotecnt &&
 	     res->hr_primary_localcnt > res->hr_secondary_remotecnt) {
 		/* Exit on split-brain. */
+		event_send(res, EVENT_SPLITBRAIN);
 		exit(EX_CONFIG);
 	}
 }
@@ -306,6 +335,7 @@ init_remote(struct hast_resource *res, struct nv *nvin)
 void
 hastd_secondary(struct hast_resource *res, struct nv *nvin)
 {
+	sigset_t mask;
 	pthread_t td;
 	pid_t pid;
 	int error;
@@ -318,6 +348,14 @@ hastd_secondary(struct hast_resource *res, struct nv *nvin)
 		pjdlog_exit(EX_OSERR,
 		    "Unable to create control sockets between parent and child");
 	}
+	/*
+	 * Create communication channel between child and parent.
+	 */
+	if (proto_client("socketpair://", &res->hr_event) < 0) {
+		KEEP_ERRNO((void)pidfile_remove(pfh));
+		pjdlog_exit(EX_OSERR,
+		    "Unable to create event sockets between child and parent");
+	}
 
 	pid = fork();
 	if (pid < 0) {
@@ -331,13 +369,25 @@ hastd_secondary(struct hast_resource *res, struct nv *nvin)
 		res->hr_remotein = NULL;
 		proto_close(res->hr_remoteout);
 		res->hr_remoteout = NULL;
+		/* Declare that we are receiver. */
+		proto_recv(res->hr_event, NULL, 0);
 		res->hr_workerpid = pid;
 		return;
 	}
+
+	gres = res;
+
 	(void)pidfile_close(pfh);
+	hook_fini();
 
 	setproctitle("%s (secondary)", res->hr_name);
 
+	PJDLOG_VERIFY(sigemptyset(&mask) == 0);
+	PJDLOG_VERIFY(sigprocmask(SIG_SETMASK, &mask, NULL) == 0);
+
+	/* Declare that we are sender. */
+	proto_send(res->hr_event, NULL, 0);
+
 	/* Error in setting timeout is not critical, but why should it fail? */
 	if (proto_timeout(res->hr_remotein, 0) < 0)
 		pjdlog_errno(LOG_WARNING, "Unable to set connection timeout");
@@ -345,16 +395,27 @@ hastd_secondary(struct hast_resource *res, struct nv *nvin)
 		pjdlog_errno(LOG_WARNING, "Unable to set connection timeout");
 
 	init_local(res);
-	init_remote(res, nvin);
 	init_environment();
 
+	/*
+	 * Create the control thread before sending any event to the parent,
+	 * as we can deadlock when parent sends control request to worker,
+	 * but worker has no control thread started yet, so parent waits.
+	 * In the meantime worker sends an event to the parent, but parent
+	 * is unable to handle the event, because it waits for control
+	 * request response.
+	 */
+	error = pthread_create(&td, NULL, ctrl_thread, res);
+	assert(error == 0);
+
+	init_remote(res, nvin);
+	event_send(res, EVENT_CONNECT);
+
 	error = pthread_create(&td, NULL, recv_thread, res);
 	assert(error == 0);
 	error = pthread_create(&td, NULL, disk_thread, res);
 	assert(error == 0);
-	error = pthread_create(&td, NULL, send_thread, res);
-	assert(error == 0);
-	(void)ctrl_thread(res);
+	(void)send_thread(res);
 }
 
 static void
@@ -387,6 +448,9 @@ reqlog(int loglevel, int debuglevel, int error, struct hio *hio, const char *fmt
 			    "WRITE(%ju, %ju).", (uintmax_t)hio->hio_offset,
 			    (uintmax_t)hio->hio_length);
 			break;
+		case HIO_KEEPALIVE:
+			(void)snprintf(msg + len, sizeof(msg) - len, "KEEPALIVE.");
+			break;
 		default:
 			(void)snprintf(msg + len, sizeof(msg) - len,
 			    "UNKNOWN(%u).", (unsigned int)hio->hio_cmd);
@@ -407,6 +471,8 @@ requnpack(struct hast_resource *res, struct hio *hio)
 		goto end;
 	}
 	switch (hio->hio_cmd) {
+	case HIO_KEEPALIVE:
+		break;
 	case HIO_READ:
 	case HIO_WRITE:
 	case HIO_DELETE:
@@ -465,6 +531,19 @@ end:
 	return (hio->hio_error);
 }
 
+static __dead2 void
+secondary_exit(int exitcode, const char *fmt, ...)
+{
+	va_list ap;
+
+	assert(exitcode != EX_OK);
+	va_start(ap, fmt);
+	pjdlogv_errno(LOG_ERR, fmt, ap);
+	va_end(ap);
+	event_send(gres, EVENT_DISCONNECT);
+	exit(exitcode);
+}
+
 /*
  * Thread receives requests from the primary node.
  */
@@ -473,51 +552,41 @@ recv_thread(void *arg)
 {
 	struct hast_resource *res = arg;
 	struct hio *hio;
-	bool wakeup;
 
 	for (;;) {
 		pjdlog_debug(2, "recv: Taking free request.");
-		mtx_lock(&hio_free_list_lock);
-		while ((hio = TAILQ_FIRST(&hio_free_list)) == NULL) {
-			pjdlog_debug(2, "recv: No free requests, waiting.");
-			cv_wait(&hio_free_list_cond, &hio_free_list_lock);
-		}
-		TAILQ_REMOVE(&hio_free_list, hio, hio_next);
-		mtx_unlock(&hio_free_list_lock);
+		QUEUE_TAKE(free, hio);
 		pjdlog_debug(2, "recv: (%p) Got request.", hio);
 		if (hast_proto_recv_hdr(res->hr_remotein, &hio->hio_nv) < 0) {
-			pjdlog_exit(EX_TEMPFAIL,
+			secondary_exit(EX_TEMPFAIL,
 			    "Unable to receive request header");
 		}
-		if (requnpack(res, hio) != 0)
-			goto send_queue;
+		if (requnpack(res, hio) != 0) {
+			pjdlog_debug(2,
+			    "recv: (%p) Moving request to the send queue.",
+			    hio);
+			QUEUE_INSERT(send, hio);
+			continue;
+		}
 		reqlog(LOG_DEBUG, 2, -1, hio,
 		    "recv: (%p) Got request header: ", hio);
-		if (hio->hio_cmd == HIO_WRITE) {
+		if (hio->hio_cmd == HIO_KEEPALIVE) {
+			pjdlog_debug(2,
+			    "recv: (%p) Moving request to the free queue.",
+			    hio);
+			nv_free(hio->hio_nv);
+			QUEUE_INSERT(free, hio);
+			continue;
+		} else if (hio->hio_cmd == HIO_WRITE) {
 			if (hast_proto_recv_data(res, res->hr_remotein,
 			    hio->hio_nv, hio->hio_data, MAXPHYS) < 0) {
-				pjdlog_exit(EX_TEMPFAIL,
-				    "Unable to receive reply data");
+				secondary_exit(EX_TEMPFAIL,
+				    "Unable to receive request data");
 			}
 		}
 		pjdlog_debug(2, "recv: (%p) Moving request to the disk queue.",
 		    hio);
-		mtx_lock(&hio_disk_list_lock);
-		wakeup = TAILQ_EMPTY(&hio_disk_list);
-		TAILQ_INSERT_TAIL(&hio_disk_list, hio, hio_next);
-		mtx_unlock(&hio_disk_list_lock);
-		if (wakeup)
-			cv_signal(&hio_disk_list_cond);
-		continue;
-send_queue:
-		pjdlog_debug(2, "recv: (%p) Moving request to the send queue.",
-		    hio);
-		mtx_lock(&hio_send_list_lock);
-		wakeup = TAILQ_EMPTY(&hio_send_list);
-		TAILQ_INSERT_TAIL(&hio_send_list, hio, hio_next);
-		mtx_unlock(&hio_send_list_lock);
-		if (wakeup)
-			cv_signal(&hio_send_list_cond);
+		QUEUE_INSERT(disk, hio);
 	}
 	/* NOTREACHED */
 	return (NULL);
@@ -533,19 +602,13 @@ disk_thread(void *arg)
 	struct hast_resource *res = arg;
 	struct hio *hio;
 	ssize_t ret;
-	bool clear_activemap, wakeup;
+	bool clear_activemap;
 
 	clear_activemap = true;
 
 	for (;;) {
 		pjdlog_debug(2, "disk: Taking request.");
-		mtx_lock(&hio_disk_list_lock);
-		while ((hio = TAILQ_FIRST(&hio_disk_list)) == NULL) {
-			pjdlog_debug(2, "disk: No requests, waiting.");
-			cv_wait(&hio_disk_list_cond, &hio_disk_list_lock);
-		}
-		TAILQ_REMOVE(&hio_disk_list, hio, hio_next);
-		mtx_unlock(&hio_disk_list_lock);
+		QUEUE_TAKE(disk, hio);
 		while (clear_activemap) {
 			unsigned char *map;
 			size_t mapsize;
@@ -623,12 +686,7 @@ disk_thread(void *arg)
 		}
 		pjdlog_debug(2, "disk: (%p) Moving request to the send queue.",
 		    hio);
-		mtx_lock(&hio_send_list_lock);
-		wakeup = TAILQ_EMPTY(&hio_send_list);
-		TAILQ_INSERT_TAIL(&hio_send_list, hio, hio_next);
-		mtx_unlock(&hio_send_list_lock);
-		if (wakeup)
-			cv_signal(&hio_send_list_cond);
+		QUEUE_INSERT(send, hio);
 	}
 	/* NOTREACHED */
 	return (NULL);
@@ -645,17 +703,10 @@ send_thread(void *arg)
 	struct hio *hio;
 	void *data;
 	size_t length;
-	bool wakeup;
 
 	for (;;) {
 		pjdlog_debug(2, "send: Taking request.");
-		mtx_lock(&hio_send_list_lock);
-		while ((hio = TAILQ_FIRST(&hio_send_list)) == NULL) {
-			pjdlog_debug(2, "send: No requests, waiting.");
-			cv_wait(&hio_send_list_cond, &hio_send_list_lock);
-		}
-		TAILQ_REMOVE(&hio_send_list, hio, hio_next);
-		mtx_unlock(&hio_send_list_lock);
+		QUEUE_TAKE(send, hio);
 		reqlog(LOG_DEBUG, 2, -1, hio, "send: (%p) Got request: ", hio);
 		nvout = nv_alloc();
 		/* Copy sequence number. */
@@ -685,19 +736,14 @@ send_thread(void *arg)
 			nv_add_int16(nvout, hio->hio_error, "error");
 		if (hast_proto_send(res, res->hr_remoteout, nvout, data,
 		    length) < 0) {
-			pjdlog_exit(EX_TEMPFAIL, "Unable to send reply.");
+			secondary_exit(EX_TEMPFAIL, "Unable to send reply.");
 		}
 		nv_free(nvout);
 		pjdlog_debug(2, "send: (%p) Moving request to the free queue.",
 		    hio);
 		nv_free(hio->hio_nv);
 		hio->hio_error = 0;
-		mtx_lock(&hio_free_list_lock);
-		wakeup = TAILQ_EMPTY(&hio_free_list);
-		TAILQ_INSERT_TAIL(&hio_free_list, hio, hio_next);
-		mtx_unlock(&hio_free_list_lock);
-		if (wakeup)
-			cv_signal(&hio_free_list_cond);
+		QUEUE_INSERT(free, hio);
 	}
 	/* NOTREACHED */
 	return (NULL);
diff --git a/sbin/hastd/synch.h b/sbin/hastd/synch.h
index 7269aea6607a..1dda49d7eb71 100644
--- a/sbin/hastd/synch.h
+++ b/sbin/hastd/synch.h
@@ -33,7 +33,9 @@
 #define	_SYNCH_H_
 
 #include <assert.h>
+#include <errno.h>
 #include <pthread.h>
+#include <pthread_np.h>
 #include <stdbool.h>
 #include <time.h>
 
@@ -46,6 +48,14 @@ mtx_init(pthread_mutex_t *lock)
 	assert(error == 0);
 }
 static __inline void
+mtx_destroy(pthread_mutex_t *lock)
+{
+	int error;
+
+	error = pthread_mutex_destroy(lock);
+	assert(error == 0);
+}
+static __inline void
 mtx_lock(pthread_mutex_t *lock)
 {
 	int error;
@@ -70,6 +80,12 @@ mtx_unlock(pthread_mutex_t *lock)
 	error = pthread_mutex_unlock(lock);
 	assert(error == 0);
 }
+static __inline bool
+mtx_owned(pthread_mutex_t *lock)
+{
+
+	return (pthread_mutex_isowned_np(lock) != 0);
+}
 
 static __inline void
 rw_init(pthread_rwlock_t *lock)
@@ -80,6 +96,14 @@ rw_init(pthread_rwlock_t *lock)
 	assert(error == 0);
 }
 static __inline void
+rw_destroy(pthread_rwlock_t *lock)
+{
+	int error;
+
+	error = pthread_rwlock_destroy(lock);
+	assert(error == 0);
+}
+static __inline void
 rw_rlock(pthread_rwlock_t *lock)
 {
 	int error;
diff --git a/sbin/hastd/token.l b/sbin/hastd/token.l
index e5d4ca10df21..05d600c6ef10 100644
--- a/sbin/hastd/token.l
+++ b/sbin/hastd/token.l
@@ -49,6 +49,7 @@ listen			{ DP; return LISTEN; }
 port			{ DP; return PORT; }
 replication		{ DP; return REPLICATION; }
 timeout			{ DP; return TIMEOUT; }
+exec			{ DP; return EXEC; }
 resource		{ DP; return RESOURCE; }
 name			{ DP; return NAME; }
 local			{ DP; return LOCAL; }