--- a/usr/src/Makefile.lint	Tue Nov 03 21:31:39 2009 -0800
+++ b/usr/src/Makefile.lint	Tue Nov 03 23:15:19 2009 -0800
@@ -72,6 +72,7 @@
 	cmd/cmd-inet/usr.bin \
 	cmd/cmd-inet/usr.lib/bridged \
 	cmd/cmd-inet/usr.lib/dsvclockd \
+	cmd/cmd-inet/usr.lib/ilbd \
 	cmd/cmd-inet/usr.lib/in.dhcpd \
 	cmd/cmd-inet/usr.lib/in.mpathd \
 	cmd/cmd-inet/usr.lib/in.ndpd \
@@ -82,6 +83,7 @@
 	cmd/cmd-inet/usr.lib/wanboot \
 	cmd/cmd-inet/usr.sadm \
 	cmd/cmd-inet/usr.sbin \
+	cmd/cmd-inet/usr.sbin/ilbadm \
 	cmd/col \
 	cmd/compress \
 	cmd/consadm \
@@ -365,6 +367,7 @@
 	lib/libgss \
 	lib/libhotplug \
 	lib/libidmap \
+	lib/libilb \
 	lib/libinetcfg \
 	lib/libinetsvc \
 	lib/libinetutil \

--- a/usr/src/Targetdirs	Tue Nov 03 21:31:39 2009 -0800
+++ b/usr/src/Targetdirs	Tue Nov 03 23:15:19 2009 -0800
@@ -234,6 +234,7 @@
 	/usr/lib/inet/dhcp/nsu \
 	/usr/lib/inet/dhcp/svc \
 	/usr/lib/inet/dhcp/svcadm \
+	/usr/lib/inet/ilb \
 	/usr/lib/inet/$(MACH32) \
 	$(XDIRS) \
 	/usr/lib/krb5 \

--- a/usr/src/cmd/Makefile.check	Tue Nov 03 21:31:39 2009 -0800
+++ b/usr/src/cmd/Makefile.check	Tue Nov 03 23:15:19 2009 -0800
@@ -82,6 +82,7 @@
 	agents/snmp/snmprelayd		\
 	boot/scripts			\
 	cmd-crypto/scripts		\
+	cmd-inet/usr.lib/ilbd		\
 	cmd-inet/usr.lib/in.chargend	\
 	cmd-inet/usr.lib/in.daytimed	\
 	cmd-inet/usr.lib/in.dhcpd	\

--- a/usr/src/cmd/cmd-inet/usr.lib/Makefile	Tue Nov 03 21:31:39 2009 -0800
+++ b/usr/src/cmd/cmd-inet/usr.lib/Makefile	Tue Nov 03 23:15:19 2009 -0800
@@ -23,12 +23,12 @@
 # Use is subject to license terms.
 #
 
-SUBDIRS=	bridged dhcp dsvclockd in.chargend in.daytimed \
+SUBDIRS=	bridged dhcp dsvclockd ilbd in.chargend in.daytimed \
 		in.discardd in.echod in.dhcpd in.mpathd in.ndpd \
 		in.ripngd in.timed inetd mdnsd ncaconfd pppoe \
 		slpd wanboot wpad
 
-MSGSUBDIRS=	dsvclockd in.dhcpd inetd ncaconfd wanboot
+MSGSUBDIRS=	dsvclockd ilbd in.dhcpd inetd ncaconfd wanboot
 
 include ../../Makefile.cmd
 

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/usr/src/cmd/cmd-inet/usr.lib/ilbd/Makefile	Tue Nov 03 23:15:19 2009 -0800
@@ -0,0 +1,112 @@
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+# Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
+# Use is subject to license terms.
+#
+
+ILBD=		ilbd
+PROG=		$(ILBD)
+MANIFEST=	ilbd.xml
+
+ILBD_OBJS=	ilbd_main.o ilbd_support.o ilbd_sg.o ilbd_rules.o ilbd_hc.o \
+		ilbd_nat.o ilbd_scf.o
+ILBD_SRCS=	$(ILBD_OBJS:%.o=%.c)
+HDRS=		ilbd.h
+
+LIST_OBJS=	list.o
+LIST_SRCS=	$(LIST_OBJS:%.o=../../../uts/common/os/%.c) 
+
+LIB_INC=	$(SRC)/lib/libilb/common
+
+OBJS=		$(ILBD_OBJS) $(LIST_OBJS)
+SRCS=		$(ILBD_SRCS) $(LIST_SRCS)
+
+ILBSUBDIRS=	ilb
+
+include		../../../Makefile.cmd
+include		../../Makefile.cmd-inet
+
+ROOTMANIFESTDIR=	$(ROOTSVCNETWORK)/loadbalancer
+
+CHECKHDRS=	$(HDRS:%.h=%.check)
+
+CPPFLAGS +=	-D_FILE_OFFSET_BITS=64 -I$(CMDINETCOMMONDIR) -D_REENTRANT
+CPPFLAGS +=	-I$(LIB_INC)
+CPPFLAGS +=	-I$(SRC)/uts/common
+
+C99MODE =      $(C99_ENABLE)
+
+# I18n
+POFILE =        $(ILBD).po
+POFILES =       $(ILBD_SRCS:%.c=%.po)
+
+all:=		TARGET= all
+install:=	TARGET= install
+clean:=		TARGET= clean
+clobber:=	TARGET= clobber
+lint:=		TARGET= lint
+
+# used for debugging ONLY:
+
+CFLAGS =		-g -K pic 
+STRIP_STABS=		:
+CTFCVTFLAGS +=		-g
+
+
+LDLIBS +=	-lsocket -lsecdb -lnsl -lilb -lscf -linetutil -lbsm
+
+.KEEP_STATE:
+
+all:		$(PROG)
+
+$(ILBD): 	$(OBJS)
+		$(LINK.c) $(OBJS) -o $@ $(LDLIBS) $(CTFMERGE_HOOK)
+		$(POST_PROCESS)
+
+include ../Makefile.lib
+
+install:	all $(ROOTLIBINETPROG) $(ETCILBDFILES) $(ROOTMANIFEST)
+
+check:	$(CHKMANIFEST) $(CHECKHDRS)
+
+clean:
+	$(RM) $(OBJS)
+
+lint:
+	$(LINT.c) $(ILBD_SRCS) $(LDLIBS)
+
+$(POFILE):	$(POFILES)
+		$(RM) $@
+		$(CAT) $(POFILES) > $@
+
+all install clean clobber lint: $(ILBSUBDIRS)
+
+include		../../../Makefile.targ
+
+# the below is needed to get list.o built
+%.o: ../../../../uts/common/os/%.c
+	$(COMPILE.c) -o $@ $<
+	$(POST_PROCESS_O)
+
+$(ILBSUBDIRS): FRC
+	@cd $@; pwd; $(MAKE) $(TARGET)
+
+FRC:

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/usr/src/cmd/cmd-inet/usr.lib/ilbd/ilb/Makefile	Tue Nov 03 23:15:19 2009 -0800
@@ -0,0 +1,55 @@
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+# Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
+# Use is subject to license terms.
+#
+
+PROG =		ilb_probe
+OBJS =		$(PROG:%=%.o)
+SRCS =		$(OBJS:%.o=%.c)
+
+include	../../../../Makefile.cmd
+
+LDLIBS +=	-lsocket -lnsl
+
+.KEEP_STATE:
+
+all: $(PROG)
+
+include	../../Makefile.lib
+
+ROOTLIBINETILB =	$(ROOTLIBINET)/ilb
+ROOTLIBINETILBPROG =	$(PROG:%=$(ROOTLIBINETILB)/%)
+
+$(ROOTLIBINETILB): $(ROOTLIBINET)
+	$(INS.dir)
+
+$(ROOTLIBINETILB)/%: % $(ROOTLIBINETILB)
+	$(INS.file)
+
+install: all $(ROOTLIBINETILBPROG)
+
+clean:
+	$(RM) $(OBJS)
+
+lint:		lint_SRCS
+
+include		../../../../Makefile.targ

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/usr/src/cmd/cmd-inet/usr.lib/ilbd/ilb/ilb_probe.c	Tue Nov 03 23:15:19 2009 -0800
@@ -0,0 +1,875 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <sys/time.h>
+
+#include <netinet/in_systm.h>
+#include <netinet/in.h>
+#include <netinet/ip.h>
+#include <netinet/ip6.h>
+#include <arpa/inet.h>
+#include <netinet/tcp.h>
+#include <netinet/ip_icmp.h>
+#include <netinet/icmp6.h>
+#include <netinet/udp.h>
+#include <netdb.h>
+#include <unistd.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <strings.h>
+#include <errno.h>
+#include <limits.h>
+#include <signal.h>
+#include <libgen.h>
+#include <fcntl.h>
+
+/*
+ * The following values are what ilbd will set argv[0] to.  This determines
+ * what type of probe to send out.
+ */
+#define	PROBE_PING	"ilb_ping"
+#define	PROBE_PROTO	"ilb_probe"
+
+/* The transport protocol to use in the probe.  Value of argv[3]. */
+#define	PROTO_TCP	"TCP"
+#define	PROTO_UDP	"UDP"
+
+enum probe_type { ping_probe, tcp_probe, udp_probe };
+
+/* Load balance mode.  Value of argv[4]. */
+#define	MODE_DSR	"DSR"
+#define	MODE_NAT	"NAT"
+#define	MODE_HALF_NAT	"HALF_NAT"
+
+enum lb_mode { dsr, nat, half_nat };
+
+/* Number of arguments to the command from ilbd. */
+#define	PROG_ARGC	7
+
+/* Size of buffer used to receive ICMP packet */
+#define	RECV_PKT_SZ	256
+
+/*
+ * Struct to store the probe info (most is passed in using the argv[] array to
+ * the command given by ilbd).  The argv[] contains the following.
+ *
+ * argv[0] is either PROBE_PING or PROBE_PROTO
+ * argv[1] is the VIP
+ * argv[2] is the backend server address
+ * argv[3] is the transport protocol used in the rule
+ * argv[4] is the load balance mode, "DSR", "NAT", "HALF-NAT"
+ * argv[5] is the probe port
+ * argv[6] is the probe timeout
+ *
+ * The following three fields are used in sending ICMP ECHO probe.
+ *
+ * echo_id is the ID set in the probe
+ * echo_seq is the sequence set in the probe
+ * echo_cookie is the random number data in a probe
+ * lport is the local port (in network byte order) used to send the probe
+ */
+typedef struct {
+	enum probe_type		probe;
+	struct in6_addr		vip;		/* argv[1] */
+	struct in6_addr		srv_addr;	/* argv[2] */
+	int			proto;		/* argv[3] */
+	enum lb_mode		mode;		/* argv[4] */
+	in_port_t		port;		/* argv[5] */
+	uint32_t		timeout;	/* argv[6] */
+
+	uint16_t		echo_id;
+	uint16_t		echo_seq;
+	uint32_t		echo_cookie;
+	in_port_t		lport;
+} probe_param_t;
+
+/* Global variable to indicate whether a timeout means success. */
+static boolean_t timeout_is_good;
+
+/* SIGALRM handler */
+/* ARGSUSED */
+static void
+probe_exit(int s)
+{
+	if (timeout_is_good) {
+		(void) printf("0");
+		exit(0);
+	} else {
+		(void) printf("-1");
+		exit(255);
+	}
+}
+
+/*
+ * Checksum routine for Internet Protocol family headers (C Version)
+ * (copied from ping.c)
+ */
+static ushort_t
+in_cksum(ushort_t *addr, int len)
+{
+	int nleft = len;
+	ushort_t *w = addr;
+	ushort_t answer;
+	ushort_t odd_byte = 0;
+	int sum = 0;
+
+	/*
+	 *  Our algorithm is simple, using a 32 bit accumulator (sum),
+	 *  we add sequential 16 bit words to it, and at the end, fold
+	 *  back all the carry bits from the top 16 bits into the lower
+	 *  16 bits.
+	 */
+	while (nleft > 1) {
+		sum += *w++;
+		nleft -= 2;
+	}
+
+	/* mop up an odd byte, if necessary */
+	if (nleft == 1) {
+		*(uchar_t *)(&odd_byte) = *(uchar_t *)w;
+		sum += odd_byte;
+	}
+
+	/*
+	 * add back carry outs from top 16 bits to low 16 bits
+	 */
+	sum = (sum >> 16) + (sum & 0xffff);	/* add hi 16 to low 16 */
+	sum += (sum >> 16);			/* add carry */
+	answer = ~sum;				/* truncate to 16 bits */
+	return (answer);
+}
+
+/* It is assumed that argv[] contains PROBE_ARGC arguments. */
+static boolean_t
+parse_probe_param(char *argv[], probe_param_t *param)
+{
+	int32_t port;
+	int64_t timeout;
+	struct in_addr v4addr;
+
+	if (strcmp(basename(argv[0]), PROBE_PING) == 0) {
+		param->probe = ping_probe;
+	} else {
+		if (strcmp(basename(argv[0]), PROBE_PROTO) != 0)
+			return (B_FALSE);
+
+		if (strcasecmp(argv[3], PROTO_TCP) == 0) {
+			param->probe = tcp_probe;
+			param->proto = IPPROTO_TCP;
+		} else if (strcasecmp(argv[3], PROTO_UDP) == 0) {
+			param->probe = udp_probe;
+			param->proto = IPPROTO_UDP;
+		} else {
+			return (B_FALSE);
+		}
+	}
+
+	if (strchr(argv[1], ':') != NULL) {
+		if (inet_pton(AF_INET6, argv[1], &param->vip) == 0)
+			return (B_FALSE);
+	} else if (strchr(argv[1], '.') != NULL) {
+		if (inet_pton(AF_INET, argv[1], &v4addr) == 0)
+			return (B_FALSE);
+		IN6_INADDR_TO_V4MAPPED(&v4addr, &param->vip);
+	} else {
+		return (B_FALSE);
+	}
+
+	/*
+	 * The address family of vip and srv_addr should be the same for
+	 * now.  But in future, we may allow them to be different...  So
+	 * we don't do a check here.
+	 */
+	if (strchr(argv[2], ':') != NULL) {
+		if (inet_pton(AF_INET6, argv[2], &param->srv_addr) == 0)
+			return (B_FALSE);
+	} else if (strchr(argv[2], '.') != NULL) {
+		if (inet_pton(AF_INET, argv[2], &v4addr) == 0)
+			return (B_FALSE);
+		IN6_INADDR_TO_V4MAPPED(&v4addr, &param->srv_addr);
+	} else {
+		return (B_FALSE);
+	}
+
+	if (strcasecmp(argv[4], MODE_DSR) == 0)
+		param->mode = dsr;
+	else if (strcasecmp(argv[4], MODE_NAT) == 0)
+		param->mode = nat;
+	else if (strcasecmp(argv[4], MODE_HALF_NAT) == 0)
+		param->mode = half_nat;
+	else
+		return (B_FALSE);
+
+	if ((port = atoi(argv[5])) <= 0 || port > USHRT_MAX)
+		return (B_FALSE);
+	param->port = port;
+
+	if ((timeout = strtoll(argv[6], NULL, 10)) <= 0 || timeout > UINT_MAX)
+		return (B_FALSE);
+	param->timeout = timeout;
+
+	return (B_TRUE);
+}
+
+/*
+ * Set up the destination address to be used to send a probe based on
+ * param.
+ */
+static int
+set_sockaddr(struct sockaddr_storage *addr, socklen_t *addr_len,
+    void **next_hop, probe_param_t *param)
+{
+	int af;
+	struct in6_addr *param_addr;
+	struct sockaddr_in *v4_addr;
+	struct sockaddr_in6 *v6_addr;
+	boolean_t nh = B_FALSE;
+
+	switch (param->mode) {
+	case dsr:
+		param_addr = &param->vip;
+		nh = B_TRUE;
+		break;
+	case nat:
+	case half_nat:
+		param_addr = &param->srv_addr;
+		break;
+	}
+	if (IN6_IS_ADDR_V4MAPPED(param_addr)) {
+		af = AF_INET;
+		v4_addr = (struct sockaddr_in *)addr;
+		IN6_V4MAPPED_TO_INADDR(param_addr, &v4_addr->sin_addr);
+		v4_addr->sin_family = AF_INET;
+		v4_addr->sin_port = htons(param->port);
+
+		*addr_len = sizeof (*v4_addr);
+	} else {
+		af = AF_INET6;
+		v6_addr = (struct sockaddr_in6 *)addr;
+		v6_addr->sin6_family = AF_INET6;
+		v6_addr->sin6_addr = *param_addr;
+		v6_addr->sin6_port = htons(param->port);
+		v6_addr->sin6_flowinfo = 0;
+		v6_addr->sin6_scope_id = 0;
+
+		*addr_len = sizeof (*v6_addr);
+	}
+
+	if (!nh) {
+		*next_hop = NULL;
+		return (af);
+	}
+
+	if (af == AF_INET) {
+		ipaddr_t *nh_addr;
+
+		nh_addr = malloc(sizeof (ipaddr_t));
+		IN6_V4MAPPED_TO_IPADDR(&param->srv_addr, *nh_addr);
+		*next_hop = nh_addr;
+	} else {
+		struct sockaddr_in6 *nh_addr;
+
+		nh_addr = malloc(sizeof (*nh_addr));
+		nh_addr->sin6_family = AF_INET6;
+		nh_addr->sin6_addr = param->srv_addr;
+		nh_addr->sin6_flowinfo = 0;
+		nh_addr->sin6_scope_id = 0;
+		*next_hop = nh_addr;
+	}
+
+	return (af);
+}
+
+/*
+ * Use TCP to check if the peer server is alive.  Create a TCP socket and
+ * then call connect() to reach the peer server.  If connect() does not
+ * return within the timeout period, the SIGALRM handler will be invoked
+ * and tell ilbd that the peer server is not alive.
+ */
+static int
+tcp_query(probe_param_t *param)
+{
+	int ret;
+	int sd, af;
+	struct sockaddr_storage dst_addr;
+	socklen_t dst_addr_len;
+	void *next_hop;
+	hrtime_t start, end;
+	uint32_t rtt;
+
+	ret = 0;
+	next_hop = NULL;
+
+	af = set_sockaddr(&dst_addr, &dst_addr_len, &next_hop, param);
+
+	if ((sd = socket(af, SOCK_STREAM, param->proto)) == -1)
+		return (-1);
+
+	/* DSR mode, need to set the next hop */
+	if (next_hop != NULL) {
+		if (af == AF_INET) {
+			if (setsockopt(sd, IPPROTO_IP, IP_NEXTHOP, next_hop,
+			    sizeof (ipaddr_t)) < 0) {
+				ret = -1;
+				goto out;
+			}
+		} else {
+			if (setsockopt(sd, IPPROTO_IPV6, IPV6_NEXTHOP,
+			    next_hop, sizeof (struct sockaddr_in6)) < 0) {
+				ret = -1;
+				goto out;
+			}
+		}
+	}
+
+	timeout_is_good = B_FALSE;
+	(void) alarm(param->timeout);
+	start = gethrtime();
+	if (connect(sd, (struct sockaddr *)&dst_addr, dst_addr_len) != 0) {
+		ret = -1;
+		goto out;
+	}
+	end = gethrtime();
+
+	rtt = (end - start) / (NANOSEC / MICROSEC);
+	if (rtt == 0)
+		rtt = 1;
+	(void) printf("%u", rtt);
+
+out:
+	(void) close(sd);
+	return (ret);
+}
+
+/*
+ * Check if the ICMP packet is a port unreachable message in respnsed to
+ * our probe.  Return -1 if no, 0 if yes.
+ */
+static int
+check_icmp_unreach_v4(struct icmp *icmph, probe_param_t *param)
+{
+	struct udphdr *udph;
+	struct ip *iph;
+
+	if (icmph->icmp_type != ICMP_UNREACH)
+		return (-1);
+	if (icmph->icmp_code != ICMP_UNREACH_PORT)
+		return (-1);
+
+	/* LINTED E_BAD_PTR_CAST_ALIGN */
+	iph = (struct ip *)((char *)icmph + ICMP_MINLEN);
+	if (iph->ip_p != IPPROTO_UDP)
+		return (-1);
+
+	/* LINTED E_BAD_PTR_CAST_ALIGN */
+	udph = (struct udphdr *)((char *)iph + (iph->ip_hl << 2));
+	if (udph->uh_dport != htons(param->port))
+		return (-1);
+	if (udph->uh_sport != param->lport)
+		return (-1);
+
+	/* All matched, it is a response to the probe we sent. */
+	return (0);
+}
+
+/*
+ * Check if the ICMP packet is a reply to our echo request.  Need to match
+ * the ID and sequence.
+ */
+static int
+check_icmp_echo_v4(struct icmp *icmph, probe_param_t *param)
+{
+	uint32_t cookie;
+	in_port_t port;
+
+	if (icmph->icmp_type != ICMP_ECHOREPLY)
+		return (-1);
+	if (icmph->icmp_id != param->echo_id)
+		return (-1);
+	if (icmph->icmp_seq != param->echo_seq)
+		return (-1);
+
+	bcopy(icmph->icmp_data, &cookie, sizeof (cookie));
+	if (cookie != param->echo_cookie)
+		return (-1);
+	bcopy(icmph->icmp_data + sizeof (cookie), &port, sizeof (port));
+	if (port != param->port)
+		return (-1);
+
+	/* All matched, it is a response to the echo we sent. */
+	return (0);
+}
+
+/* Verify if an ICMP packet is what we expect. */
+static int
+check_icmp_v4(char *buf, ssize_t rcvd, probe_param_t *param)
+{
+	struct ip *iph;
+	struct icmp *icmph;
+
+	/*
+	 * We can dereference the length field without worry since the stack
+	 * should not have sent up the packet if it is smaller than a normal
+	 * ICMPv4 packet.
+	 */
+	/* LINTED E_BAD_PTR_CAST_ALIGN */
+	iph = (struct ip *)buf;
+	/* LINTED E_BAD_PTR_CAST_ALIGN */
+	icmph = (struct icmp *)((char *)iph + (iph->ip_hl << 2));
+
+	/*
+	 * If we sent an UDP probe, check if the packet is a port
+	 * unreachable message in response to our probe.
+	 *
+	 * If we sent an ICMP echo request, check if the packet is a reply
+	 * to our echo request.
+	 */
+	if (param->probe == udp_probe) {
+		/* Is the packet large enough for further checking? */
+		if (rcvd < 2 * sizeof (struct ip) + ICMP_MINLEN +
+		    sizeof (struct udphdr)) {
+			return (-1);
+		}
+		return (check_icmp_unreach_v4(icmph, param));
+	} else {
+		if (rcvd < sizeof (struct ip) + ICMP_MINLEN)
+			return (-1);
+		return (check_icmp_echo_v4(icmph, param));
+	}
+}
+
+/*
+ * Check if the ICMPv6 packet is a port unreachable message in respnsed to
+ * our probe.  Return -1 if no, 0 if yes.
+ */
+static int
+check_icmp_unreach_v6(icmp6_t *icmp6h, probe_param_t *param)
+{
+	ip6_t *ip6h;
+	struct udphdr *udph;
+
+	if (icmp6h->icmp6_type != ICMP6_DST_UNREACH)
+		return (-1);
+	if (icmp6h->icmp6_code != ICMP6_DST_UNREACH_NOPORT)
+		return (-1);
+
+	/* LINTED E_BAD_PTR_CAST_ALIGN */
+	ip6h = (ip6_t *)((char *)icmp6h + ICMP6_MINLEN);
+	if (ip6h->ip6_nxt != IPPROTO_UDP)
+		return (-1);
+
+	udph = (struct udphdr *)(ip6h + 1);
+
+	if (udph->uh_dport != htons(param->port))
+		return (-1);
+	if (udph->uh_sport != param->lport)
+		return (-1);
+
+	return (0);
+}
+
+/*
+ * Check if the ICMPv6 packet is a reply to our echo request.  Need to match
+ * the ID and sequence.
+ */
+static int
+check_icmp_echo_v6(icmp6_t *icmp6h, probe_param_t *param)
+{
+	char *tmp;
+	uint32_t cookie;
+	in_port_t port;
+
+	if (icmp6h->icmp6_type != ICMP6_ECHO_REPLY)
+		return (-1);
+	if (icmp6h->icmp6_id != param->echo_id)
+		return (-1);
+	if (icmp6h->icmp6_seq != param->echo_seq)
+		return (-1);
+	tmp = (char *)icmp6h + ICMP6_MINLEN;
+	bcopy(tmp, &cookie, sizeof (cookie));
+	if (cookie != param->echo_cookie)
+		return (-1);
+	tmp += sizeof (cookie);
+	bcopy(tmp, &port, sizeof (port));
+	if (port != param->port)
+		return (-1);
+
+	/* All matched, it is a response to the echo we sent. */
+	return (0);
+}
+
+/* Verify if an ICMPv6 packet is what we expect. */
+static int
+check_icmp_v6(char *buf, ssize_t rcvd, probe_param_t *param)
+{
+	icmp6_t *icmp6h;
+
+	/* LINTED E_BAD_PTR_CAST_ALIGN */
+	icmp6h = (icmp6_t *)(buf);
+
+	/*
+	 * If we sent an UDP probe, check if the packet is a port
+	 * unreachable message.
+	 *
+	 * If we sent an ICMPv6 echo request, check if the packet is a reply.
+	 */
+	if (param->probe == udp_probe) {
+		/* Is the packet large enough for further checking? */
+		if (rcvd < sizeof (ip6_t) + ICMP6_MINLEN +
+		    sizeof (struct udphdr)) {
+			return (-1);
+		}
+		return (check_icmp_unreach_v6(icmp6h, param));
+	} else {
+		if (rcvd < ICMP6_MINLEN)
+			return (-1);
+		return (check_icmp_echo_v6(icmp6h, param));
+	}
+}
+
+/*
+ * Wait for an ICMP reply indefinitely.  If we get what we expect, return 0.
+ * If an error happnes, return -1.
+ */
+static int
+wait_icmp_reply(int af, int recv_sd, struct sockaddr_storage *exp_from,
+    probe_param_t *param)
+{
+	char buf[RECV_PKT_SZ];
+	socklen_t from_len;
+	ssize_t rcvd;
+	int ret;
+
+	for (;;) {
+		if (af == AF_INET) {
+			struct sockaddr_in v4_from;
+
+			from_len = sizeof (v4_from);
+			if ((rcvd = recvfrom(recv_sd, buf, RECV_PKT_SZ, 0,
+			    (struct sockaddr *)&v4_from, &from_len)) < 0) {
+				ret = -1;
+				break;
+			}
+
+			/* Packet not from our peer, ignore it. */
+			if ((((struct sockaddr_in *)exp_from)->sin_addr.s_addr)
+			    != v4_from.sin_addr.s_addr) {
+				continue;
+			}
+			if (check_icmp_v4(buf, rcvd, param) == 0) {
+				ret = 0;
+				break;
+			}
+		} else {
+			struct sockaddr_in6 v6_from;
+
+			from_len = sizeof (struct sockaddr_in6);
+			if ((rcvd = recvfrom(recv_sd, buf, RECV_PKT_SZ, 0,
+			    (struct sockaddr *)&v6_from, &from_len)) < 0) {
+				ret = -1;
+				break;
+			}
+
+			if (!IN6_ARE_ADDR_EQUAL(&(v6_from.sin6_addr),
+			    &((struct sockaddr_in6 *)exp_from)->sin6_addr)) {
+				continue;
+			}
+			if (check_icmp_v6(buf, rcvd, param) == 0) {
+				ret = 0;
+				break;
+			}
+		}
+	}
+	return (ret);
+}
+
+/* Return the local port used (network byte order) in a socket. */
+static int
+get_lport(int sd, in_port_t *lport)
+{
+	struct sockaddr_storage addr;
+	socklen_t addr_sz;
+
+	addr_sz = sizeof (addr);
+	if (getsockname(sd, (struct sockaddr *)&addr, &addr_sz) != 0)
+		return (-1);
+	if (addr.ss_family == AF_INET)
+		*lport = ((struct sockaddr_in *)&addr)->sin_port;
+	else
+		*lport = ((struct sockaddr_in6 *)&addr)->sin6_port;
+	return (0);
+}
+
+/*
+ * Use UDP to check if the peer server is alive.  Send a 0 length UDP packet
+ * to the peer server.  If there is no one listening, the peer IP stack
+ * should send back a port unreachable ICMP(v4/v6) packet.  If the peer
+ * server is alive, there should be no response.  So if we get SIGALRM,
+ * the peer is alive.
+ */
+static int
+udp_query(probe_param_t *param)
+{
+	int ret;
+	int send_sd, recv_sd, af;
+	struct sockaddr_storage dst_addr;
+	socklen_t addr_len;
+	void *next_hop;
+	char buf[1];
+	struct itimerval timeout;
+	uint64_t tm;
+
+	ret = 0;
+	next_hop = NULL;
+
+	af = set_sockaddr(&dst_addr, &addr_len, &next_hop, param);
+
+	if ((send_sd = socket(af, SOCK_DGRAM, param->proto)) == -1)
+		return (-1);
+	if ((recv_sd = socket(af, SOCK_RAW, (af == AF_INET) ? IPPROTO_ICMP :
+	    IPPROTO_ICMPV6)) == -1) {
+		return (-1);
+	}
+
+	/* DSR mode, need to set the next hop */
+	if (next_hop != NULL) {
+		if (af == AF_INET) {
+			if (setsockopt(send_sd, IPPROTO_IP, IP_NEXTHOP,
+			    next_hop, sizeof (ipaddr_t)) < 0) {
+				ret = -1;
+				goto out;
+			}
+		} else {
+			if (setsockopt(send_sd, IPPROTO_IPV6, IPV6_NEXTHOP,
+			    next_hop, sizeof (struct sockaddr_in6)) < 0) {
+				ret = -1;
+				goto out;
+			}
+		}
+	}
+
+	/*
+	 * If ilbd asks us to wait at most t, we will wait for at most
+	 * t', which is 3/4 of t.  If we wait for too long, ilbd may
+	 * timeout and kill us.
+	 */
+	timeout.it_interval.tv_sec = 0;
+	timeout.it_interval.tv_usec = 0;
+	tm = (param->timeout * MICROSEC >> 2) * 3;
+	if (tm > MICROSEC) {
+		timeout.it_value.tv_sec = tm / MICROSEC;
+		timeout.it_value.tv_usec = tm - (timeout.it_value.tv_sec *
+		    MICROSEC);
+	} else {
+		timeout.it_value.tv_sec = 0;
+		timeout.it_value.tv_usec = tm;
+	}
+	timeout_is_good = B_TRUE;
+	if (setitimer(ITIMER_REAL, &timeout, NULL) != 0) {
+		ret = -1;
+		goto out;
+	}
+
+	if (sendto(send_sd, buf, 0, 0, (struct sockaddr *)&dst_addr,
+	    addr_len) != 0) {
+		ret = -1;
+		goto out;
+	}
+	if ((ret = get_lport(send_sd, &param->lport)) != 0)
+		goto out;
+
+	/*
+	 * If the server app is listening, we should not get back a
+	 * response.  So if wait_icmp_reply() returns, either there
+	 * is an error or we get back something.
+	 */
+	(void) wait_icmp_reply(af, recv_sd, &dst_addr, param);
+	ret = -1;
+
+out:
+	(void) close(send_sd);
+	(void) close(recv_sd);
+	return (ret);
+}
+
+/*
+ * Size (in uint32_t) of the ping packet to be sent to server.  It includes
+ * a cookie (random number) + the target port.  The cookie and port are used
+ * for matching ping request since there can be many such ping packets sent
+ * to different servers from the same source address and using the same VIP.
+ * The last two bytes are for padding.
+ *
+ */
+#define	PING_PKT_LEN \
+	((ICMP_MINLEN + 2 * sizeof (uint32_t)) / sizeof (uint32_t))
+
+/*
+ * Try to get a random number from the pseudo random number device
+ * /dev/urandom.  If there is any error, return (uint32_t)gethrtime()
+ * as a back up.
+ */
+static uint32_t
+get_random(void)
+{
+	int fd;
+	uint32_t num;
+
+	if ((fd = open("/dev/urandom", O_RDONLY)) == -1)
+		return ((uint32_t)gethrtime());
+
+	if (read(fd, &num, sizeof (num)) != sizeof (num))
+		num = ((uint32_t)gethrtime());
+
+	(void) close(fd);
+	return (num);
+}
+
+/*
+ * Use ICMP(v4/v6) echo request to check if the peer server machine is
+ * reachable.  Send a echo request and expect to get back a echo reply.
+ */
+static int
+ping_query(probe_param_t *param)
+{
+	int ret;
+	int sd, af;
+	struct sockaddr_storage dst_addr;
+	socklen_t dst_addr_len;
+	void *next_hop;
+	hrtime_t start, end;
+	uint32_t rtt;
+	uint32_t buf[PING_PKT_LEN];
+	struct icmp *icmph;
+
+	ret = 0;
+	next_hop = NULL;
+
+	af = set_sockaddr(&dst_addr, &dst_addr_len, &next_hop, param);
+
+	if ((sd = socket(af, SOCK_RAW, (af == AF_INET) ? IPPROTO_ICMP :
+	    IPPROTO_ICMPV6)) == -1) {
+		return (-1);
+	}
+
+	/* DSR mode, need to set the next hop */
+	if (next_hop != NULL) {
+		if (af == AF_INET) {
+			if (setsockopt(sd, IPPROTO_IP, IP_NEXTHOP, next_hop,
+			    sizeof (ipaddr_t)) < 0) {
+				ret = -1;
+				goto out;
+			}
+		} else {
+			if (setsockopt(sd, IPPROTO_IPV6, IPV6_NEXTHOP,
+			    next_hop, sizeof (struct sockaddr_in6)) < 0) {
+				ret = -1;
+				goto out;
+			}
+		}
+	}
+
+	bzero(buf, sizeof (buf));
+	icmph = (struct icmp *)buf;
+	icmph->icmp_type = af == AF_INET ? ICMP_ECHO : ICMP6_ECHO_REQUEST;
+	icmph->icmp_code = 0;
+	icmph->icmp_cksum = 0;
+	icmph->icmp_id = htons(gethrtime() % USHRT_MAX);
+	icmph->icmp_seq = htons(gethrtime() % USHRT_MAX);
+
+	param->echo_cookie = get_random();
+	bcopy(&param->echo_cookie, icmph->icmp_data,
+	    sizeof (param->echo_cookie));
+	bcopy(&param->port, icmph->icmp_data + sizeof (param->echo_cookie),
+	    sizeof (param->port));
+	icmph->icmp_cksum = in_cksum((ushort_t *)buf, sizeof (buf));
+	param->echo_id = icmph->icmp_id;
+	param->echo_seq = icmph->icmp_seq;
+
+	timeout_is_good = B_FALSE;
+	(void) alarm(param->timeout);
+	start = gethrtime();
+	if (sendto(sd, buf, sizeof (buf), 0, (struct sockaddr *)&dst_addr,
+	    dst_addr_len) != sizeof (buf)) {
+		ret = -1;
+		goto out;
+	}
+	if (wait_icmp_reply(af, sd, &dst_addr, param) != 0) {
+		ret = -1;
+		goto out;
+	}
+	end = gethrtime();
+
+	rtt = (end - start) / (NANOSEC / MICROSEC);
+	if (rtt == 0)
+		rtt = 1;
+	(void) printf("%u", rtt);
+
+out:
+	(void) close(sd);
+	return (ret);
+}
+
+int
+main(int argc, char *argv[])
+{
+	probe_param_t param;
+	int ret;
+
+	/* ilbd should pass in PROG_ARGC parameters. */
+	if (argc != PROG_ARGC) {
+		(void) printf("-1");
+		return (-1);
+	}
+
+	if (signal(SIGALRM, probe_exit) == SIG_ERR) {
+		(void) printf("-1");
+		return (-1);
+	}
+
+	if (!parse_probe_param(argv, &param)) {
+		(void) printf("-1");
+		return (-1);
+	}
+
+	switch (param.probe) {
+	case ping_probe:
+		ret = ping_query(&param);
+		break;
+	case tcp_probe:
+		ret = tcp_query(&param);
+		break;
+	case udp_probe:
+		ret = udp_query(&param);
+		break;
+	}
+
+	if (ret == -1)
+		(void) printf("-1");
+
+	return (ret);
+}

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/usr/src/cmd/cmd-inet/usr.lib/ilbd/ilbd.h	Tue Nov 03 23:15:19 2009 -0800
@@ -0,0 +1,435 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+#ifndef _ILBD_H
+#define	_ILBD_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdio.h>
+#include <ucred.h>
+#include <pwd.h>
+#include <priv.h>
+#include <stdarg.h>
+#include <syslog.h>
+#include <sys/list.h>
+#include <libscf.h>
+#include <libintl.h>
+#include <locale.h>
+#include <libinetutil.h>
+#include <auth_list.h>
+#include <bsm/adt.h>
+#include <bsm/adt_event.h>
+
+#define	SGNAME_SZ	80
+#define	ILB_FMRI	"svc:/network/loadbalancer/ilb:default"
+
+#define	HC_ACTION		ILB_SRV_DISABLED_HC
+#define	ADMIN_ACTION		ILB_SRV_DISABLED_ADMIN
+
+/* Max name and value length for scf properties */
+#define	ILBD_MAX_NAME_LEN	ilbd_scf_limit(SCF_LIMIT_MAX_NAME_LENGTH)
+#define	ILBD_MAX_VALUE_LEN	ilbd_scf_limit(SCF_LIMIT_MAX_VALUE_LENGTH)
+
+/* Different events ILBD is interested in. */
+typedef enum {
+	ILBD_EVENT_NEW_REQ,	/* New client request */
+	ILBD_EVENT_REQ,		/* Client request comes in */
+	ILBD_EVENT_REP_OK,	/* Reply channel to client is writeable */
+	ILBD_EVENT_PROBE,	/* A HC returns some result */
+	ILBD_EVENT_TIMER	/* ilbd_timer_q fired */
+} ilbd_event_t;
+
+typedef enum {
+	ILBD_SCF_RULE,	/* prop group for rules */
+	ILBD_SCF_SG,	/* prop group for servergroups */
+	ILBD_SCF_HC	/* prop group for healthchecks */
+} ilbd_scf_pg_type_t;
+
+typedef enum {
+	ILBD_SCF_CREATE,
+	ILBD_SCF_DESTROY,
+	ILBD_SCF_ENABLE_DISABLE
+} ilbd_scf_cmd_t;
+
+typedef enum {
+	ILBD_STRING,	/* string */
+	ILBD_INT,	/* int */
+	ILBD_ADDR_V4,	/* ipv4 addr */
+	ILBD_ADDR_V6	/* ipv6 addr */
+} ilbd_scf_data_type_t;
+
+typedef enum {
+	stat_enable_server,
+	stat_disable_server,
+	stat_declare_srv_dead,
+	stat_declare_srv_alive
+} ilbd_srv_status_ind_t;
+
+/*
+ * All user struct pointer passed to port_associate() should have the first
+ * field as ilbd_event_t.  The following struct can be used to find the
+ * event.
+ */
+typedef struct {
+	ilbd_event_t	ev;
+} ilbd_event_obj_t;
+
+typedef struct {
+	ilbd_event_t	ev;
+	timer_t		timerid;
+} ilbd_timer_event_obj_t;
+
+typedef struct ilbd_srv {
+	list_node_t	isv_srv_link;
+	ilb_sg_srv_t	isv_srv;
+#define	isv_addr	isv_srv.sgs_addr
+#define	isv_minport	isv_srv.sgs_minport
+#define	isv_maxport	isv_srv.sgs_maxport
+#define	isv_flags	isv_srv.sgs_flags
+#define	isv_id		isv_srv.sgs_id
+#define	isv_srvID	isv_srv.sgs_srvID
+} ilbd_srv_t;
+
+#define	MAX_SRVCOUNT	1000
+#define	MAX_SRVID	(MAX_SRVCOUNT - 1)
+#define	BAD_SRVID	(-1)
+
+typedef struct ilbd_sg {
+	list_t		isg_srvlist;	/* list of ilbd_srv_t */
+	char		isg_name[ILB_SGNAME_SZ];
+	int32_t		isg_srvcount;
+	int32_t		isg_max_id;
+	list_t		isg_rulelist;	/* list of ilbd_rule_t */
+	char		isg_id_arr[MAX_SRVCOUNT]; /* for server ID allocation */
+
+	list_node_t	isg_link;	/* linkage for sg list */
+} ilbd_sg_t;
+
+typedef struct ilbd_rule {
+	list_node_t		irl_link;
+	list_node_t		irl_sglink;
+	ilbd_sg_t		*irl_sg;
+	ilb_rule_info_t		irl_info;
+#define	irl_flags	irl_info.rl_flags
+#define	irl_name	irl_info.rl_name
+#define	irl_vip		irl_info.rl_vip
+#define	irl_proto	irl_info.rl_proto
+#define	irl_ipversion	irl_info.rl_ipversion
+#define	irl_minport	irl_info.rl_minport
+#define	irl_maxport	irl_info.rl_maxport
+#define	irl_algo	irl_info.rl_algo
+#define	irl_topo	irl_info.rl_topo
+#define	irl_nat_src_start	irl_info.rl_nat_src_start
+#define	irl_nat_src_end	irl_info.rl_nat_src_end
+#define	irl_stickymask	irl_info.rl_stickymask
+#define	irl_conndrain	irl_info.rl_conndrain
+#define	irl_nat_timeout	irl_info.rl_nat_timeout
+#define	irl_sticky_timeout	irl_info.rl_sticky_timeout
+#define	irl_hcport	irl_info.rl_hcport
+#define	irl_hcpflag	irl_info.rl_hcpflag
+#define	irl_sgname	irl_info.rl_sgname
+#define	irl_hcname	irl_info.rl_hcname
+} ilbd_rule_t;
+
+/*
+ * Health check related definitions
+ */
+
+/* Default health check probe program provided */
+#define	ILB_PROBE_PROTO	"/usr/lib/inet/ilb/ilb_probe"
+
+/* Command name (argv[0]) passed to ilb_probe to indicate a ping test */
+#define	ILB_PROBE_PING	"ilb_ping"
+
+/* Use the first character of the rule's hcname to decide if rule has HC. */
+#define	RULE_HAS_HC(irl)	((irl)->irl_info.rl_hcname[0] != '\0')
+
+/* Type of probe test */
+typedef enum {
+	ILBD_HC_PING = 1,	/* ICMP Echo probe */
+	ILBD_HC_TCP,		/* TCP connect probe */
+	ILBD_HC_UDP,		/* UDP packet probe */
+	ILBD_HC_USER		/* User supplied probe */
+} ilbd_hc_test_t;
+
+/* Struct representing a hc object in ilbd */
+typedef struct {
+	list_node_t	ihc_link;	/* List linkage */
+
+	ilb_hc_info_t	ihc_info;
+/* Short hand for the fields inside ilb_hc_info_t */
+#define	ihc_name	ihc_info.hci_name
+#define	ihc_test	ihc_info.hci_test
+#define	ihc_timeout	ihc_info.hci_timeout
+#define	ihc_count	ihc_info.hci_count
+#define	ihc_interval	ihc_info.hci_interval
+#define	ihc_def_ping	ihc_info.hci_def_ping
+
+	ilbd_hc_test_t	ihc_test_type;	/* Type of probe test */
+	int		ihc_rule_cnt;	/* Num of rules associated with hc */
+	list_t		ihc_rules;	/* Rules associated with this hc */
+} ilbd_hc_t;
+
+struct ilbd_hc_srv_s;
+
+/*
+ * Struct representing a hc rule object
+ *
+ * hcr_link: list linkage
+ * hcr_rule: pointer to the ilbd rule object
+ * hcr_servers: list of servers of this rule
+ */
+typedef struct {
+	list_node_t		hcr_link;
+	ilbd_rule_t const 	*hcr_rule;
+	list_t			hcr_servers;
+} ilbd_hc_rule_t;
+
+struct ilbd_hc_srv_s;
+
+/*
+ * Struct representing a event of the probe process
+ *
+ * ihp_ev: the event type, which is ILBD_EVENT_PROBE
+ * ihp_srv: pointer to the hc server object
+ * ihp_pid: pid of the probe process
+ * ihp_done: is ilbd done reading the output of the probe process
+ */
+typedef struct {
+	ilbd_event_t		ihp_ev;
+	struct ilbd_hc_srv_s	*ihp_srv;
+	pid_t			ihp_pid;
+	boolean_t		ihp_done;
+} ilbd_hc_probe_event_t;
+
+/*
+ * ilbd_hc_srv_t state
+ *
+ * ihd_hc_def_pinging: the default ping should be run
+ * ihd-hc_probing: the probe process should be started
+ */
+enum ilbd_hc_state {
+	ilbd_hc_def_pinging,
+	ilbd_hc_probing
+};
+
+/*
+ * Struct representing a server associated with a hc object
+ *
+ * shc_srv_link: list linkage
+ * shc_hc: pointer to the hc object
+ * shc_hc_rule: pointer to the hc rule object
+ * shc_sg_srv: pointer to the server group object
+ * shc_tid: timeout ID
+ * shc_cur_cnt: number of times the hc probe has been run
+ * shc_fail_cnt: number of consecutive probe failure
+ * shc_status: health status
+ * shc_rtt: rtt (in micro sec) to the backend server
+ * shc_lasttimer: last time a probe sequence is executed
+ * shc_nexttime: next time a probe sequence is executed
+ * shc_state: hc probe state
+ * shc_child_pid: pid of the probe process
+ * shc_child_fd: fd to the output of the probe process
+ * shc_ev: event object of the probe process
+ * shc_ev_port: event port of the event object
+ */
+typedef struct ilbd_hc_srv_s {
+	list_node_t		shc_srv_link;
+	ilbd_hc_t		*shc_hc;
+	ilbd_hc_rule_t		*shc_hc_rule;
+	ilb_sg_srv_t const	*shc_sg_srv;
+
+	iu_timer_id_t		shc_tid;
+	uint_t			shc_cur_cnt;
+	uint_t			shc_fail_cnt;
+	ilb_hc_srv_status_t	shc_status;
+	uint32_t		shc_rtt;
+	time_t			shc_lasttime;
+	time_t			shc_nexttime;
+
+	enum ilbd_hc_state	shc_state;
+	pid_t			shc_child_pid;
+	int			shc_child_fd;
+	ilbd_hc_probe_event_t	*shc_ev;
+	int			shc_ev_port;
+} ilbd_hc_srv_t;
+
+/*
+ * Structure for holding audit server and servergroup event
+ * data. Not all events use all members of the structure.
+ */
+typedef struct audit_sg_event_data {
+	char	*ed_server_address;	/* server's IP address */
+	char	*ed_serverid;   /* serverid. */
+	uint16_t	ed_minport;	/* server's minport */
+	uint16_t	ed_maxport;	/* server's maxport */
+	char		*ed_sgroup;	/* servergroup */
+} audit_sg_event_data_t;
+
+/* Struct to store client info */
+typedef struct {
+	ilbd_event_t	cli_ev;
+	int	cli_sd;
+	struct passwd	cli_pw;
+	size_t		cli_pw_bufsz;
+	char		*cli_pw_buf;
+	ilbd_cmd_t	cli_cmd;
+	ilb_comm_t	*cli_saved_reply;
+	size_t		cli_saved_size;
+	ucred_t		*cli_peer_ucredp; /* needed for auditing */
+} ilbd_client_t;
+
+void		ilbd_reply_ok(uint32_t *, size_t *);
+void		ilbd_reply_err(uint32_t *, size_t *, ilb_status_t);
+
+ilb_status_t	ilbd_check_client_config_auth(const struct passwd *);
+ilb_status_t	ilbd_check_client_enable_auth(const struct passwd *);
+ilb_status_t	ilbd_retrieve_names(ilbd_cmd_t, uint32_t *, size_t *);
+void		i_setup_sg_hlist(void);
+void		i_setup_rule_hlist(void);
+void		logperror(const char *);
+ilb_status_t	ilbd_add_server_to_group(ilb_sg_info_t *, int,
+	const struct passwd *, ucred_t *);
+ilb_status_t	ilbd_rem_server_from_group(ilb_sg_info_t *, int,
+	const struct passwd *, ucred_t *);
+ilb_status_t	ilbd_create_sg(ilb_sg_info_t *, int,
+	const struct passwd *, ucred_t *);
+
+ilb_status_t	ilbd_destroy_sg(const char *, const struct passwd *,
+		ucred_t *);
+ilb_status_t	ilbd_retrieve_sg_hosts(const char *, uint32_t *, size_t *);
+
+ilb_status_t	ilbd_enable_server(ilb_sg_info_t *, const struct passwd *,
+		ucred_t *);
+ilb_status_t	ilbd_disable_server(ilb_sg_info_t *, const struct passwd *,
+		ucred_t *);
+ilb_status_t	ilbd_k_Xable_server(const struct in6_addr *, const char *,
+		    ilbd_srv_status_ind_t);
+
+ilb_status_t	i_add_srv2krules(list_t *, ilb_sg_srv_t *, int);
+ilb_status_t	i_rem_srv_frm_krules(list_t *, ilb_sg_srv_t *, int);
+int		ilbd_get_num_krules(void);
+ilb_status_t	ilbd_get_krule_names(ilbd_namelist_t **, int);
+ilb_status_t	ilb_get_krule_servers(ilb_sg_info_t *);
+ilbd_sg_t	*i_find_sg_byname(const char *);
+ilb_status_t	i_check_srv2rules(list_t *, ilb_sg_srv_t *);
+
+ilb_status_t	ilbd_address_to_srvID(ilb_sg_info_t *, uint32_t *, size_t *);
+ilb_status_t	ilbd_srvID_to_address(ilb_sg_info_t *, uint32_t *, size_t *);
+
+ilb_status_t	do_ioctl(void *, ssize_t);
+
+ilb_status_t	ilbd_create_rule(ilb_rule_info_t *, int, const struct passwd *,
+		ucred_t *);
+ilb_status_t	ilbd_retrieve_rule(ilbd_name_t, uint32_t *, size_t *);
+
+ilb_status_t	ilbd_destroy_rule(ilbd_name_t, const struct passwd *,
+		ucred_t *);
+ilb_status_t	ilbd_enable_rule(ilbd_name_t, const struct passwd *, ucred_t *);
+ilb_status_t	ilbd_disable_rule(ilbd_name_t, const struct passwd *,
+		ucred_t *);
+
+boolean_t	is_debugging_on(void);
+ilb_status_t	ilbd_sg_check_rule_port(ilbd_sg_t *, ilb_rule_info_t *);
+
+void		ilbd_enable_debug(void);
+ilb_status_t	ilb_map_errno2ilbstat(int);
+
+ilb_status_t	i_attach_rule2sg(ilbd_sg_t *, ilbd_rule_t *);
+
+/* Logging routine and macros */
+void		ilbd_log(int, const char *, ...);
+#define	logerr(...)	ilbd_log(LOG_ERR, __VA_ARGS__)
+#define	logdebug(...)	ilbd_log(LOG_DEBUG, __VA_ARGS__)
+
+/* Health check manipulation routines */
+void		i_ilbd_setup_hc_list(void);
+ilb_status_t	ilbd_create_hc(const ilb_hc_info_t *, int,
+		    const struct passwd *, ucred_t *);
+ilb_status_t	ilbd_destroy_hc(const char *, const struct passwd *, ucred_t *);
+ilbd_hc_t	*ilbd_get_hc(const char *);
+ilb_status_t	ilbd_get_hc_info(const char *, uint32_t *, size_t *);
+ilb_status_t	ilbd_get_hc_srvs(const char *, uint32_t *, size_t *);
+ilb_status_t	ilbd_hc_associate_rule(const ilbd_rule_t *, int);
+ilb_status_t	ilbd_hc_dissociate_rule(const ilbd_rule_t *);
+ilb_status_t	ilbd_hc_add_server(const ilbd_rule_t *, const ilb_sg_srv_t *,
+		    int);
+ilb_status_t	ilbd_hc_del_server(const ilbd_rule_t *, const ilb_sg_srv_t *);
+ilb_status_t	ilbd_hc_enable_rule(const ilbd_rule_t *);
+ilb_status_t	ilbd_hc_disable_rule(const ilbd_rule_t *);
+ilb_status_t	ilbd_hc_enable_server(const ilbd_rule_t *,
+		    const ilb_sg_srv_t *);
+ilb_status_t	ilbd_hc_disable_server(const ilbd_rule_t *,
+		    const ilb_sg_srv_t *);
+
+/* Health check timer routines */
+void		ilbd_hc_probe_return(int, int, int, ilbd_hc_probe_event_t *);
+void		ilbd_hc_timer_init(int, ilbd_timer_event_obj_t *);
+void		ilbd_hc_timeout(void);
+void		ilbd_hc_timer_update(ilbd_timer_event_obj_t *);
+
+/* Show NAT info routines */
+ilb_status_t	ilbd_show_nat(void *, const ilb_comm_t *, uint32_t *,
+		    size_t *);
+void		ilbd_show_nat_cleanup(void);
+
+
+/* Show sticky info routines */
+ilb_status_t	ilbd_show_sticky(void *, const ilb_comm_t *, uint32_t *,
+		    size_t *);
+void		ilbd_show_sticky_cleanup(void);
+
+ilb_status_t	ilbd_create_pg(ilbd_scf_pg_type_t, void *);
+ilb_status_t	ilbd_destroy_pg(ilbd_scf_pg_type_t, const char *);
+ilb_status_t	ilbd_change_prop(ilbd_scf_pg_type_t, const char *,
+		    const char *, void *);
+void		ilbd_scf_str_to_ip(int, char *, struct in6_addr *);
+ilb_status_t	ilbd_scf_ip_to_str(uint16_t, struct in6_addr *, scf_type_t *,
+		    char *);
+ilb_status_t	ilbd_scf_add_srv(ilbd_sg_t *, ilbd_srv_t *);
+ilb_status_t	ilbd_scf_del_srv(ilbd_sg_t *, ilbd_srv_t *);
+int		ilbd_scf_limit(int);
+
+ilb_status_t	ilbd_walk_rule_pgs(ilb_status_t (*)(ilb_rule_info_t *, int,
+		    const struct passwd *, ucred_t *), void *, void *);
+ilb_status_t	ilbd_walk_sg_pgs(ilb_status_t (*)(ilb_sg_info_t *, int,
+		    const struct passwd *, ucred_t *), void *, void *);
+ilb_status_t	ilbd_walk_hc_pgs(ilb_status_t (*)(const ilb_hc_info_t *, int,
+		    const struct passwd *, ucred_t *), void *, void *);
+void		ilbd_addr2str(struct in6_addr *, char *, size_t);
+void		addr2str(ilb_ip_addr_t, char *, size_t);
+void		ilbd_algo_to_str(ilb_algo_t, char *);
+void		ilbd_topo_to_str(ilb_topo_t, char *);
+void		ilbd_ip_to_str(uint16_t, struct in6_addr *, char *);
+int		ilberror2auditerror(ilb_status_t);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _ILBD_H */

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/usr/src/cmd/cmd-inet/usr.lib/ilbd/ilbd.xml	Tue Nov 03 23:15:19 2009 -0800
@@ -0,0 +1,115 @@
+<?xml version="1.0"?>
+<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
+<!--
+ Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
+ Use is subject to license terms.
+
+ CDDL HEADER START
+
+ The contents of this file are subject to the terms of the
+ Common Development and Distribution License (the "License").
+ You may not use this file except in compliance with the License.
+
+ You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ or http://www.opensolaris.org/os/licensing.
+ See the License for the specific language governing permissions
+ and limitations under the License.
+
+ When distributing Covered Code, include this CDDL HEADER in each
+ file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ If applicable, add the following below this CDDL HEADER, with the
+ fields enclosed by brackets "[]" replaced with your own identifying
+ information: Portions Copyright [yyyy] [name of copyright owner]
+
+ CDDL HEADER END
+
+	NOTE:  This service manifest is not editable; its contents will
+	be overwritten by package or patch operations, including
+	operating system upgrade.  Make customizations in a different
+	file.
+
+	Service manifest for Integrated Load balancer(ILB).
+-->
+
+<service_bundle type='manifest' name='SUNWilbr:ilb'>
+
+<service
+	name='network/loadbalancer/ilb'
+	type='service'
+	version='1'>
+
+	<create_default_instance enabled='false' />
+
+	<single_instance />
+
+	<!--
+	  Ensure that name services is enabled before ILB service begins.
+	-->
+	<dependency
+		name='name-services'
+		grouping='require_all'
+		restart_on='error'
+		type='service'>
+		<service_fmri value='svc:/milestone/name-services' />
+	</dependency>
+
+
+	<!--
+	  Ensure that forwarding service is enabled before load balancing
+	  service begins. The forwarding service is dependent on
+	  the filesystem and devices services.
+	-->
+	<dependency
+		name='network'
+		grouping='require_any'
+		restart_on='error'
+		type='service'>
+		<service_fmri value='svc:/network/ipv4-forwarding' />
+		<service_fmri value='svc:/network/ipv6-forwarding' />
+	</dependency>
+
+	<exec_method
+		type='method'
+		name='stop'
+		exec=':kill'
+		timeout_seconds='60' >
+	</exec_method>
+
+	<exec_method
+		type='method'
+		name='start'
+		exec='/usr/lib/inet/ilbd'
+		timeout_seconds='60' >
+		<method_context>
+			<method_credential user='root' group='root' />
+		</method_context>
+	</exec_method>
+
+        <!-- to start stop routing services -->
+        <property_group name='general' type='framework'>
+                <propval name='action_authorization' type='astring'
+                        value='solaris.smf.manage.ilb' />
+                <propval name='value_authorization' type='astring'
+                        value='solaris.smf.manage.ilb' />
+        </property_group>
+
+
+	<stability value='Unstable' />
+
+	<template>
+
+	<common_name>
+		<loctext xml:lang='C'> Integrated layer 3/4 load balancer
+		</loctext>
+	</common_name>
+	<documentation>
+		<manpage title='ilbd' section='1M'
+			manpath='/usr/share/man' />
+		<manpage title='ilbadm' section='1M'
+			manpath='/usr/share/man' />
+	</documentation>
+
+	</template>
+</service>
+
+</service_bundle>

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/usr/src/cmd/cmd-inet/usr.lib/ilbd/ilbd_hc.c	Tue Nov 03 23:15:19 2009 -0800
@@ -0,0 +1,1574 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <sys/list.h>
+#include <sys/stropts.h>
+#include <sys/siginfo.h>
+#include <sys/wait.h>
+#include <arpa/inet.h>
+#include <netinet/in.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <strings.h>
+#include <stddef.h>
+#include <unistd.h>
+#include <libilb.h>
+#include <port.h>
+#include <time.h>
+#include <signal.h>
+#include <assert.h>
+#include <errno.h>
+#include <spawn.h>
+#include <fcntl.h>
+#include <limits.h>
+#include "libilb_impl.h"
+#include "ilbd.h"
+
+/* Global list of HC objects */
+list_t ilbd_hc_list;
+
+/* Timer queue for all hc related timers. */
+static iu_tq_t *ilbd_hc_timer_q;
+
+/* Indicate whether the timer needs to be updated */
+static boolean_t hc_timer_restarted;
+
+static void ilbd_hc_probe_timer(iu_tq_t *, void *);
+static ilb_status_t ilbd_hc_restart_timer(ilbd_hc_t *, ilbd_hc_srv_t *);
+static boolean_t ilbd_run_probe(ilbd_hc_srv_t *);
+
+#define	MAX(a, b)	((a) > (b) ? (a) : (b))
+
+/*
+ * Number of arguments passed to a probe.  argc[0] is the path name of
+ * the probe.
+ */
+#define	HC_PROBE_ARGC	8
+
+/*
+ * Max number of characters to be read from the output of a probe.  It
+ * is long enough to read in a 64 bit integer.
+ */
+#define	HC_MAX_PROBE_OUTPUT	24
+
+void
+i_ilbd_setup_hc_list(void)
+{
+	list_create(&ilbd_hc_list, sizeof (ilbd_hc_t),
+	    offsetof(ilbd_hc_t, ihc_link));
+}
+
+/*
+ * Given a hc object name, return a pointer to hc object if found.
+ */
+ilbd_hc_t *
+ilbd_get_hc(const char *name)
+{
+	ilbd_hc_t *hc;
+
+	for (hc = list_head(&ilbd_hc_list); hc != NULL;
+	    hc = list_next(&ilbd_hc_list, hc)) {
+		if (strcasecmp(hc->ihc_name, name) == 0)
+			return (hc);
+	}
+	return (NULL);
+}
+
+/*
+ * Generates an audit record for create-healthcheck,
+ * delete-healtcheck subcommands.
+ */
+static void
+ilbd_audit_hc_event(const char *audit_hcname,
+    const ilb_hc_info_t *audit_hcinfo, ilbd_cmd_t cmd,
+    ilb_status_t rc, ucred_t *ucredp)
+{
+	adt_session_data_t	*ah;
+	adt_event_data_t	*event;
+	au_event_t	flag;
+	int	audit_error;
+
+	if ((ucredp == NULL) && (cmd == ILBD_CREATE_HC))  {
+		/*
+		 * we came here from the path where ilbd incorporates
+		 * the configuration that is listed in SCF:
+		 * i_ilbd_read_config->ilbd_walk_hc_pgs->
+		 *   ->ilbd_scf_instance_walk_pg->ilbd_create_hc
+		 * We skip auditing in that case
+		 */
+		logdebug("ilbd_audit_hc_event: skipping auditing");
+		return;
+	}
+
+	if (adt_start_session(&ah, NULL, 0) != 0) {
+		logerr("ilbd_audit_hc_event: adt_start_session failed");
+		exit(EXIT_FAILURE);
+	}
+	if (adt_set_from_ucred(ah, ucredp, ADT_NEW) != 0) {
+		(void) adt_end_session(ah);
+		logerr("ilbd_audit_rule_event: adt_set_from_ucred failed");
+		exit(EXIT_FAILURE);
+	}
+	if (cmd == ILBD_CREATE_HC)
+		flag = ADT_ilb_create_healthcheck;
+	else if (cmd == ILBD_DESTROY_HC)
+		flag = ADT_ilb_delete_healthcheck;
+
+	if ((event = adt_alloc_event(ah, flag)) == NULL) {
+		logerr("ilbd_audit_hc_event: adt_alloc_event failed");
+		exit(EXIT_FAILURE);
+	}
+	(void) memset((char *)event, 0, sizeof (adt_event_data_t));
+
+	switch (cmd) {
+	case ILBD_CREATE_HC:
+		event->adt_ilb_create_healthcheck.auth_used =
+		    NET_ILB_CONFIG_AUTH;
+		event->adt_ilb_create_healthcheck.hc_test =
+		    (char *)audit_hcinfo->hci_test;
+		event->adt_ilb_create_healthcheck.hc_name =
+		    (char *)audit_hcinfo->hci_name;
+
+		/*
+		 * If the value 0 is stored, the default values are
+		 * set in the kernel. User land does not know about them
+		 * So if the user does not specify them, audit record
+		 * will show them as 0
+		 */
+		event->adt_ilb_create_healthcheck.hc_timeout =
+		    audit_hcinfo->hci_timeout;
+		event->adt_ilb_create_healthcheck.hc_count =
+		    audit_hcinfo->hci_count;
+		event->adt_ilb_create_healthcheck.hc_interval =
+		    audit_hcinfo->hci_interval;
+		break;
+	case ILBD_DESTROY_HC:
+		event->adt_ilb_delete_healthcheck.auth_used =
+		    NET_ILB_CONFIG_AUTH;
+		event->adt_ilb_delete_healthcheck.hc_name =
+		    (char *)audit_hcname;
+		break;
+	}
+
+	/* Fill in success/failure */
+	if (rc == ILB_STATUS_OK) {
+		if (adt_put_event(event, ADT_SUCCESS, ADT_SUCCESS) != 0) {
+			logerr("ilbd_audit_hc_event: adt_put_event failed");
+			exit(EXIT_FAILURE);
+		}
+	} else {
+		audit_error = ilberror2auditerror(rc);
+		if (adt_put_event(event, ADT_FAILURE, audit_error) != 0) {
+			logerr("ilbd_audit_hc_event: adt_put_event failed");
+			exit(EXIT_FAILURE);
+		}
+	}
+	adt_free_event(event);
+	(void) adt_end_session(ah);
+}
+
+/*
+ * Given the ilb_hc_info_t passed in (from the libilb), create a hc object
+ * in ilbd.  The parameter ev_port is not used, refer to comments of
+ * ilbd_create_sg() in ilbd_sg.c
+ */
+/* ARGSUSED */
+ilb_status_t
+ilbd_create_hc(const ilb_hc_info_t *hc_info, int ev_port,
+    const struct passwd *ps, ucred_t *ucredp)
+{
+	ilbd_hc_t *hc;
+	ilb_status_t ret = ILB_STATUS_OK;
+
+	/*
+	 * ps == NULL is from the daemon when it starts and load configuration
+	 * ps != NULL is from client.
+	 */
+	if (ps != NULL) {
+		ret = ilbd_check_client_config_auth(ps);
+		if (ret != ILB_STATUS_OK) {
+			ilbd_audit_hc_event(NULL, hc_info, ILBD_CREATE_HC,
+			    ret, ucredp);
+			return (ret);
+		}
+	}
+
+	if (hc_info->hci_name[0] == '\0') {
+		logdebug("ilbd_create_hc: missing healthcheck info");
+		ilbd_audit_hc_event(NULL, hc_info, ILBD_CREATE_HC,
+		    ILB_STATUS_ENOHCINFO, ucredp);
+		return (ILB_STATUS_ENOHCINFO);
+	}
+
+	hc = ilbd_get_hc(hc_info->hci_name);
+	if (hc != NULL) {
+		logdebug("ilbd_create_hc: healthcheck name %s already"
+		    " exists", hc_info->hci_name);
+		ilbd_audit_hc_event(NULL, hc_info, ILBD_CREATE_HC,
+		    ILB_STATUS_EEXIST, ucredp);
+		return (ILB_STATUS_EEXIST);
+	}
+
+	/*
+	 * Sanity check on user supplied probe.  The given path name
+	 * must be a full path name (starts with '/') and is
+	 * executable.
+	 */
+	if (strcasecmp(hc_info->hci_test, ILB_HC_STR_TCP) != 0 &&
+	    strcasecmp(hc_info->hci_test, ILB_HC_STR_UDP) != 0 &&
+	    strcasecmp(hc_info->hci_test, ILB_HC_STR_PING) != 0 &&
+	    (hc_info->hci_test[0] != '/' ||
+	    access(hc_info->hci_test, X_OK) == -1)) {
+		if (errno == ENOENT) {
+			logdebug("ilbd_create_hc: user script %s doesn't "
+			    "exist", hc_info->hci_test);
+			ilbd_audit_hc_event(NULL, hc_info, ILBD_CREATE_HC,
+			    ILB_STATUS_ENOENT, ucredp);
+			return (ILB_STATUS_ENOENT);
+		} else {
+			logdebug("ilbd_create_hc: user script %s is "
+			    "invalid", hc_info->hci_test);
+			ilbd_audit_hc_event(NULL, hc_info, ILBD_CREATE_HC,
+			    ILB_STATUS_EINVAL, ucredp);
+			return (ILB_STATUS_EINVAL);
+		}
+	}
+
+	/* Create and add the hc object */
+	hc = calloc(1, sizeof (ilbd_hc_t));
+	if (hc == NULL) {
+		ilbd_audit_hc_event(NULL, hc_info, ILBD_CREATE_HC,
+		    ILB_STATUS_ENOMEM, ucredp);
+		return (ILB_STATUS_ENOMEM);
+	}
+	(void) memcpy(&hc->ihc_info, hc_info, sizeof (ilb_hc_info_t));
+	if (strcasecmp(hc->ihc_test, ILB_HC_STR_TCP) == 0)
+		hc->ihc_test_type = ILBD_HC_TCP;
+	else if (strcasecmp(hc->ihc_test, ILB_HC_STR_UDP) == 0)
+		hc->ihc_test_type = ILBD_HC_UDP;
+	else if (strcasecmp(hc->ihc_test, ILB_HC_STR_PING) == 0)
+		hc->ihc_test_type = ILBD_HC_PING;
+	else
+		hc->ihc_test_type = ILBD_HC_USER;
+	list_create(&hc->ihc_rules, sizeof (ilbd_hc_rule_t),
+	    offsetof(ilbd_hc_rule_t, hcr_link));
+
+	/* Update SCF */
+	if (ps != NULL) {
+		if ((ret = ilbd_create_pg(ILBD_SCF_HC, (void *)hc)) !=
+		    ILB_STATUS_OK) {
+			ilbd_audit_hc_event(NULL, hc_info, ILBD_CREATE_HC,
+			    ret, ucredp);
+			free(hc);
+			return (ret);
+		}
+	}
+
+	/* Everything is fine, now add it to the global list. */
+	list_insert_tail(&ilbd_hc_list, hc);
+	ilbd_audit_hc_event(NULL, hc_info, ILBD_CREATE_HC, ret, ucredp);
+	return (ret);
+}
+
+/*
+ * Given a name of a hc object, destroy it.
+ */
+ilb_status_t
+ilbd_destroy_hc(const char *hc_name, const struct passwd *ps,
+    ucred_t *ucredp)
+{
+	ilb_status_t ret;
+	ilbd_hc_t *hc;
+
+	/*
+	 * No need to check ps == NULL, daemon won't call any destroy func
+	 * at start up.
+	 */
+	ret = ilbd_check_client_config_auth(ps);
+	if (ret != ILB_STATUS_OK) {
+		ilbd_audit_hc_event(hc_name, NULL, ILBD_DESTROY_HC,
+		    ret, ucredp);
+		return (ret);
+	}
+
+	hc = ilbd_get_hc(hc_name);
+	if (hc == NULL) {
+		logdebug("ilbd_destroy_hc: healthcheck %s does not exist",
+		    hc_name);
+		ilbd_audit_hc_event(hc_name, NULL, ILBD_DESTROY_HC,
+		    ILB_STATUS_ENOENT, ucredp);
+		return (ILB_STATUS_ENOENT);
+	}
+
+	/* If hc is in use, cannot delete it */
+	if (hc->ihc_rule_cnt > 0) {
+		logdebug("ilbd_destroy_hc: healthcheck %s is associated"
+		    " with a rule - cannot remove", hc_name);
+		ilbd_audit_hc_event(hc_name, NULL, ILBD_DESTROY_HC,
+		    ILB_STATUS_INUSE, ucredp);
+		return (ILB_STATUS_INUSE);
+	}
+
+	if ((ret = ilbd_destroy_pg(ILBD_SCF_HC, hc_name)) !=
+	    ILB_STATUS_OK) {
+		logdebug("ilbd_destroy_hc: cannot destroy healthcheck %s "
+		    "property group", hc_name);
+		ilbd_audit_hc_event(hc_name, NULL, ILBD_DESTROY_HC,
+		    ret, ucredp);
+		return (ret);
+	}
+
+	list_remove(&ilbd_hc_list, hc);
+	free(hc);
+	ilbd_audit_hc_event(hc_name, NULL, ILBD_DESTROY_HC, ret, ucredp);
+	return (ret);
+}
+
+/*
+ * Given a hc object name, return its information.  Used by libilb to
+ * get hc info.
+ */
+ilb_status_t
+ilbd_get_hc_info(const char *hc_name, uint32_t *rbuf, size_t *rbufsz)
+{
+	ilbd_hc_t	*hc;
+	ilb_hc_info_t	*hc_info;
+	ilb_comm_t	*ic = (ilb_comm_t *)rbuf;
+
+	hc = ilbd_get_hc(hc_name);
+	if (hc == NULL) {
+		logdebug("%s: healthcheck %s does not exist", __func__,
+		    hc_name);
+		return (ILB_STATUS_ENOENT);
+	}
+	ilbd_reply_ok(rbuf, rbufsz);
+	hc_info = (ilb_hc_info_t *)&ic->ic_data;
+
+	(void) strlcpy(hc_info->hci_name, hc->ihc_name, sizeof (hc->ihc_name));
+	(void) strlcpy(hc_info->hci_test, hc->ihc_test, sizeof (hc->ihc_test));
+	hc_info->hci_timeout = hc->ihc_timeout;
+	hc_info->hci_count = hc->ihc_count;
+	hc_info->hci_interval = hc->ihc_interval;
+	hc_info->hci_def_ping = hc->ihc_def_ping;
+
+	*rbufsz += sizeof (ilb_hc_info_t);
+
+	return (ILB_STATUS_OK);
+}
+
+static void
+ilbd_hc_copy_srvs(uint32_t *rbuf, size_t *rbufsz, ilbd_hc_rule_t *hc_rule,
+    const char *rulename)
+{
+	ilbd_hc_srv_t		*tmp_srv;
+	ilb_hc_srv_t		*dst_srv;
+	ilb_hc_rule_srv_t	*srvs;
+	size_t			tmp_rbufsz;
+	int			i;
+
+	tmp_rbufsz = *rbufsz;
+	/* Set up the reply buffer.  rbufsz will be set to the new size. */
+	ilbd_reply_ok(rbuf, rbufsz);
+
+	/* Calculate how much space is left for holding server info. */
+	*rbufsz += sizeof (ilb_hc_rule_srv_t);
+	tmp_rbufsz -= *rbufsz;
+
+	srvs = (ilb_hc_rule_srv_t *)&((ilb_comm_t *)rbuf)->ic_data;
+
+	tmp_srv = list_head(&hc_rule->hcr_servers);
+	for (i = 0; tmp_srv != NULL && tmp_rbufsz >= sizeof (*dst_srv); i++) {
+		dst_srv = &srvs->rs_srvs[i];
+
+		(void) strlcpy(dst_srv->hcs_rule_name, rulename, ILB_NAMESZ);
+		(void) strlcpy(dst_srv->hcs_ID, tmp_srv->shc_sg_srv->sgs_srvID,
+		    ILB_NAMESZ);
+		(void) strlcpy(dst_srv->hcs_hc_name,
+		    tmp_srv->shc_hc->ihc_name, ILB_NAMESZ);
+		dst_srv->hcs_IP = tmp_srv->shc_sg_srv->sgs_addr;
+		dst_srv->hcs_fail_cnt = tmp_srv->shc_fail_cnt;
+		dst_srv->hcs_status = tmp_srv->shc_status;
+		dst_srv->hcs_rtt = tmp_srv->shc_rtt;
+		dst_srv->hcs_lasttime = tmp_srv->shc_lasttime;
+		dst_srv->hcs_nexttime = tmp_srv->shc_nexttime;
+
+		tmp_srv = list_next(&hc_rule->hcr_servers, tmp_srv);
+		tmp_rbufsz -= sizeof (*dst_srv);
+	}
+	srvs->rs_num_srvs = i;
+	*rbufsz += i * sizeof (*dst_srv);
+}
+
+/*
+ * Given a rule name, return the hc status of its servers.
+ */
+ilb_status_t
+ilbd_get_hc_srvs(const char *rulename, uint32_t *rbuf, size_t *rbufsz)
+{
+	ilbd_hc_t	*hc;
+	ilbd_hc_rule_t	*hc_rule;
+
+	for (hc = list_head(&ilbd_hc_list); hc != NULL;
+	    hc = list_next(&ilbd_hc_list, hc)) {
+		for (hc_rule = list_head(&hc->ihc_rules); hc_rule != NULL;
+		    hc_rule = list_next(&hc->ihc_rules, hc_rule)) {
+			if (strcasecmp(hc_rule->hcr_rule->irl_name,
+			    rulename) != 0) {
+				continue;
+			}
+			ilbd_hc_copy_srvs(rbuf, rbufsz, hc_rule, rulename);
+			return (ILB_STATUS_OK);
+		}
+	}
+	return (ILB_STATUS_RULE_NO_HC);
+}
+
+/*
+ * Initialize the hc timer and associate the notification of timeout to
+ * the given event port.
+ */
+void
+ilbd_hc_timer_init(int ev_port, ilbd_timer_event_obj_t *ev_obj)
+{
+	struct sigevent sigev;
+	port_notify_t notify;
+
+	if ((ilbd_hc_timer_q = iu_tq_create()) == NULL) {
+		logerr("%s: cannot create hc timer queue", __func__);
+		exit(EXIT_FAILURE);
+	}
+	hc_timer_restarted = B_FALSE;
+
+	ev_obj->ev = ILBD_EVENT_TIMER;
+	ev_obj->timerid = -1;
+
+	notify.portnfy_port = ev_port;
+	notify.portnfy_user = ev_obj;
+	sigev.sigev_notify = SIGEV_PORT;
+	sigev.sigev_value.sival_ptr = &notify;
+	if (timer_create(CLOCK_REALTIME, &sigev, &ev_obj->timerid) == -1) {
+		logerr("%s: cannot create timer", __func__);
+		exit(EXIT_FAILURE);
+	}
+}
+
+/*
+ * HC timeout handler.
+ */
+void
+ilbd_hc_timeout(void)
+{
+	(void) iu_expire_timers(ilbd_hc_timer_q);
+	hc_timer_restarted = B_TRUE;
+}
+
+/*
+ * Set up the timer to fire at the earliest timeout.
+ */
+void
+ilbd_hc_timer_update(ilbd_timer_event_obj_t *ev_obj)
+{
+	itimerspec_t itimeout;
+	int timeout;
+
+	/*
+	 * There is no change on the timer list, so no need to set up the
+	 * timer again.
+	 */
+	if (!hc_timer_restarted)
+		return;
+
+restart:
+	if ((timeout = iu_earliest_timer(ilbd_hc_timer_q)) == INFTIM) {
+		hc_timer_restarted = B_FALSE;
+		return;
+	} else if (timeout == 0) {
+		/*
+		 * Handle the timeout immediately.  After that (clearing all
+		 * the expired timers), check to  see if there are still
+		 * timers running.  If yes, start them.
+		 */
+		(void) iu_expire_timers(ilbd_hc_timer_q);
+		goto restart;
+	}
+
+	itimeout.it_value.tv_sec = timeout / MILLISEC + 1;
+	itimeout.it_value.tv_nsec = 0;
+	itimeout.it_interval.tv_sec = 0;
+	itimeout.it_interval.tv_nsec = 0;
+
+	/*
+	 * Failure to set a timeout is "OK" since hopefully there will be
+	 * other events and timer_settime() will be called again.  So
+	 * we will only miss some timeouts.  But in the worst case, no event
+	 * will happen and ilbd will get stuck...
+	 */
+	if (timer_settime(ev_obj->timerid, 0, &itimeout, NULL) == -1)
+		logerr("%s: cannot set timer", __func__);
+	hc_timer_restarted = B_FALSE;
+}
+
+/*
+ * Kill the probe process of a server.
+ */
+static void
+ilbd_hc_kill_probe(ilbd_hc_srv_t *srv)
+{
+	/*
+	 * First dissociate the fd from the event port.  It should not
+	 * fail.
+	 */
+	if (port_dissociate(srv->shc_ev_port, PORT_SOURCE_FD,
+	    srv->shc_child_fd) != 0) {
+		logdebug("%s: port_dissociate: %s", __func__, strerror(errno));
+	}
+	(void) close(srv->shc_child_fd);
+	free(srv->shc_ev);
+	srv->shc_ev = NULL;
+
+	/* Then kill the probe process. */
+	if (kill(srv->shc_child_pid, SIGKILL) != 0) {
+		logerr("%s: rule %s server %s: %s", __func__,
+		    srv->shc_hc_rule->hcr_rule->irl_name,
+		    srv->shc_sg_srv->sgs_srvID, strerror(errno));
+	}
+	/* Should not fail... */
+	if (waitpid(srv->shc_child_pid, NULL, 0) != srv->shc_child_pid) {
+		logdebug("%s: waitpid: rule %s server %s", __func__,
+		    srv->shc_hc_rule->hcr_rule->irl_name,
+		    srv->shc_sg_srv->sgs_srvID);
+	}
+	srv->shc_child_pid = 0;
+}
+
+/*
+ * Disable the server, either because the server is dead or because a timer
+ * cannot be started for this server.  Note that this only affects the
+ * transient configuration, meaning only in memory.  The persistent
+ * configuration is not affected.
+ */
+static void
+ilbd_mark_server_disabled(ilbd_hc_srv_t *srv)
+{
+	srv->shc_status = ILB_HCS_DISABLED;
+
+	/* Disable the server in kernel. */
+	if (ilbd_k_Xable_server(&srv->shc_sg_srv->sgs_addr,
+	    srv->shc_hc_rule->hcr_rule->irl_name,
+	    stat_declare_srv_dead) != ILB_STATUS_OK) {
+		logerr("%s: cannot disable server in kernel: rule %s "
+		    "server %s", __func__,
+		    srv->shc_hc_rule->hcr_rule->irl_name,
+		    srv->shc_sg_srv->sgs_srvID);
+	}
+}
+
+/*
+ * A probe fails, set the state of the server.
+ */
+static void
+ilbd_set_fail_state(ilbd_hc_srv_t *srv)
+{
+	if (++srv->shc_fail_cnt < srv->shc_hc->ihc_count) {
+		/* Probe again */
+		ilbd_hc_probe_timer(ilbd_hc_timer_q, srv);
+		return;
+	}
+
+	logdebug("%s: rule %s server %s fails %u", __func__,
+	    srv->shc_hc_rule->hcr_rule->irl_name, srv->shc_sg_srv->sgs_srvID,
+	    srv->shc_fail_cnt);
+
+	/*
+	 * If this is a ping test, mark the server as
+	 * unreachable instead of dead.
+	 */
+	if (srv->shc_hc->ihc_test_type == ILBD_HC_PING ||
+	    srv->shc_state == ilbd_hc_def_pinging) {
+		srv->shc_status = ILB_HCS_UNREACH;
+	} else {
+		srv->shc_status = ILB_HCS_DEAD;
+	}
+
+	/* Disable the server in kernel. */
+	if (ilbd_k_Xable_server(&srv->shc_sg_srv->sgs_addr,
+	    srv->shc_hc_rule->hcr_rule->irl_name, stat_declare_srv_dead) !=
+	    ILB_STATUS_OK) {
+		logerr("%s: cannot disable server in kernel: rule %s "
+		    "server %s", __func__,
+		    srv->shc_hc_rule->hcr_rule->irl_name,
+		    srv->shc_sg_srv->sgs_srvID);
+	}
+
+	/* Still keep probing in case the server is alive again. */
+	if (ilbd_hc_restart_timer(srv->shc_hc, srv) != ILB_STATUS_OK) {
+		/* Only thing to do is to disable the server... */
+		logerr("%s: cannot restart timer: rule %s server %s", __func__,
+		    srv->shc_hc_rule->hcr_rule->irl_name,
+		    srv->shc_sg_srv->sgs_srvID);
+		srv->shc_status = ILB_HCS_DISABLED;
+	}
+}
+
+/*
+ * A probe process has not returned for the ihc_timeout period, we should
+ * kill it.  This function is the handler of this.
+ */
+/* ARGSUSED */
+static void
+ilbd_hc_kill_timer(iu_tq_t *tq, void *arg)
+{
+	ilbd_hc_srv_t *srv = (ilbd_hc_srv_t *)arg;
+
+	ilbd_hc_kill_probe(srv);
+	ilbd_set_fail_state(srv);
+}
+
+/*
+ * Probe timeout handler.  Send out the appropriate probe.
+ */
+/* ARGSUSED */
+static void
+ilbd_hc_probe_timer(iu_tq_t *tq, void *arg)
+{
+	ilbd_hc_srv_t *srv = (ilbd_hc_srv_t *)arg;
+
+	/*
+	 * If starting the probe fails, just pretend that the timeout has
+	 * extended.
+	 */
+	if (!ilbd_run_probe(srv)) {
+		/*
+		 * If we cannot restart the timer, the only thing we can do
+		 * is to disable this server.  Hopefully the sys admin will
+		 * notice this and enable this server again later.
+		 */
+		if (ilbd_hc_restart_timer(srv->shc_hc, srv) != ILB_STATUS_OK) {
+			logerr("%s: cannot restart timer: rule %s server %s, "
+			    "disabling it", __func__,
+			    srv->shc_hc_rule->hcr_rule->irl_name,
+			    srv->shc_sg_srv->sgs_srvID);
+			ilbd_mark_server_disabled(srv);
+		}
+		return;
+	}
+
+	/*
+	 * Similar to above, if kill timer cannot be started, disable the
+	 * server.
+	 */
+	if ((srv->shc_tid = iu_schedule_timer(ilbd_hc_timer_q,
+	    srv->shc_hc->ihc_timeout, ilbd_hc_kill_timer, srv)) == -1) {
+		logerr("%s: cannot start kill timer: rule %s server %s, "
+		    "disabling it", __func__,
+		    srv->shc_hc_rule->hcr_rule->irl_name,
+		    srv->shc_sg_srv->sgs_srvID);
+		ilbd_mark_server_disabled(srv);
+	}
+	hc_timer_restarted = B_TRUE;
+}
+
+/* Restart the periodic timer for a given server. */
+static ilb_status_t
+ilbd_hc_restart_timer(ilbd_hc_t *hc, ilbd_hc_srv_t *srv)
+{
+	int timeout;
+
+	/* Don't allow the timeout interval to be less than 1s */
+	timeout = MAX((hc->ihc_interval >> 1) + (gethrtime() %
+	    (hc->ihc_interval + 1)), 1);
+
+	/*
+	 * If the probe is actually a ping probe, there is no need to
+	 * do default pinging.  Just skip the step.
+	 */
+	if (hc->ihc_def_ping && hc->ihc_test_type != ILBD_HC_PING)
+		srv->shc_state = ilbd_hc_def_pinging;
+	else
+		srv->shc_state = ilbd_hc_probing;
+	srv->shc_tid = iu_schedule_timer(ilbd_hc_timer_q, timeout,
+	    ilbd_hc_probe_timer, srv);
+
+	if (srv->shc_tid == -1)
+		return (ILB_STATUS_TIMER);
+	srv->shc_lasttime = time(NULL);
+	srv->shc_nexttime = time(NULL) + timeout;
+
+	hc_timer_restarted = B_TRUE;
+	return (ILB_STATUS_OK);
+}
+
+/* Helper routine to associate a server with its hc object. */
+static ilb_status_t
+ilbd_hc_srv_add(ilbd_hc_t *hc, ilbd_hc_rule_t *hc_rule,
+    const ilb_sg_srv_t *srv, int ev_port)
+{
+	ilbd_hc_srv_t *new_srv;
+	ilb_status_t ret;
+
+	if ((new_srv = calloc(1, sizeof (ilbd_hc_srv_t))) == NULL)
+		return (ILB_STATUS_ENOMEM);
+	new_srv->shc_hc = hc;
+	new_srv->shc_hc_rule = hc_rule;
+	new_srv->shc_sg_srv = srv;
+	new_srv->shc_ev_port = ev_port;
+	new_srv->shc_tid = -1;
+	new_srv->shc_nexttime = time(NULL);
+	new_srv->shc_lasttime = new_srv->shc_nexttime;
+
+	if ((hc_rule->hcr_rule->irl_flags & ILB_FLAGS_RULE_ENABLED) &&
+	    ILB_IS_SRV_ENABLED(srv->sgs_flags)) {
+		new_srv->shc_status = ILB_HCS_UNINIT;
+		ret = ilbd_hc_restart_timer(hc, new_srv);
+		if (ret != ILB_STATUS_OK) {
+			free(new_srv);
+			return (ret);
+		}
+	} else {
+		new_srv->shc_status = ILB_HCS_DISABLED;
+	}
+
+	list_insert_tail(&hc_rule->hcr_servers, new_srv);
+	return (ILB_STATUS_OK);
+}
+
+/* Handy macro to cancel a server's timer. */
+#define	HC_CANCEL_TIMER(srv)						\
+{									\
+	void *arg;							\
+	int ret;							\
+	if ((srv)->shc_tid != -1) {					\
+		ret = iu_cancel_timer(ilbd_hc_timer_q, (srv)->shc_tid, &arg); \
+		(srv)->shc_tid = -1;					\
+		assert(ret == 1);					\
+		assert(arg == (srv));					\
+	}								\
+	hc_timer_restarted = B_TRUE;					\
+}
+
+/* Helper routine to dissociate a server from its hc object. */
+static ilb_status_t
+ilbd_hc_srv_rem(ilbd_hc_rule_t *hc_rule, const ilb_sg_srv_t *srv)
+{
+	ilbd_hc_srv_t *tmp_srv;
+
+	for (tmp_srv = list_head(&hc_rule->hcr_servers); tmp_srv != NULL;
+	    tmp_srv = list_next(&hc_rule->hcr_servers, tmp_srv)) {
+		if (tmp_srv->shc_sg_srv == srv) {
+			list_remove(&hc_rule->hcr_servers, tmp_srv);
+			HC_CANCEL_TIMER(tmp_srv);
+			if (tmp_srv->shc_child_pid != 0)
+				ilbd_hc_kill_probe(tmp_srv);
+			free(tmp_srv);
+			return (ILB_STATUS_OK);
+		}
+	}
+	return (ILB_STATUS_ENOENT);
+}
+
+/* Helper routine to dissociate all servers of a rule from its hc object. */
+static void
+ilbd_hc_srv_rem_all(ilbd_hc_rule_t *hc_rule)
+{
+	ilbd_hc_srv_t *srv;
+
+	while ((srv = list_remove_head(&hc_rule->hcr_servers)) != NULL) {
+		HC_CANCEL_TIMER(srv);
+		if (srv->shc_child_pid != 0)
+			ilbd_hc_kill_probe(srv);
+		free(srv);
+	}
+}
+
+/* Associate a rule with its hc object. */
+ilb_status_t
+ilbd_hc_associate_rule(const ilbd_rule_t *rule, int ev_port)
+{
+	ilbd_hc_t	*hc;
+	ilbd_hc_rule_t	*hc_rule;
+	ilb_status_t	ret;
+	ilbd_sg_t	*sg;
+	ilbd_srv_t	*ilbd_srv;
+
+	/* The rule is assumed to be initialized appropriately. */
+	if ((hc = ilbd_get_hc(rule->irl_hcname)) == NULL) {
+		logdebug("ilbd_hc_associate_rule: healthcheck %s does not "
+		    "exist", rule->irl_hcname);
+		return (ILB_STATUS_ENOHCINFO);
+	}
+	if ((hc->ihc_test_type == ILBD_HC_TCP &&
+	    rule->irl_proto != IPPROTO_TCP) ||
+	    (hc->ihc_test_type == ILBD_HC_UDP &&
+	    rule->irl_proto != IPPROTO_UDP)) {
+		return (ILB_STATUS_RULE_HC_MISMATCH);
+	}
+	if ((hc_rule = calloc(1, sizeof (ilbd_hc_rule_t))) == NULL) {
+		logdebug("ilbd_hc_associate_rule: out of memory");
+		return (ILB_STATUS_ENOMEM);
+	}
+
+	hc_rule->hcr_rule = rule;
+	list_create(&hc_rule->hcr_servers, sizeof (ilbd_hc_srv_t),
+	    offsetof(ilbd_hc_srv_t, shc_srv_link));
+
+	/* Add all the servers. */
+	sg = rule->irl_sg;
+	for (ilbd_srv = list_head(&sg->isg_srvlist); ilbd_srv != NULL;
+	    ilbd_srv = list_next(&sg->isg_srvlist, ilbd_srv)) {
+		if ((ret = ilbd_hc_srv_add(hc, hc_rule, &ilbd_srv->isv_srv,
+		    ev_port)) != ILB_STATUS_OK) {
+			/* Remove all previously added servers */
+			ilbd_hc_srv_rem_all(hc_rule);
+			free(hc_rule);
+			return (ret);
+		}
+	}
+	list_insert_tail(&hc->ihc_rules, hc_rule);
+	hc->ihc_rule_cnt++;
+
+	return (ILB_STATUS_OK);
+}
+
+/* Dissociate a rule from its hc object. */
+ilb_status_t
+ilbd_hc_dissociate_rule(const ilbd_rule_t *rule)
+{
+	ilbd_hc_t	*hc;
+	ilbd_hc_rule_t	*hc_rule;
+
+	/* The rule is assumed to be initialized appropriately. */
+	if ((hc = ilbd_get_hc(rule->irl_hcname)) == NULL) {
+		logdebug("ilbd_hc_dissociate_rule: healthcheck %s does not "
+		    "exist", rule->irl_hcname);
+		return (ILB_STATUS_ENOENT);
+	}
+	for (hc_rule = list_head(&hc->ihc_rules); hc_rule != NULL;
+	    hc_rule = list_next(&hc->ihc_rules, hc_rule)) {
+		if (hc_rule->hcr_rule == rule)
+			break;
+	}
+	if (hc_rule == NULL) {
+		logdebug("ilbd_hc_dissociate_rule: rule %s is not associated "
+		    "with healtcheck %s", rule->irl_hcname, hc->ihc_name);
+		return (ILB_STATUS_ENOENT);
+	}
+	ilbd_hc_srv_rem_all(hc_rule);
+	list_remove(&hc->ihc_rules, hc_rule);
+	hc->ihc_rule_cnt--;
+	return (ILB_STATUS_OK);
+}
+
+/*
+ * Given a hc object name and a rule, check to see if the rule is associated
+ * with the hc object.  If it is, the hc object is returned in **hc and the
+ * ilbd_hc_rule_t is returned in **hc_rule.
+ */
+static boolean_t
+ilbd_hc_check_rule(const char *hc_name, const ilbd_rule_t *rule,
+    ilbd_hc_t **hc, ilbd_hc_rule_t **hc_rule)
+{
+	ilbd_hc_t	*tmp_hc;
+	ilbd_hc_rule_t	*tmp_hc_rule;
+
+	if ((tmp_hc = ilbd_get_hc(hc_name)) == NULL)
+		return (B_FALSE);
+	for (tmp_hc_rule = list_head(&tmp_hc->ihc_rules); tmp_hc_rule != NULL;
+	    tmp_hc_rule = list_next(&tmp_hc->ihc_rules, tmp_hc_rule)) {
+		if (tmp_hc_rule->hcr_rule == rule) {
+			*hc = tmp_hc;
+			*hc_rule = tmp_hc_rule;
+			return (B_TRUE);
+		}
+	}
+	return (B_FALSE);
+}
+
+/* Associate a server with its hc object. */
+ilb_status_t
+ilbd_hc_add_server(const ilbd_rule_t *rule, const ilb_sg_srv_t *srv,
+    int ev_port)
+{
+	ilbd_hc_t	*hc;
+	ilbd_hc_rule_t	*hc_rule;
+
+	if (!ilbd_hc_check_rule(rule->irl_hcname, rule, &hc, &hc_rule))
+		return (ILB_STATUS_ENOENT);
+	return (ilbd_hc_srv_add(hc, hc_rule, srv, ev_port));
+}
+
+/* Dissociate a server from its hc object. */
+ilb_status_t
+ilbd_hc_del_server(const ilbd_rule_t *rule, const ilb_sg_srv_t *srv)
+{
+	ilbd_hc_t	*hc;
+	ilbd_hc_rule_t	*hc_rule;
+
+	if (!ilbd_hc_check_rule(rule->irl_hcname, rule, &hc, &hc_rule))
+		return (ILB_STATUS_ENOENT);
+	return (ilbd_hc_srv_rem(hc_rule, srv));
+}
+
+/* Helper routine to enable/disable a server's hc probe. */
+static ilb_status_t
+ilbd_hc_toggle_server(const ilbd_rule_t *rule, const ilb_sg_srv_t *srv,
+    boolean_t enable)
+{
+	ilbd_hc_t	*hc;
+	ilbd_hc_rule_t	*hc_rule;
+	ilbd_hc_srv_t	*tmp_srv;
+	ilb_status_t	ret;
+
+	if (!ilbd_hc_check_rule(rule->irl_hcname, rule, &hc, &hc_rule))
+		return (ILB_STATUS_ENOENT);
+	for (tmp_srv = list_head(&hc_rule->hcr_servers); tmp_srv != NULL;
+	    tmp_srv = list_next(&hc_rule->hcr_servers, tmp_srv)) {
+		if (tmp_srv->shc_sg_srv != srv) {
+			continue;
+		}
+		if (enable) {
+			if (tmp_srv->shc_status == ILB_HCS_DISABLED) {
+				ret = ilbd_hc_restart_timer(hc, tmp_srv);
+				if (ret != ILB_STATUS_OK) {
+					logerr("%s: cannot start timers for "
+					    "rule %s server %s", __func__,
+					    rule->irl_name,
+					    tmp_srv->shc_sg_srv->sgs_srvID);
+					return (ret);
+				}
+				/* Start from fresh... */
+				tmp_srv->shc_status = ILB_HCS_UNINIT;
+				tmp_srv->shc_rtt = 0;
+				tmp_srv->shc_fail_cnt = 0;
+			}
+		} else {
+			if (tmp_srv->shc_status != ILB_HCS_DISABLED) {
+				tmp_srv->shc_status = ILB_HCS_DISABLED;
+				HC_CANCEL_TIMER(tmp_srv);
+				if (tmp_srv->shc_child_pid != 0)
+					ilbd_hc_kill_probe(tmp_srv);
+			}
+		}
+		return (ILB_STATUS_OK);
+	}
+	return (ILB_STATUS_ENOENT);
+}
+
+ilb_status_t
+ilbd_hc_enable_server(const ilbd_rule_t *rule, const ilb_sg_srv_t *srv)
+{
+	return (ilbd_hc_toggle_server(rule, srv, B_TRUE));
+}
+
+ilb_status_t
+ilbd_hc_disable_server(const ilbd_rule_t *rule, const ilb_sg_srv_t *srv)
+{
+	return (ilbd_hc_toggle_server(rule, srv, B_FALSE));
+}
+
+/*
+ * Helper routine to enable/disable a rule's hc probe (including all its
+ * servers).
+ */
+static ilb_status_t
+ilbd_hc_toggle_rule(const ilbd_rule_t *rule, boolean_t enable)
+{
+	ilbd_hc_t	*hc;
+	ilbd_hc_rule_t	*hc_rule;
+	ilbd_hc_srv_t	*tmp_srv;
+	int		ret;
+
+	if (!ilbd_hc_check_rule(rule->irl_hcname, rule, &hc, &hc_rule))
+		return (ILB_STATUS_ENOENT);
+
+	for (tmp_srv = list_head(&hc_rule->hcr_servers); tmp_srv != NULL;
+	    tmp_srv = list_next(&hc_rule->hcr_servers, tmp_srv)) {
+		if (enable) {
+			/*
+			 * If the server is disabled in the rule, do not
+			 * restart its timer.
+			 */
+			if (tmp_srv->shc_status == ILB_HCS_DISABLED &&
+			    ILB_IS_SRV_ENABLED(
+			    tmp_srv->shc_sg_srv->sgs_flags)) {
+				ret = ilbd_hc_restart_timer(hc, tmp_srv);
+				if (ret != ILB_STATUS_OK) {
+					logerr("%s: cannot start timers for "
+					    "rule %s server %s", __func__,
+					    rule->irl_name,
+					    tmp_srv->shc_sg_srv->sgs_srvID);
+					goto rollback;
+				} else {
+					/* Start from fresh... */
+					tmp_srv->shc_status = ILB_HCS_UNINIT;
+					tmp_srv->shc_rtt = 0;
+					tmp_srv->shc_fail_cnt = 0;
+				}
+			}
+		} else {
+			if (tmp_srv->shc_status != ILB_HCS_DISABLED) {
+				HC_CANCEL_TIMER(tmp_srv);
+				tmp_srv->shc_status = ILB_HCS_DISABLED;
+				if (tmp_srv->shc_child_pid != 0)
+					ilbd_hc_kill_probe(tmp_srv);
+			}
+		}
+	}
+	return (ILB_STATUS_OK);
+rollback:
+	enable = !enable;
+	for (tmp_srv = list_prev(&hc_rule->hcr_servers, tmp_srv);
+	    tmp_srv != NULL;
+	    tmp_srv = list_prev(&hc_rule->hcr_servers, tmp_srv)) {
+		if (enable) {
+			if (tmp_srv->shc_status == ILB_HCS_DISABLED &&
+			    ILB_IS_SRV_ENABLED(
+			    tmp_srv->shc_sg_srv->sgs_flags)) {
+				(void) ilbd_hc_restart_timer(hc, tmp_srv);
+				tmp_srv->shc_status = ILB_HCS_UNINIT;
+				tmp_srv->shc_rtt = 0;
+				tmp_srv->shc_fail_cnt = 0;
+			}
+		} else {
+			if (tmp_srv->shc_status != ILB_HCS_DISABLED) {
+				HC_CANCEL_TIMER(tmp_srv);
+				tmp_srv->shc_status = ILB_HCS_DISABLED;
+				if (tmp_srv->shc_child_pid != 0)
+					ilbd_hc_kill_probe(tmp_srv);
+			}
+		}
+	}
+	return (ret);
+}
+
+ilb_status_t
+ilbd_hc_enable_rule(const ilbd_rule_t *rule)
+{
+	return (ilbd_hc_toggle_rule(rule, B_TRUE));
+}
+
+ilb_status_t
+ilbd_hc_disable_rule(const ilbd_rule_t *rule)
+{
+	return (ilbd_hc_toggle_rule(rule, B_FALSE));
+}
+
+static const char *
+topo_2_str(ilb_topo_t topo)
+{
+	switch (topo) {
+	case ILB_TOPO_DSR:
+		return ("DSR");
+		break;
+	case ILB_TOPO_NAT:
+		return ("NAT");
+		break;
+	case ILB_TOPO_HALF_NAT:
+		return ("HALF_NAT");
+		break;
+	default:
+		/* Should not happen. */
+		logerr("%s: unknown topology", __func__);
+		break;
+	}
+	return ("");
+}
+
+/*
+ * Create the argument list to be passed to a hc probe command.
+ * The passed in argv is assumed to have HC_PROBE_ARGC elements.
+ */
+static boolean_t
+create_argv(ilbd_hc_srv_t *srv, char *argv[])
+{
+	char buf[INET6_ADDRSTRLEN];
+	ilbd_rule_t const *rule;
+	ilb_sg_srv_t const *sg_srv;
+	struct in_addr v4_addr;
+	in_port_t port;
+	int i;
+
+	rule = srv->shc_hc_rule->hcr_rule;
+	sg_srv = srv->shc_sg_srv;
+
+	if (srv->shc_state == ilbd_hc_def_pinging) {
+		if ((argv[0] = strdup(ILB_PROBE_PING)) == NULL)
+			return (B_FALSE);
+	} else {
+		switch (srv->shc_hc->ihc_test_type) {
+		case ILBD_HC_USER:
+			if ((argv[0] = strdup(srv->shc_hc->ihc_test)) == NULL)
+				return (B_FALSE);
+			break;
+		case ILBD_HC_TCP:
+		case ILBD_HC_UDP:
+			if ((argv[0] = strdup(ILB_PROBE_PROTO)) ==
+			    NULL) {
+				return (B_FALSE);
+			}
+			break;
+		case ILBD_HC_PING:
+			if ((argv[0] = strdup(ILB_PROBE_PING)) == NULL) {
+				return (B_FALSE);
+			}
+			break;
+		}
+	}
+
+	/*
+	 * argv[1] is the VIP.
+	 *
+	 * Right now, the VIP and the backend server addresses should be
+	 * in the same IP address family.  Here we don't do that in case
+	 * this assumption is changed in future.
+	 */
+	if (IN6_IS_ADDR_V4MAPPED(&rule->irl_vip)) {
+		IN6_V4MAPPED_TO_INADDR(&rule->irl_vip, &v4_addr);
+		if (inet_ntop(AF_INET, &v4_addr, buf, sizeof (buf)) == NULL)
+			goto cleanup;
+	} else {
+		if (inet_ntop(AF_INET6, &rule->irl_vip, buf,
+		    sizeof (buf)) == NULL) {
+			goto cleanup;
+		}
+	}
+	if ((argv[1] = strdup(buf)) == NULL)
+		goto cleanup;
+
+	/*
+	 * argv[2] is the backend server address.
+	 */
+	if (IN6_IS_ADDR_V4MAPPED(&sg_srv->sgs_addr)) {
+		IN6_V4MAPPED_TO_INADDR(&sg_srv->sgs_addr, &v4_addr);
+		if (inet_ntop(AF_INET, &v4_addr, buf, sizeof (buf)) == NULL)
+			goto cleanup;
+	} else {
+		if (inet_ntop(AF_INET6, &sg_srv->sgs_addr, buf,
+		    sizeof (buf)) == NULL) {
+			goto cleanup;
+		}
+	}
+	if ((argv[2] = strdup(buf)) == NULL)
+		goto cleanup;
+
+	/*
+	 * argv[3] is the transport protocol used in the rule.
+	 */
+	switch (rule->irl_proto) {
+	case IPPROTO_TCP:
+		argv[3] = strdup("TCP");
+		break;
+	case IPPROTO_UDP:
+		argv[3] = strdup("UDP");
+		break;
+	default:
+		logerr("%s: unknown protocol", __func__);
+		goto cleanup;
+		break;
+	}
+	if (argv[3] == NULL)
+		goto cleanup;
+
+	/*
+	 * argv[4] is the load balance mode, DSR, NAT, HALF-NAT.
+	 */
+	if ((argv[4] = strdup(topo_2_str(rule->irl_topo))) == NULL)
+		goto cleanup;
+
+	/*
+	 * argv[5] is the port range.  Right now, there should only be 1 port.
+	 */
+	switch (rule->irl_hcpflag) {
+	case ILB_HCI_PROBE_FIX:
+		port = ntohs(rule->irl_hcport);
+		break;
+	case ILB_HCI_PROBE_ANY: {
+		in_port_t min, max;
+
+		if (ntohs(sg_srv->sgs_minport) == 0) {
+			min = ntohs(rule->irl_minport);
+			max = ntohs(rule->irl_maxport);
+		} else {
+			min = ntohs(sg_srv->sgs_minport);
+			max = ntohs(sg_srv->sgs_maxport);
+		}
+		if (max > min)
+			port = min + gethrtime() % (max - min + 1);
+		else
+			port = min;
+		break;
+	}
+	default:
+		logerr("%s: unknown HC flag", __func__);
+		goto cleanup;
+		break;
+	}
+	(void) sprintf(buf, "%d", port);
+	if ((argv[5] = strdup(buf)) == NULL)
+		goto cleanup;
+
+	/*
+	 * argv[6] is the probe timeout.
+	 */
+	(void) sprintf(buf, "%d", srv->shc_hc->ihc_timeout);
+	if ((argv[6] = strdup(buf)) == NULL)
+		goto cleanup;
+
+	argv[7] = NULL;
+	return (B_TRUE);
+
+cleanup:
+	for (i = 0; i < HC_PROBE_ARGC; i++) {
+		if (argv[i] != NULL)
+			free(argv[i]);
+	}
+	return (B_FALSE);
+}
+
+static void
+destroy_argv(char *argv[])
+{
+	int i;
+
+	for (i = 0; argv[i] != NULL; i++)
+		free(argv[i]);
+}
+
+/* Spawn a process to run the hc probe on the given server. */
+static boolean_t
+ilbd_run_probe(ilbd_hc_srv_t *srv)
+{
+	posix_spawn_file_actions_t	fd_actions;
+	posix_spawnattr_t		attr;
+	sigset_t			child_sigset;
+	int				fds[2];
+	int				fdflags;
+	pid_t				pid;
+	char				*child_argv[HC_PROBE_ARGC];
+	ilbd_hc_probe_event_t		*probe_ev;
+	char				*probe_name;
+
+	bzero(child_argv, HC_PROBE_ARGC * sizeof (char *));
+	if ((probe_ev = calloc(1, sizeof (*probe_ev))) == NULL) {
+		logdebug("ilbd_run_probe: calloc");
+		return (B_FALSE);
+	}
+
+	/* Set up a pipe to get output from probe command. */
+	if (pipe(fds) < 0) {
+		logdebug("ilbd_run_probe: cannot create pipe");
+		free(probe_ev);
+		return (B_FALSE);
+	}
+	/* Set our side of the pipe to be non-blocking */
+	if ((fdflags = fcntl(fds[0], F_GETFL, 0)) == -1) {
+		logdebug("ilbd_run_probe: fcntl(F_GETFL)");
+		goto cleanup;
+	}
+	if (fcntl(fds[0], F_SETFL, fdflags | O_NONBLOCK) == -1) {
+		logdebug("ilbd_run_probe: fcntl(F_SETFL)");
+		goto cleanup;
+	}
+
+	if (posix_spawn_file_actions_init(&fd_actions) != 0) {
+		logdebug("ilbd_run_probe: posix_spawn_file_actions_init");
+		goto cleanup;
+	}
+	if (posix_spawnattr_init(&attr) != 0) {
+		logdebug("ilbd_run_probe: posix_spawnattr_init");
+		goto cleanup;
+	}
+	if (posix_spawn_file_actions_addclose(&fd_actions, fds[0]) != 0) {
+		logdebug("ilbd_run_probe: posix_spawn_file_actions_addclose");
+		goto cleanup;
+	}
+	if (posix_spawn_file_actions_adddup2(&fd_actions, fds[1],
+	    STDOUT_FILENO) != 0) {
+		logdebug("ilbd_run_probe: posix_spawn_file_actions_dup2");
+		goto cleanup;
+	}
+	if (posix_spawn_file_actions_addclose(&fd_actions, fds[1]) != 0) {
+		logdebug("ilbd_run_probe: posix_spawn_file_actions_addclose");
+		goto cleanup;
+	}
+
+	/* Reset all signal handling of the child to default. */
+	(void) sigfillset(&child_sigset);
+	if (posix_spawnattr_setsigdefault(&attr, &child_sigset) != 0) {
+		logdebug("ilbd_run_probe: posix_spawnattr_setsigdefault");
+		goto cleanup;
+	}
+	/* Don't want SIGCHLD. */
+	if (posix_spawnattr_setflags(&attr, POSIX_SPAWN_NOSIGCHLD_NP|
+	    POSIX_SPAWN_SETSIGDEF) != 0) {
+		logdebug("ilbd_run_probe: posix_spawnattr_setflags");
+		goto cleanup;
+	}
+
+	if (!create_argv(srv, child_argv)) {
+		logdebug("ilbd_run_probe: create_argv");
+		goto cleanup;
+	}
+
+	/*
+	 * If we are doing default pinging or not using a user supplied
+	 * probe, we should execute our standard supplied probe.  The
+	 * supplied probe command handles all types of probes.  And the
+	 * type used depends on argv[0], as filled in by create_argv().
+	 */
+	if (srv->shc_state == ilbd_hc_def_pinging ||
+	    srv->shc_hc->ihc_test_type != ILBD_HC_USER) {
+		probe_name = ILB_PROBE_PROTO;
+	} else {
+		probe_name = srv->shc_hc->ihc_test;
+	}
+	if (posix_spawn(&pid, probe_name, &fd_actions, &attr, child_argv,
+	    NULL) != 0) {
+		logerr("%s: posix_spawn: %s for server %s: %s", __func__,
+		    srv->shc_hc->ihc_test, srv->shc_sg_srv->sgs_srvID,
+		    strerror(errno));
+		goto cleanup;
+	}
+
+	(void) close(fds[1]);
+	destroy_argv(child_argv);
+	srv->shc_child_pid = pid;
+	srv->shc_child_fd = fds[0];
+	srv->shc_ev = probe_ev;
+
+	probe_ev->ihp_ev = ILBD_EVENT_PROBE;
+	probe_ev->ihp_srv = srv;
+	probe_ev->ihp_pid = pid;
+	if (port_associate(srv->shc_ev_port, PORT_SOURCE_FD, fds[0],
+	    POLLRDNORM, probe_ev) != 0) {
+		/*
+		 * Need to kill the child.  It will free the srv->shc_ev,
+		 * which is probe_ev.  So set probe_ev to NULL.
+		 */
+		ilbd_hc_kill_probe(srv);
+		probe_ev = NULL;
+		goto cleanup;
+	}
+
+	return (B_TRUE);
+
+cleanup:
+	(void) close(fds[0]);
+	(void) close(fds[1]);
+	destroy_argv(child_argv);
+	if (probe_ev != NULL)
+		free(probe_ev);
+	return (B_FALSE);
+}
+
+/*
+ * Called by ild_hc_probe_return() to re-associate the fd to a child to
+ * the event port.
+ */
+static void
+reassociate_port(int ev_port, int fd, ilbd_hc_probe_event_t *ev)
+{
+	if (port_associate(ev_port, PORT_SOURCE_FD, fd,
+	    POLLRDNORM, ev) != 0) {
+		/*
+		 * If we cannot reassociate with the port, the only
+		 * thing we can do now is to kill the child and
+		 * do a blocking wait here...
+		 */
+		logdebug("%s: port_associate: %s", __func__, strerror(errno));
+		if (kill(ev->ihp_pid, SIGKILL) != 0)
+			logerr("%s: kill: %s", __func__, strerror(errno));
+		if (waitpid(ev->ihp_pid, NULL, 0) != ev->ihp_pid)
+			logdebug("%s: waitpid: %s", __func__, strerror(errno));
+		free(ev);
+	}
+}
+
+/*
+ * To handle a child probe process hanging up.
+ */
+static void
+ilbd_hc_child_hup(int ev_port, int fd, ilbd_hc_probe_event_t *ev)
+{
+	ilbd_hc_srv_t *srv;
+	pid_t ret_pid;
+	int ret;
+
+	srv = ev->ihp_srv;
+
+	if (!ev->ihp_done) {
+		/* ilbd does not care about this process anymore ... */
+		ev->ihp_done = B_TRUE;
+		srv->shc_ev = NULL;
+		srv->shc_child_pid = 0;
+		HC_CANCEL_TIMER(srv);
+		ilbd_set_fail_state(srv);
+	}
+	ret_pid = waitpid(ev->ihp_pid, &ret, WNOHANG);
+	switch (ret_pid) {
+	case -1:
+		logperror("ilbd_hc_child_hup: waitpid");
+		/* FALLTHROUGH */
+	case 0:
+		/* The child has not completed the exit. Wait again. */
+		reassociate_port(ev_port, fd, ev);
+		break;
+	default:
+		/* Right now, we just ignore the exit status. */
+		if (WIFEXITED(ret))
+			ret = WEXITSTATUS(ret);
+		(void) close(fd);
+		free(ev);
+	}
+}
+
+/*
+ * To read the output of a child probe process.
+ */
+static void
+ilbd_hc_child_data(int fd, ilbd_hc_probe_event_t *ev)
+{
+	ilbd_hc_srv_t *srv;
+	char buf[HC_MAX_PROBE_OUTPUT];
+	int ret;
+	int64_t rtt;
+
+	srv = ev->ihp_srv;
+
+	bzero(buf, HC_MAX_PROBE_OUTPUT);
+	ret = read(fd, buf, HC_MAX_PROBE_OUTPUT - 1);
+	/* Should not happen since event port should have caught this. */
+	assert(ret > 0);
+
+	/*
+	 * We expect the probe command to print out the RTT only.  But
+	 * the command may misbehave and print out more than what we intend to
+	 * read in.  So need to do this check below to "flush" out all the
+	 * output from the command.
+	 */
+	if (!ev->ihp_done) {
+		ev->ihp_done = B_TRUE;
+		/* We don't need to know about this event anymore. */
+		srv->shc_ev = NULL;
+		srv->shc_child_pid = 0;
+		HC_CANCEL_TIMER(srv);
+	} else {
+		return;
+	}
+
+	rtt = strtoll(buf, NULL, 10);
+
+	/*
+	 * -1 means the server is dead or the probe somehow fails.  Treat
+	 * them both as server is dead.
+	 */
+	if (rtt == -1) {
+		ilbd_set_fail_state(srv);
+		return;
+	} else if (rtt > 0) {
+		/* If the returned RTT value is not valid, just ignore it. */
+		if (rtt > 0 && rtt <= UINT_MAX) {
+			/* Set rtt to be the simple smoothed average. */
+			if (srv->shc_rtt == 0) {
+				srv->shc_rtt = rtt;
+			} else {
+				srv->shc_rtt = 3 * ((srv)->shc_rtt >> 2) +
+				    (rtt >> 2);
+			}
+		}
+
+	}
+
+	switch (srv->shc_state) {
+	case ilbd_hc_def_pinging:
+		srv->shc_state = ilbd_hc_probing;
+
+		/* Ping is OK, now start the probe. */
+		ilbd_hc_probe_timer(ilbd_hc_timer_q, srv);
+		break;
+	case ilbd_hc_probing:
+		srv->shc_fail_cnt = 0;
+
+		/* Server is dead before, re-enable it. */
+		if (srv->shc_status == ILB_HCS_UNREACH ||
+		    srv->shc_status == ILB_HCS_DEAD) {
+			/*
+			 * If enabling the server in kernel fails now,
+			 * hopefully when the timer fires again later, the
+			 * enabling can be done.
+			 */
+			if (ilbd_k_Xable_server(&srv->shc_sg_srv->sgs_addr,
+			    srv->shc_hc_rule->hcr_rule->irl_name,
+			    stat_declare_srv_alive) != ILB_STATUS_OK) {
+				logerr("%s: cannot enable server in kernel: "
+				    " rule %s server %s", __func__,
+				    srv->shc_hc_rule->hcr_rule->irl_name,
+				    srv->shc_sg_srv->sgs_srvID);
+			} else {
+				srv->shc_status = ILB_HCS_ALIVE;
+			}
+		} else {
+			srv->shc_status = ILB_HCS_ALIVE;
+		}
+		if (ilbd_hc_restart_timer(srv->shc_hc, srv) != ILB_STATUS_OK) {
+			logerr("%s: cannot restart timer: rule %s server %s",
+			    __func__, srv->shc_hc_rule->hcr_rule->irl_name,
+			    srv->shc_sg_srv->sgs_srvID);
+			ilbd_mark_server_disabled(srv);
+		}
+		break;
+	default:
+		logdebug("%s: unknown state", __func__);
+		break;
+	}
+}
+
+/*
+ * Handle the return event of a child probe fd.
+ */
+void
+ilbd_hc_probe_return(int ev_port, int fd, int port_events,
+    ilbd_hc_probe_event_t *ev)
+{
+	/*
+	 * Note that there can be more than one events delivered to us at
+	 * the same time.  So we need to check them individually.
+	 */
+	if (port_events & POLLRDNORM)
+		ilbd_hc_child_data(fd, ev);
+
+	if (port_events & (POLLHUP|POLLERR)) {
+		ilbd_hc_child_hup(ev_port, fd, ev);
+		return;
+	}
+
+	/*
+	 * Re-associate the fd with the port so that when the child
+	 * exits, we can reap the status.
+	 */
+	reassociate_port(ev_port, fd, ev);
+}

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/usr/src/cmd/cmd-inet/usr.lib/ilbd/ilbd_main.c	Tue Nov 03 23:15:19 2009 -0800
@@ -0,0 +1,1025 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+/*
+ * The core of ilbd daemon is a single-threaded event loop using
+ * event completion framework; it receives requests from client using
+ * the libilb functions, handles timeouts, initiates health checks, and
+ * populates the kernel state.
+ *
+ * The daemon has the following privileges (in addition to the basic ones):
+ *
+ * 	PRIV_PROC_OWNER, PRIV_NET_ICMPACCESS,
+ *	PRIV_SYS_IP_CONFIG, PRIV_PROC_AUDIT
+ *
+ * The aforementioned  privileges will be specified in the SMF manifest.
+ *
+ * AF_UNIX socket is used for IPC between libilb and this daemon as
+ * both processes will run on the same machine.
+ *
+ * To do health check, the daemon will create a timer for every health
+ * check probe. Each of these timers will be  associated with the
+ * event port. When a timer goes off, the daemon will initiate a
+ * pipe to a separate process to execute the specific health check
+ * probe. This new process will run with the same user-id as that of
+ * ilbd daemon and will inherit all the privileges from the ilbd
+ * daemon parent process except the following:
+ *
+ * PRIV_PROC_OWNER, PRIV_PROC_AUDIT
+ *
+ * All health checks, will be implemented as external methods
+ * (binary or script). The following arguments will be passed
+ * to external methods:
+ *
+ *	$1	VIP (literal IPv4 or IPv6 address)
+ *	$2	Server IP (literal IPv4 or IPv6 address)
+ *	$3	Protocol (UDP, TCP as a string)
+ *	$4	The load balance mode, "DSR", "NAT", "HALF_NAT"
+ *	$5	Numeric port range
+ *	$6	maximum time (in seconds) the method
+ * should wait before returning failure. If the method runs for
+ * longer, it may be killed, and the test considered failed.
+ *
+ * Upon success, a health check method should print the RTT to the
+ * it finds to its STDOUT for ilbd to consume.  The implicit unit
+ * is microseconds but only the number needs to be printed.  If it
+ * cannot find the RTT, it should print 0.  If the method decides
+ * that the server is dead, it should print -1 to its STDOUT.
+ *
+ * By default, an user-supplied health check probe process will
+ * also run with the same set of privileges as ILB's built-in
+ * probes.  If the administrator has an user-supplied health check
+ * program that requires a larger privilege set, he/she will have
+ * to implement setuid program.
+ *
+ * Each health check will have a timeout, such that if the health
+ * check process is hung, it will be killed after the timeout interval
+ * and the daemon will notify the kernel ILB engine of the server's
+ * unresponsiveness, so that load distribution can be appropriately
+ * adjusted.  If on the other hand the health check is successful
+ * the timeout timer is cancelled.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <strings.h>
+#include <libgen.h>
+#include <fcntl.h>
+#include <stddef.h>
+#include <signal.h>
+#include <port.h>
+#include <ctype.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <sys/stat.h>
+#include <sys/note.h>
+#include <sys/resource.h>
+#include <unistd.h>
+#include <sys/socket.h>
+#include <errno.h>
+#include <ucred.h>
+#include <priv_utils.h>
+#include <net/if.h>
+#include <libilb.h>
+#include <assert.h>
+#include <inet/ilb.h>
+#include <libintl.h>
+#include <fcntl.h>
+#include <rpcsvc/daemon_utils.h>
+#include "libilb_impl.h"
+#include "ilbd.h"
+
+/*
+ * NOTE: The following needs to be kept up to date.
+ */
+#define	ILBD_VERSION	"1.0"
+#define	ILBD_COPYRIGHT	\
+	"Copyright 2009 Sun Microsystems, Inc.  All rights reserved.\n" \
+	"Use is subject to license terms.\n"
+
+/*
+ * Global reply buffer to client request.  Note that ilbd is single threaded,
+ * so a global buffer is OK.  If ilbd becomes multi-threaded, this needs to
+ * be changed.
+ */
+static uint32_t reply_buf[ILBD_MSG_SIZE / sizeof (uint32_t)];
+
+static void
+ilbd_free_cli(ilbd_client_t *cli)
+{
+	(void) close(cli->cli_sd);
+	if (cli->cli_cmd == ILBD_SHOW_NAT)
+		ilbd_show_nat_cleanup();
+	if (cli->cli_cmd == ILBD_SHOW_PERSIST)
+		ilbd_show_sticky_cleanup();
+	if (cli->cli_saved_reply != NULL)
+		free(cli->cli_saved_reply);
+	free(cli->cli_pw_buf);
+	free(cli);
+}
+
+static void
+ilbd_reset_kernel_state(void)
+{
+	ilb_status_t	rc;
+	ilb_name_cmd_t	kcmd;
+
+	kcmd.cmd = ILB_DESTROY_RULE;
+	kcmd.flags = ILB_RULE_ALLRULES;
+	kcmd.name[0] = '\0';
+
+	rc = do_ioctl(&kcmd, 0);
+	if (rc != ILB_STATUS_OK)
+		logdebug("ilbd_reset_kernel_state: do_ioctl failed: %s",
+		    strerror(errno));
+}
+
+/* Signal handler to do clean up. */
+/* ARGSUSED */
+static void
+ilbd_cleanup(int sig)
+{
+	(void) remove(SOCKET_PATH);
+	ilbd_reset_kernel_state();
+	exit(0);
+}
+
+/*
+ * Create a socket and return it to caller.  If there is a failure, this
+ * function calls exit(2).  Hence it always returns a valid listener socket.
+ *
+ * Note that this function is called before ilbd becomes a daemon.  So
+ * we call perror(3C) to print out error message directly so that SMF can
+ * catch them.
+ */
+static int
+ilbd_create_client_socket(void)
+{
+	int			s;
+	mode_t			omask;
+	struct sockaddr_un	sa;
+	int			sobufsz;
+
+	s = socket(PF_UNIX, SOCK_SEQPACKET, 0);
+	if (s == -1) {
+		perror("ilbd_create_client_socket: socket to"
+		    " client failed");
+		exit(errno);
+	}
+	if (fcntl(s, F_SETFD, FD_CLOEXEC) == -1) {
+		perror("ilbd_create_client_socket: fcntl(FD_CLOEXEC)");
+		exit(errno);
+	}
+
+	sobufsz = ILBD_MSG_SIZE;
+	if (setsockopt(s, SOL_SOCKET, SO_SNDBUF, &sobufsz,
+	    sizeof (sobufsz)) != 0) {
+		perror("ilbd_creat_client_socket: setsockopt(SO_SNDBUF) "
+		    "failed");
+		exit(errno);
+	}
+	if (setsockopt(s, SOL_SOCKET, SO_RCVBUF, &sobufsz,
+	    sizeof (sobufsz)) != 0) {
+		perror("ilbd_creat_client_socket: setsockopt(SO_RCVBUF) "
+		    "failed");
+		exit(errno);
+	}
+
+	/*
+	 * since everybody can talk to us, we need to open up permissions
+	 * we check peer privileges on a per-operation basis.
+	 * This is no security issue as long as we're single-threaded.
+	 */
+	omask = umask(0);
+
+	/* just in case we didn't clean up properly after last exit */
+	(void) remove(SOCKET_PATH);
+
+	bzero(&sa, sizeof (sa));
+	sa.sun_family = AF_UNIX;
+	(void) strlcpy(sa.sun_path, SOCKET_PATH, sizeof (sa.sun_path));
+
+	if (bind(s, (struct sockaddr *)&sa, sizeof (sa)) != 0) {
+		perror("ilbd_create_client_socket(): bind to client"
+		    " socket failed");
+		exit(errno);
+	}
+
+	/* re-instate old umask */
+	(void) umask(omask);
+
+#define	QLEN	16
+
+	if (listen(s, QLEN) != 0) {
+		perror("ilbd_create_client_socket: listen to client"
+		    " socket failed");
+		exit(errno);
+	}
+
+	(void) signal(SIGHUP, SIG_IGN);
+	(void) signal(SIGPIPE, SIG_IGN);
+	(void) signal(SIGSTOP, SIG_IGN);
+	(void) signal(SIGTSTP, SIG_IGN);
+	(void) signal(SIGTTIN, SIG_IGN);
+	(void) signal(SIGTTOU, SIG_IGN);
+
+	(void) signal(SIGINT, ilbd_cleanup);
+	(void) signal(SIGTERM, ilbd_cleanup);
+	(void) signal(SIGQUIT, ilbd_cleanup);
+
+	return (s);
+}
+
+/*
+ * Return the minimum size of a given request.  The returned size does not
+ * include the variable part of a request.
+ */
+static size_t
+ilbd_cmd_size(const ilb_comm_t *ic)
+{
+	size_t cmd_sz;
+
+	cmd_sz = sizeof (*ic);
+	switch (ic->ic_cmd) {
+	case ILBD_RETRIEVE_SG_NAMES:
+	case ILBD_RETRIEVE_RULE_NAMES:
+	case ILBD_RETRIEVE_HC_NAMES:
+	case ILBD_CMD_OK:
+		break;
+	case ILBD_CMD_ERROR:
+		cmd_sz += sizeof (ilb_status_t);
+		break;
+	case ILBD_RETRIEVE_SG_HOSTS:
+	case ILBD_CREATE_SERVERGROUP:
+	case ILBD_DESTROY_SERVERGROUP:
+	case ILBD_DESTROY_RULE:
+	case ILBD_ENABLE_RULE:
+	case ILBD_DISABLE_RULE:
+	case ILBD_RETRIEVE_RULE:
+	case ILBD_DESTROY_HC:
+	case ILBD_GET_HC_INFO:
+	case ILBD_GET_HC_SRVS:
+		cmd_sz += sizeof (ilbd_name_t);
+		break;
+	case ILBD_ENABLE_SERVER:
+	case ILBD_DISABLE_SERVER:
+	case ILBD_ADD_SERVER_TO_GROUP:
+	case ILBD_REM_SERVER_FROM_GROUP:
+		cmd_sz += sizeof (ilb_sg_info_t);
+		break;
+	case ILBD_SRV_ADDR2ID:
+	case ILBD_SRV_ID2ADDR:
+		cmd_sz += sizeof (ilb_sg_info_t) + sizeof (ilb_sg_srv_t);
+		break;
+	case ILBD_CREATE_RULE:
+		cmd_sz += sizeof (ilb_rule_info_t);
+		break;
+	case ILBD_CREATE_HC:
+		cmd_sz += sizeof (ilb_hc_info_t);
+		break;
+	case ILBD_SHOW_NAT:
+	case ILBD_SHOW_PERSIST:
+		cmd_sz += sizeof (ilb_show_info_t);
+		break;
+	}
+
+	return (cmd_sz);
+}
+
+/*
+ * Given a request and its size, check that the size is big enough to
+ * contain the variable part of a request.
+ */
+static ilb_status_t
+ilbd_check_req_size(ilb_comm_t *ic, size_t ic_sz)
+{
+	ilb_status_t rc = ILB_STATUS_OK;
+	ilb_sg_info_t *sg_info;
+	ilbd_namelist_t *nlist;
+
+	switch (ic->ic_cmd) {
+	case ILBD_CREATE_SERVERGROUP:
+	case ILBD_ENABLE_SERVER:
+	case ILBD_DISABLE_SERVER:
+	case ILBD_ADD_SERVER_TO_GROUP:
+	case ILBD_REM_SERVER_FROM_GROUP:
+		sg_info = (ilb_sg_info_t *)&ic->ic_data;
+
+		if (ic_sz < ilbd_cmd_size(ic) + sg_info->sg_srvcount *
+		    sizeof (ilb_sg_srv_t)) {
+			rc = ILB_STATUS_EINVAL;
+		}
+		break;
+	case ILBD_ENABLE_RULE:
+	case ILBD_DISABLE_RULE:
+	case ILBD_DESTROY_RULE:
+		nlist = (ilbd_namelist_t *)&ic->ic_data;
+
+		if (ic_sz < ilbd_cmd_size(ic) + nlist->ilbl_count *
+		    sizeof (ilbd_name_t)) {
+			rc = ILB_STATUS_EINVAL;
+		}
+		break;
+	}
+	return (rc);
+}
+
+/*
+ * this function *relies* on a complete message/data struct
+ * being passed in (currently via the SOCK_SEQPACKET socket type).
+ *
+ * Note that the size of ip is at most ILBD_MSG_SIZE.
+ */
+static ilb_status_t
+consume_common_struct(ilb_comm_t *ic, size_t ic_sz, ilbd_client_t *cli,
+    int ev_port)
+{
+	ilb_status_t	rc;
+	struct passwd	*ps;
+	size_t		rbufsz;
+	ssize_t		ret;
+	boolean_t	standard_reply = B_TRUE;
+	ilbd_name_t	name;
+
+	/*
+	 * cli_ev must be overridden during handling of individual commands,
+	 * if there's a special need; otherwise, leave this for
+	 * the "default" case
+	 */
+	cli->cli_ev = ILBD_EVENT_REQ;
+
+	ps = &cli->cli_pw;
+	rbufsz = ILBD_MSG_SIZE;
+
+	/* Sanity check on the size of the static part of a request. */
+	if (ic_sz < ilbd_cmd_size(ic)) {
+		rc = ILB_STATUS_EINVAL;
+		goto out;
+	}
+
+	switch (ic->ic_cmd) {
+	case ILBD_CREATE_SERVERGROUP: {
+		ilb_sg_info_t sg_info;
+
+		/*
+		 * ilbd_create_sg() only needs the sg_name field.  But it
+		 * takes in a ilb_sg_info_t because it is used as a callback
+		 * in ilbd_walk_sg_pgs().
+		 */
+		(void) strlcpy(sg_info.sg_name, (char *)&(ic->ic_data),
+		    sizeof (sg_info.sg_name));
+		rc = ilbd_create_sg(&sg_info, ev_port, ps,
+		    cli->cli_peer_ucredp);
+		break;
+	}
+
+	case ILBD_DESTROY_SERVERGROUP:
+		(void) strlcpy(name, (char *)&(ic->ic_data), sizeof (name));
+		rc = ilbd_destroy_sg(name, ps, cli->cli_peer_ucredp);
+		break;
+
+	case ILBD_ADD_SERVER_TO_GROUP:
+		if ((rc = ilbd_check_req_size(ic, ic_sz)) != ILB_STATUS_OK)
+			break;
+		rc = ilbd_add_server_to_group((ilb_sg_info_t *)&ic->ic_data,
+		    ev_port, ps, cli->cli_peer_ucredp);
+		break;
+
+	case ILBD_REM_SERVER_FROM_GROUP:
+		if ((rc = ilbd_check_req_size(ic, ic_sz)) != ILB_STATUS_OK)
+			break;
+		rc = ilbd_rem_server_from_group((ilb_sg_info_t *)&ic->ic_data,
+		    ev_port, ps, cli->cli_peer_ucredp);
+		break;
+
+	case ILBD_ENABLE_SERVER:
+		if ((rc = ilbd_check_req_size(ic, ic_sz)) != ILB_STATUS_OK)
+			break;
+		rc = ilbd_enable_server((ilb_sg_info_t *)&ic->ic_data, ps,
+		    cli->cli_peer_ucredp);
+		break;
+
+	case ILBD_DISABLE_SERVER:
+		if ((rc = ilbd_check_req_size(ic, ic_sz)) != ILB_STATUS_OK)
+			break;
+		rc = ilbd_disable_server((ilb_sg_info_t *)&ic->ic_data, ps,
+		    cli->cli_peer_ucredp);
+		break;
+
+	case ILBD_SRV_ADDR2ID:
+		rc = ilbd_address_to_srvID((ilb_sg_info_t *)&ic->ic_data,
+		    reply_buf, &rbufsz);
+		if (rc == ILB_STATUS_OK)
+			standard_reply = B_FALSE;
+		break;
+
+	case ILBD_SRV_ID2ADDR:
+		rc = ilbd_srvID_to_address((ilb_sg_info_t *)&ic->ic_data,
+		    reply_buf, &rbufsz);
+		if (rc == ILB_STATUS_OK)
+			standard_reply = B_FALSE;
+		break;
+
+	case ILBD_RETRIEVE_SG_HOSTS:
+		(void) strlcpy(name, (char *)&(ic->ic_data), sizeof (name));
+		rc = ilbd_retrieve_sg_hosts(name, reply_buf, &rbufsz);
+		if (rc == ILB_STATUS_OK)
+			standard_reply = B_FALSE;
+		break;
+
+	case ILBD_RETRIEVE_SG_NAMES:
+	case ILBD_RETRIEVE_RULE_NAMES:
+	case ILBD_RETRIEVE_HC_NAMES:
+		rc = ilbd_retrieve_names(ic->ic_cmd, reply_buf, &rbufsz);
+		if (rc == ILB_STATUS_OK)
+			standard_reply = B_FALSE;
+		break;
+
+	case ILBD_CREATE_RULE:
+		rc = ilbd_create_rule((ilb_rule_info_t *)&ic->ic_data, ev_port,
+		    ps, cli->cli_peer_ucredp);
+		break;
+
+	case ILBD_DESTROY_RULE:
+		/* Copy the name to ensure that name is NULL terminated. */
+		(void) strlcpy(name, (char *)&(ic->ic_data), sizeof (name));
+		rc = ilbd_destroy_rule(name, ps, cli->cli_peer_ucredp);
+		break;
+
+	case ILBD_ENABLE_RULE:
+		(void) strlcpy(name, (char *)&(ic->ic_data), sizeof (name));
+		rc = ilbd_enable_rule(name, ps, cli->cli_peer_ucredp);
+		break;
+
+	case ILBD_DISABLE_RULE:
+		(void) strlcpy(name, (char *)&(ic->ic_data), sizeof (name));
+		rc = ilbd_disable_rule(name, ps, cli->cli_peer_ucredp);
+		break;
+
+	case ILBD_RETRIEVE_RULE:
+		(void) strlcpy(name, (char *)&(ic->ic_data), sizeof (name));
+		rc = ilbd_retrieve_rule(name, reply_buf, &rbufsz);
+		if (rc == ILB_STATUS_OK)
+			standard_reply = B_FALSE;
+		break;
+
+	case ILBD_CREATE_HC:
+		rc = ilbd_create_hc((ilb_hc_info_t *)&ic->ic_data, ev_port, ps,
+		    cli->cli_peer_ucredp);
+		break;
+
+	case ILBD_DESTROY_HC:
+		(void) strlcpy(name, (char *)&(ic->ic_data), sizeof (name));
+		rc = ilbd_destroy_hc(name, ps, cli->cli_peer_ucredp);
+		break;
+
+	case ILBD_GET_HC_INFO:
+		(void) strlcpy(name, (char *)&(ic->ic_data), sizeof (name));
+		rc = ilbd_get_hc_info(name, reply_buf, &rbufsz);
+		if (rc == ILB_STATUS_OK)
+			standard_reply = B_FALSE;
+		break;
+
+	case ILBD_GET_HC_SRVS:
+		(void) strlcpy(name, (char *)&(ic->ic_data), sizeof (name));
+		rc = ilbd_get_hc_srvs(name, reply_buf, &rbufsz);
+		if (rc == ILB_STATUS_OK)
+			standard_reply = B_FALSE;
+		break;
+
+	case ILBD_SHOW_NAT:
+		rc = ilbd_show_nat(cli, ic, reply_buf, &rbufsz);
+		if (rc == ILB_STATUS_OK)
+			standard_reply = B_FALSE;
+		break;
+
+	case ILBD_SHOW_PERSIST:
+		rc = ilbd_show_sticky(cli, ic, reply_buf, &rbufsz);
+		if (rc == ILB_STATUS_OK)
+			standard_reply = B_FALSE;
+		break;
+
+	default:
+		logdebug("consume_common_struct: unknown command");
+		rc = ILB_STATUS_INVAL_CMD;
+		break;
+	}
+
+out:
+	/*
+	 * The message exchange is always in pairs, request/response.  If
+	 * a transaction requires multiple exchanges, the client will send
+	 * in multiple requests to get multiple responses.  The show-nat and
+	 * show-persist request are examples of this.  The end of transaction
+	 * is marked with ic_flags set to ILB_COMM_END.
+	 */
+
+	/* This is the standard reply. */
+	if (standard_reply) {
+		if (rc == ILB_STATUS_OK)
+			ilbd_reply_ok(reply_buf, &rbufsz);
+		else
+			ilbd_reply_err(reply_buf, &rbufsz, rc);
+	}
+
+	if ((ret = send(cli->cli_sd, reply_buf, rbufsz, 0)) != rbufsz) {
+		if (ret == -1) {
+			if (errno != EWOULDBLOCK) {
+				logdebug("consume_common_struct: send: %s",
+				    strerror(errno));
+				rc = ILB_STATUS_SEND;
+				goto err_out;
+			}
+			/*
+			 * The reply is blocked, save the reply.  handle_req()
+			 * will associate the event port for the re-send.
+			 */
+			assert(cli->cli_saved_reply == NULL);
+			if ((cli->cli_saved_reply = malloc(rbufsz)) == NULL) {
+				/*
+				 * Set the error to ILB_STATUS_SEND so that
+				 * handle_req() will free the client.
+				 */
+				logdebug("consume_common_struct: failure to "
+				    "allocate memory to save reply");
+				rc = ILB_STATUS_SEND;
+				goto err_out;
+			}
+			bcopy(reply_buf, cli->cli_saved_reply, rbufsz);
+			cli->cli_saved_size = rbufsz;
+			return (ILB_STATUS_EWOULDBLOCK);
+		}
+	}
+err_out:
+	return (rc);
+}
+
+/*
+ * Accept a new client request.  A struct ilbd_client_t is allocated to
+ * store the client info.  The accepted socket is port_associate() with
+ * the given port.  And the allocated ilbd_client_t struct is passed as
+ * the user pointer.
+ */
+static void
+new_req(int ev_port, int listener, void *ev_obj)
+{
+	struct sockaddr	sa;
+	int		sa_len;
+	int		new_sd;
+	int		sflags;
+	ilbd_client_t	*cli;
+	int		res;
+	uid_t		uid;
+
+	sa_len = sizeof (sa);
+	if ((new_sd = accept(listener, &sa, &sa_len)) == -1) {
+		/* don't log if we're out of file descriptors */
+		if (errno != EINTR && errno != EMFILE)
+			logperror("new_req: accept failed");
+		goto done;
+	}
+
+	/* Set the new socket to be non-blocking. */
+	if ((sflags = fcntl(new_sd, F_GETFL, 0)) == -1) {
+		logperror("new_req: fcntl(F_GETFL)");
+		goto clean_up;
+	}
+	if (fcntl(new_sd, F_SETFL, sflags | O_NONBLOCK) == -1) {
+		logperror("new_req: fcntl(F_SETFL)");
+		goto clean_up;
+	}
+	if (fcntl(new_sd, F_SETFD, FD_CLOEXEC) == -1) {
+		logperror("new_req: fcntl(FD_CLOEXEC)");
+		goto clean_up;
+	}
+	if ((cli = calloc(1, sizeof (ilbd_client_t))) == NULL) {
+		logerr("new_req: malloc(ilbd_client_t)");
+		goto clean_up;
+	}
+	res = getpeerucred(new_sd, &cli->cli_peer_ucredp);
+	if (res == -1) {
+		logperror("new_req: getpeerucred failed");
+		goto clean_up;
+	}
+	if ((uid = ucred_getruid(cli->cli_peer_ucredp)) == (uid_t)-1) {
+		logperror("new_req: ucred_getruid failed");
+		goto clean_up;
+	}
+	cli->cli_pw_bufsz = (size_t)sysconf(_SC_GETPW_R_SIZE_MAX);
+	if ((cli->cli_pw_buf = malloc(cli->cli_pw_bufsz)) == NULL) {
+		free(cli);
+		logerr("new_req: malloc(cli_pw_buf)");
+		goto clean_up;
+	}
+	if (getpwuid_r(uid, &cli->cli_pw, cli->cli_pw_buf,
+	    cli->cli_pw_bufsz) == NULL) {
+		free(cli->cli_pw_buf);
+		free(cli);
+		logperror("new_req: invalid user");
+		goto clean_up;
+	}
+	cli->cli_ev = ILBD_EVENT_REQ;
+	cli->cli_sd = new_sd;
+	cli->cli_cmd = ILBD_BAD_CMD;
+	cli->cli_saved_reply = NULL;
+	cli->cli_saved_size = 0;
+	if (port_associate(ev_port, PORT_SOURCE_FD, new_sd, POLLRDNORM,
+	    cli) == -1) {
+		logperror("new_req: port_associate(cli) failed");
+		free(cli->cli_pw_buf);
+		free(cli);
+clean_up:
+		(void) close(new_sd);
+	}
+
+done:
+	/* Re-associate the listener with the event port. */
+	if (port_associate(ev_port, PORT_SOURCE_FD, listener, POLLRDNORM,
+	    ev_obj) == -1) {
+		logperror("new_req: port_associate(listener) failed");
+		exit(1);
+	}
+}
+
+static void
+handle_req(int ev_port, ilbd_event_t event, ilbd_client_t *cli)
+{
+	/* All request should be smaller than ILBD_MSG_SIZE */
+	union {
+		ilb_comm_t	ic;
+		uint32_t	buf[ILBD_MSG_SIZE / sizeof (uint32_t)];
+	} ic_u;
+	int	rc = ILB_STATUS_OK;
+	ssize_t	r;
+
+	if (event == ILBD_EVENT_REQ) {
+		/*
+		 * Something is wrong with the client since there is a
+		 * pending reply, the client should not send us another
+		 * request.  Kill this client.
+		 */
+		if (cli->cli_saved_reply != NULL) {
+			logerr("handle_req: misbehaving client, more than one "
+			    "outstanding request");
+			rc = ILB_STATUS_INTERNAL;
+			goto err_out;
+		}
+
+		/*
+		 * Our socket is message based so we should be able
+		 * to get the request in one single read.
+		 */
+		r = recv(cli->cli_sd, (void *)ic_u.buf, sizeof (ic_u.buf), 0);
+		if (r < 0) {
+			if (errno != EINTR) {
+				logperror("handle_req: read failed");
+				rc = ILB_STATUS_READ;
+				goto err_out;
+			}
+			/*
+			 * If interrupted, just re-associate the cli_sd
+			 * with the port.
+			 */
+			goto done;
+		}
+		cli->cli_cmd = ic_u.ic.ic_cmd;
+
+		rc = consume_common_struct(&ic_u.ic, r, cli, ev_port);
+		if (rc == ILB_STATUS_EWOULDBLOCK)
+			goto blocked;
+		/* Fatal error communicating with client, free it. */
+		if (rc == ILB_STATUS_SEND)
+			goto err_out;
+	} else {
+		assert(event == ILBD_EVENT_REP_OK);
+		assert(cli->cli_saved_reply != NULL);
+
+		/*
+		 * The reply to client was previously blocked, we will
+		 * send again.
+		 */
+		if (send(cli->cli_sd, cli->cli_saved_reply,
+		    cli->cli_saved_size, 0) != cli->cli_saved_size) {
+			if (errno != EWOULDBLOCK) {
+				logdebug("handle_req: send: %s",
+				    strerror(errno));
+				rc = ILB_STATUS_SEND;
+				goto err_out;
+			}
+			goto blocked;
+		}
+		free(cli->cli_saved_reply);
+		cli->cli_saved_reply = NULL;
+		cli->cli_saved_size = 0;
+	}
+done:
+	/* Re-associate with the event port for more requests. */
+	cli->cli_ev = ILBD_EVENT_REQ;
+	if (port_associate(ev_port, PORT_SOURCE_FD, cli->cli_sd,
+	    POLLRDNORM, cli) == -1) {
+		logperror("handle_req: port_associate(POLLRDNORM)");
+		rc = ILB_STATUS_INTERNAL;
+		goto err_out;
+	}
+	return;
+
+blocked:
+	/* Re-associate with the event port. */
+	cli->cli_ev = ILBD_EVENT_REP_OK;
+	if (port_associate(ev_port, PORT_SOURCE_FD, cli->cli_sd, POLLWRNORM,
+	    cli) == -1) {
+		logperror("handle_req: port_associate(POLLWRNORM)");
+		rc = ILB_STATUS_INTERNAL;
+		goto err_out;
+	}
+	return;
+
+err_out:
+	ilbd_free_cli(cli);
+}
+
+static void
+i_ilbd_read_config(int ev_port)
+{
+	logdebug("i_ilbd_read_config: port %d", ev_port);
+	(void) ilbd_walk_sg_pgs(ilbd_create_sg, &ev_port, NULL);
+	(void) ilbd_walk_hc_pgs(ilbd_create_hc, &ev_port, NULL);
+	(void) ilbd_walk_rule_pgs(ilbd_create_rule, &ev_port, NULL);
+}
+
+/*
+ * main event loop for ilbd
+ * asserts that argument 'listener' is a server socket ready to accept() on.
+ */
+static void
+main_loop(int listener)
+{
+	port_event_t		p_ev;
+	int			ev_port, ev_port_obj;
+	ilbd_event_obj_t	ev_obj;
+	ilbd_timer_event_obj_t	timer_ev_obj;
+
+	ev_port = port_create();
+	if (ev_port == -1) {
+		logperror("main_loop: port_create failed");
+		exit(-1);
+	}
+	ilbd_hc_timer_init(ev_port, &timer_ev_obj);
+
+	ev_obj.ev = ILBD_EVENT_NEW_REQ;
+	if (port_associate(ev_port, PORT_SOURCE_FD, listener, POLLRDNORM,
+	    &ev_obj) == -1) {
+		logperror("main_loop: port_associate failed");
+		exit(1);
+	}
+
+	i_ilbd_read_config(ev_port);
+	ilbd_hc_timer_update(&timer_ev_obj);
+
+	_NOTE(CONSTCOND)
+	while (B_TRUE) {
+		int r;
+		ilbd_event_t event;
+		ilbd_client_t *cli;
+
+		r = port_get(ev_port, &p_ev, NULL);
+		if (r == -1) {
+			if (errno == EINTR)
+				continue;
+			logperror("main_loop: port_get failed");
+			break;
+		}
+
+		ev_port_obj = p_ev.portev_object;
+		event = ((ilbd_event_obj_t *)p_ev.portev_user)->ev;
+
+		switch (event) {
+		case ILBD_EVENT_TIMER:
+			ilbd_hc_timeout();
+			break;
+
+		case ILBD_EVENT_PROBE:
+			ilbd_hc_probe_return(ev_port, ev_port_obj,
+			    p_ev.portev_events,
+			    (ilbd_hc_probe_event_t *)p_ev.portev_user);
+			break;
+
+		case ILBD_EVENT_NEW_REQ:
+			assert(ev_port_obj == listener);
+			/*
+			 * An error happens in the listener.  Exit
+			 * for now....
+			 */
+			if (p_ev.portev_events & (POLLHUP|POLLERR)) {
+				logerr("main_loop: listener error");
+				exit(1);
+			}
+			new_req(ev_port, ev_port_obj, &ev_obj);
+			break;
+
+		case ILBD_EVENT_REP_OK:
+		case ILBD_EVENT_REQ:
+			cli = (ilbd_client_t *)p_ev.portev_user;
+			assert(ev_port_obj == cli->cli_sd);
+
+			/*
+			 * An error happens in the newly accepted
+			 * client request.  Clean up the client.
+			 * this also happens when client closes socket,
+			 * so not necessarily a reason for alarm
+			 */
+			if (p_ev.portev_events & (POLLHUP|POLLERR)) {
+				ilbd_free_cli(cli);
+				break;
+			}
+
+			handle_req(ev_port, event, cli);
+			break;
+
+		default:
+			logerr("main_loop: unknown event %d", event);
+			exit(EXIT_FAILURE);
+			break;
+		}
+
+		ilbd_hc_timer_update(&timer_ev_obj);
+	}
+}
+
+static void
+i_ilbd_setup_lists(void)
+{
+	i_setup_sg_hlist();
+	i_setup_rule_hlist();
+	i_ilbd_setup_hc_list();
+}
+
+/*
+ * Usage message - call only during startup. it will print its
+ * message on stderr and exit
+ */
+static void
+Usage(char *name)
+{
+	(void) fprintf(stderr, gettext("Usage: %s [-d|--debug]\n"), name);
+	exit(1);
+}
+
+static void
+print_version(char *name)
+{
+	(void) printf("%s %s\n", basename(name), ILBD_VERSION);
+	(void) printf(gettext(ILBD_COPYRIGHT));
+	exit(0);
+}
+
+/*
+ * Increase the file descriptor limit for handling a lot of health check
+ * processes (each requires a pipe).
+ *
+ * Note that this function is called before ilbd becomes a daemon.  So
+ * we call perror(3C) to print out error message directly so that SMF
+ * can catch them.
+ */
+static void
+set_rlim(void)
+{
+	struct rlimit rlp;
+
+	if (getrlimit(RLIMIT_NOFILE, &rlp) == -1) {
+		perror("ilbd: getrlimit");
+		exit(errno);
+	}
+	rlp.rlim_cur = rlp.rlim_max;
+	if (setrlimit(RLIMIT_NOFILE, &rlp) == -1) {
+		perror("ilbd: setrlimit");
+		exit(errno);
+	}
+}
+
+int
+main(int argc, char **argv)
+{
+	int	s;
+	int	c;
+
+	(void) setlocale(LC_ALL, "");
+#if !defined(TEXT_DOMAIN)
+#define	TEXT_DOMAIN "SYS_TEST"
+#endif
+	static const char daemon_dir[] = DAEMON_DIR;
+
+	(void) textdomain(TEXT_DOMAIN);
+
+	while ((c = getopt(argc, argv, ":V?d(debug)")) != -1) {
+		switch ((char)c) {
+		case '?': Usage(argv[0]);
+			/* not reached */
+			break;
+		case 'V': print_version(argv[0]);
+			/* not reached */
+			break;
+		case 'd': ilbd_enable_debug();
+			break;
+		default: Usage(argv[0]);
+			/* not reached */
+			break;
+		}
+	}
+
+	/*
+	 * Whenever the daemon starts, it needs to start with a clean
+	 * slate in the kernel. We need sys_ip_config privilege for
+	 * this.
+	 */
+	ilbd_reset_kernel_state();
+
+	/* Increase the limit on the number of file descriptors. */
+	set_rlim();
+
+	/*
+	 * ilbd daemon starts off as root, just so it can create
+	 * /var/run/daemon if one does not exist. After that is done
+	 * the daemon switches to "daemon" uid. This is similar to what
+	 * rpcbind does.
+	 */
+	if (mkdir(daemon_dir, DAEMON_DIR_MODE) == 0 || errno == EEXIST) {
+		(void) chmod(daemon_dir, DAEMON_DIR_MODE);
+		(void) chown(daemon_dir, DAEMON_UID, DAEMON_GID);
+	} else {
+		perror("main: mkdir failed");
+		exit(errno);
+	}
+	/*
+	 * Now lets switch ilbd as uid = daemon, gid = daemon with a
+	 * trimmed down privilege set
+	 */
+	if (__init_daemon_priv(PU_RESETGROUPS | PU_LIMITPRIVS | PU_INHERITPRIVS,
+	    DAEMON_UID, DAEMON_GID, PRIV_PROC_OWNER, PRIV_PROC_AUDIT,
+	    PRIV_NET_ICMPACCESS, PRIV_SYS_IP_CONFIG, NULL) == -1) {
+		(void) fprintf(stderr, "Insufficient privileges\n");
+		exit(EXIT_FAILURE);
+	}
+
+	/*
+	 * Opens a PF_UNIX socket to the client. No privilege needed
+	 * for this.
+	 */
+	s = ilbd_create_client_socket();
+
+	/*
+	 * Daemonify if ilbd is not running with -d option
+	 * Need proc_fork privilege for this
+	 */
+	if (!is_debugging_on()) {
+		logdebug("daemonizing...");
+		if (daemon(0, 0) != 0) {
+			logperror("daemon failed");
+			exit(EXIT_FAILURE);
+		}
+	}
+	(void) priv_set(PRIV_OFF, PRIV_INHERITABLE, PRIV_PROC_OWNER,
+	    PRIV_PROC_AUDIT, NULL);
+
+	/* if daemonified then set up syslog */
+	if (!is_debugging_on())
+		openlog("ilbd", LOG_PID, LOG_DAEMON);
+
+	i_ilbd_setup_lists();
+
+	main_loop(s);
+
+	/*
+	 * if we come here, then we experienced an error or a shutdown
+	 * indicator, so clean up after ourselves.
+	 */
+	logdebug("main(): terminating");
+
+	(void) remove(SOCKET_PATH);
+	ilbd_reset_kernel_state();
+
+	return (0);
+}