Implement periodic "heartbeat" event.

Provide a sample handler for checking if the established link is still
alive and terminating the link command if it is not.

* Makefile.am (SUBDIRS): Add extra.
* configure.ac: Build extra/Makefile.am
* src/config.c (str_to_cidr): Minor fix.
(parse_event): New event: HEARTBEAT
(jumper_kw): New keyword: heartbeat
(config_finish): Initialize config.heartbeat
* src/environ.c (JUMPER_PID): New environment variable.
* src/jumper.h (jumper_config) <heartbeat>: New field.
(event_heartbeat): New event.
* src/listener.c (listener_kve_init): Don't define envvars with
NULL value.
(listener_run_action): Change to extern
(runaction): Define "pid" variable for listeners.
(listener_start): Use config.heartbeat as the timeout.
* src/progman.c (progman_expire): Special handling for type_listener
(process_register): Update listener_t if required
(progman_cleanup): Switch to stat_down from stat_onexit only.
(progman_terminate): Call progman_cleanup

* extra/Makefile.am: New file.
* extra/ifalive.c: New file.

11 files changed, 509 insertions, 51 deletions

diff --git a/Makefile.am b/Makefile.am
index c1ebd06..29b2adf 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -16,7 +16,7 @@
 
 ACLOCAL_AMFLAGS = -I grecs/am
 
-SUBDIRS=grecs src doc
+SUBDIRS=grecs src extra doc
 
 .PHONY: ChangeLog
 ChangeLog:
diff --git a/configure.ac b/configure.ac
index ef3da6e..c27e39c 100644
--- a/configure.ac
+++ b/configure.ac
@@ -75,5 +75,6 @@ GRECS_SETUP([grecs],[tree-api git2chg getopt])
 
 AC_CONFIG_FILES([Makefile
                  src/Makefile
+		 extra/Makefile
 		 doc/Makefile])
 AC_OUTPUT
diff --git a/extra/Makefile.am b/extra/Makefile.am
new file mode 100644
index 0000000..8cbcf01
--- /dev/null
+++ b/extra/Makefile.am
@@ -0,0 +1,2 @@
+libexec_PROGRAMS=ifalive
+ifalive_SOURCES=ifalive.c
diff --git a/extra/ifalive.c b/extra/ifalive.c
new file mode 100644
index 0000000..8453b74
--- /dev/null
+++ b/extra/ifalive.c
@@ -0,0 +1,403 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <errno.h>
+#include <stdarg.h>
+#include <string.h>
+#include <stdint.h>
+#include <signal.h>
+#include <assert.h>
+
+#ifndef PATH_PROCNET_DEV
+# define PATH_PROCNET_DEV "/proc/net/dev"
+#endif
+
+char *progname;
+char *spooldir = "/var/run/ifalive";
+int debug;
+int dry_run;
+int sig = SIGTERM;
+int parent;
+
+#define EX_OK 0
+#define EX_FAIL 1
+
+void
+error(int ec, char const *fmt, ...)
+{
+	va_list ap;
+	fprintf (stderr, "%s: ", progname);
+	va_start (ap, fmt);
+	vfprintf (stderr, fmt, ap);
+	va_end (ap);
+	if (ec) 
+		fprintf(stderr, ": %s", strerror(ec));
+	fputc('\n', stderr);
+}
+
+void
+usage(int code)
+{
+	FILE *fp = code ? stderr : stdout;
+	fprintf(fp,
+		"Usage: ifalive -i [-dnp] IFACE [PID]\n"
+		"       ifalive [-c] [-dn] [-s SIG] PID\n"
+		"       ifalive -r [-dnp] [PID]\n"
+		"       ifalive -h\n"
+		"\n"
+		"Actions:\n"
+		"\n"
+		"   -i          initializes the state file\n"
+		"   -c          checks the state of the link (the default)\n"
+		"   -r          removes the state file for PID\n"
+		"   -h          displays this help summary\n"
+		"\n"
+		"Options:\n"
+		"\n"
+		"   -d          debug information\n"
+		"   -n          dry-run: print everything, do nothing\n"
+		"   -p          use parent of PID (grand-parent for -pp, etc.)\n"
+		"   -s SIG      terminate idle process with signal SIG, instead\n"
+		"               of SIGTERM\n"
+		);
+	exit(code);
+}
+
+static size_t
+trimnl(char *s)
+{
+	size_t n = strlen(s);
+	if (n > 0 && s[n-1] == '\n')
+		s[--n] = 0;
+	return n;
+}
+
+enum {
+	stat_rx_bytes,
+	stat_rx_packets,
+	stat_rx_errors,
+	stat_rx_dropped,
+	stat_rx_fifo_errors,
+	stat_rx_frame_errors,
+	stat_rx_compressed,
+	stat_rx_multicast,
+	stat_tx_bytes,
+	stat_tx_packets,
+	stat_tx_errors,
+	stat_tx_dropped,
+	stat_tx_fifo_errors,
+	stat_collisions,
+	stat_tx_carrier_errors,
+	stat_tx_compressed,
+
+	MAX_STATS
+};
+
+static int
+getnum(uintmax_t *ret, char **pptr)
+{
+	char *p = *pptr;
+	uintmax_t acc = 0;
+	static char digits[] = "0123456789";
+
+	while (*p && (*p == ' ' || *p == '\t'))
+		p++;
+
+	if (!*p || !strchr(digits, *p)) {
+		*pptr = p;
+		return -1;
+	}
+	
+	while (*p) {
+		char *q = strchr(digits, *p);
+		if (!q)
+			break;
+		p++;
+		acc = acc * 10 + q - digits;
+	}
+	
+	*pptr = p;
+	*ret = acc;
+	return 0;
+}
+
+static int
+procnet_parse_line(char *p, char **iface, uintmax_t statbuf[MAX_STATS])
+{
+	int i;
+	char *ifp;
+	
+	while (*p && (*p == ' ' || *p == '\t'))
+		p++;
+	ifp = p;
+	while (*p && *p != ':')
+		p++;
+	if (!*p)
+		return -1;
+	*p++ = 0;
+	*iface = ifp;
+	for (i = 0; i < MAX_STATS; i++) {
+		if (getnum(&statbuf[i], &p))
+			return -1;
+	}
+	return 0;
+}
+
+void
+procnet_write_line(FILE *fp, char const *iface, uintmax_t statbuf[MAX_STATS])
+{
+	int i;
+	
+	fprintf(fp, "%s:", iface);
+	for (i = 0; i < MAX_STATS; i++) 
+		fprintf(fp, " %jd", statbuf[i]);
+	fputc('\n', fp);
+}
+
+int
+procnet_read(char const *iface, uintmax_t statbuf[MAX_STATS])
+{
+	FILE *fp;
+	char buf[1024];
+	int rc;
+	
+	fp = fopen(PATH_PROCNET_DEV, "r");
+	if (!fp) {
+		error(errno, "can't open \"%s\"", PATH_PROCNET_DEV);
+		return EX_FAIL;
+	}
+
+	if (!fgets(buf, sizeof buf, fp)
+	    || !fgets(buf, sizeof buf, fp)) {
+		error(errno, "%s: malformed file");
+		return EX_FAIL;
+	}
+	
+	rc = EX_FAIL;
+	while (fgets(buf, sizeof buf, fp)) {
+		char *ifp;
+
+		trimnl(buf);
+		if (procnet_parse_line(buf, &ifp, statbuf)) {
+			error(errno, "%s: malformed file", PATH_PROCNET_DEV);
+			break;
+		}
+		
+		if (strcmp(iface, ifp) == 0) {
+			rc = EX_OK;
+			break;
+		}
+	}
+	fclose(fp);
+	return rc;
+}
+
+static pid_t
+getparentpid(pid_t pid)
+{
+	char fname[512];
+	FILE *fp;
+	unsigned long ppid;
+	int n, ec;
+	
+	snprintf(fname, sizeof fname, "/proc/%lu/stat", (unsigned long) pid);
+	fp = fopen(fname, "r");
+	if (!fp) {
+		error(errno, "can't open \"%s\" for reading", fname);
+		return (pid_t) -1;
+	}
+
+	n = fscanf(fp, "%*u %*s %*s %lu", &ppid);
+	ec = errno;
+	fclose(fp);
+	if (n != 1) {
+		error(ec, "error reading from \"%s\"", fname);
+		return (pid_t) -1;
+	}
+
+	return ppid;
+}
+
+char *
+argpid(char *arg)
+{
+	char pbuf[512];
+	char *pstr;
+	pid_t pid;
+	
+	if (arg) {
+		char *p;
+		pid = strtoul(arg, &p, 10);
+		if (*p) {
+			error(0, "invalid PID");
+			return NULL;
+		}
+	} else
+		pid = getppid();
+
+	for (; parent; parent--) {
+		pid = getparentpid(pid);
+		if (pid == 1 || pid == (pid_t)-1)
+			return NULL;
+	}
+	snprintf(pbuf, sizeof(pbuf), "%lu", (unsigned long)pid);
+	pstr = strdup(pbuf);
+	if (!pstr) 
+		error(errno, "strdup");
+	return pstr;
+}
+
+int
+ifalive_init(int argc, char **argv)
+{
+	FILE *fp;
+	char *pstr;
+	
+	if (argc < 1 || argc > 2)
+		usage(EX_FAIL);
+	
+	pstr = argpid(argv[1]);
+	if (!pstr)
+		return EX_FAIL;
+
+	if (debug)
+		printf("writing \"%s/%s\"; %s\n", spooldir, pstr, argv[0]);
+	if (dry_run)
+		return EX_OK;
+	
+	fp = fopen(pstr, "w");
+	if (!fp) {
+		error(errno, "can't open \"%s/%s\" for writing", spooldir,
+		      pstr);
+		return EX_FAIL;
+	}
+	fprintf(fp, "%s\n", argv[0]);
+	fclose(fp);
+	return EX_OK;
+}
+
+int
+ifalive_remove(int argc, char **argv)
+{
+	char *pstr;
+	
+	if (argc > 1)
+		usage(EX_FAIL);
+	
+	pstr = argpid(argv[0]);
+	if (!pstr)
+		return EX_FAIL;
+
+	if (debug)
+		printf("removing %s/%s\n", spooldir, pstr);
+			
+	if (!dry_run && unlink(pstr)) {
+		error(errno, "%s/%s: can't unlink", spooldir, pstr);
+		return EX_FAIL;
+	}
+	return EX_OK;
+}
+
+int
+ifalive_check(int argc, char **argv)
+{
+	FILE *fp;
+	char buf[512];
+	char *iface;
+	uintmax_t statbuf[MAX_STATS], snapshot[MAX_STATS];
+	
+	if (argc != 1)
+		usage(EX_FAIL);		
+	
+	fp = fopen(argv[0], "r+");
+	if (!fp) {
+		error(errno, "can't open \"%s/%s\"", spooldir, argv[0]);
+		return EX_FAIL;
+	}
+	
+	if (!fgets(buf, sizeof buf, fp)) {
+		error(errno, "fgets");
+		return EX_FAIL;
+	}
+	
+	trimnl(buf);
+	if (strchr(buf, ':') == 0) {
+		iface = buf;
+		memset(snapshot, 0, sizeof snapshot);
+	} else if (procnet_parse_line(buf, &iface, snapshot)) {
+		error(0, "%s/%s: malformed snapshot", spooldir, argv[0]);
+		return EX_FAIL;
+	}
+	
+	if (debug)
+		printf("interface: %s\n", iface);
+	
+	if (procnet_read(iface, statbuf))
+		return EX_FAIL;
+	
+	if (snapshot[stat_rx_bytes] == statbuf[stat_rx_bytes]
+	    && snapshot[stat_tx_bytes] == statbuf[stat_tx_bytes]) {
+		if (debug)
+			printf("%s is IDLE; terminate %s\n", iface, argv[0]);
+		if (!dry_run) {
+			char *endp;
+			pid_t pid = strtoul(argv[0], &endp, 10);
+			assert(pid >= 2);
+			kill(pid, sig);
+		}
+	}
+	rewind(fp);
+	procnet_write_line(fp, iface, statbuf);
+	fclose(fp);
+	return EX_OK;
+}
+
+int
+main(int argc, char **argv)
+{
+	int c;
+	int (*action) (int argc, char **argv) = ifalive_check;
+	
+	progname = argv[0];
+	
+	while ((c = getopt(argc, argv, "cdhinprs:")) != EOF) {
+		switch (c) {
+		case 'c':
+			action = ifalive_check;
+			break;
+		case 'd':
+			debug++;
+			break;
+		case 'h':
+			usage(EX_OK);
+		case 'i':
+			action = ifalive_init;
+			break;
+		case 'n':
+			dry_run = 1;
+			debug++;
+			break;
+		case 'p':
+			parent++;
+			break;
+		case 'r':
+			action = ifalive_remove;
+			break;
+		case 's':
+			sig = strtoul(optarg, NULL, 10);
+			break;
+		default:
+			exit(EX_FAIL);
+		}
+	}
+	
+	if (argc == 1) 
+		usage(EX_FAIL);
+	
+	if (chdir(spooldir)) {
+		error(errno, "can't change to %s", spooldir);
+		return EX_FAIL;
+	}
+	
+	return action(argc - optind, argv + optind);
+}
diff --git a/src/config.c b/src/config.c
index c9004b7..e9d710b 100644
--- a/src/config.c
+++ b/src/config.c
@@ -186,7 +186,7 @@ str_to_cidr(grecs_value_t *val, ipv4_match_list_t **phead,
 
 		n = strtoul(p, &q, 10);
 		if (*q == 0) {
-			if (n <= 0 || n > 32) {
+			if (n < 0 || n > 32) {
 				grecs_error(&val->locus, 0,
 					    "invalid network mask");
 				return 1;
@@ -482,6 +482,8 @@ parse_event(grecs_value_t *val)
 		type = event_startup;
 	} else if (strcmp(arg, "CLEANUP") == 0) {
 		type = event_cleanup;
+	} else if (strcmp(arg, "HEARTBEAT") == 0) {
+		type = event_heartbeat;
 	} else if (isdigit(arg[0])) {
 		errno = 0;
 		n = strtoul(arg, &p, 10);
@@ -736,6 +738,8 @@ static struct grecs_keyword jumper_kw[] = {
 	  grecs_type_int, GRECS_DFLT, &newcfg.debug_level },
 	{ "shutdown-timeout", "seconds", "Set shutdown timeout",
 	  grecs_type_uint, GRECS_DFLT, &newcfg.shutdown_timeout },
+	{ "heartbeat", "seconds", "Set heartbeat interval",
+	  grecs_type_uint, GRECS_DFLT, &newcfg.heartbeat },
 	{ "listen", "id: string", "Configure listener",
 	  grecs_type_section, GRECS_DFLT, NULL, 0,
 	  cb_listen, NULL, listen_kw },
@@ -785,6 +789,9 @@ config_finish(struct grecs_node *tree, int reconfig)
 
 		if (!newcfg.pidfile)
 			newcfg.pidfile = estrdup(DEFAULT_PIDFILE);
+
+		if (!newcfg.heartbeat)
+			newcfg.heartbeat = DEFAULT_HEARTBEAT;
 		
 		if (reconfig) {
 			if (strcmp(newcfg.pidfile, config.pidfile)) {
@@ -800,6 +807,7 @@ config_finish(struct grecs_node *tree, int reconfig)
 
 		config.debug_level = newcfg.debug_level;
 		config.shutdown_timeout = newcfg.shutdown_timeout;
+		config.heartbeat = newcfg.heartbeat;
 		config.print_priority = newcfg.print_priority;
 
 		if (!newcfg.tag)
diff --git a/src/cons_ether.c b/src/cons_ether.c
index 2dbd863..62ac3f2 100644
--- a/src/cons_ether.c
+++ b/src/cons_ether.c
@@ -33,7 +33,7 @@ cons_ether_packet(u_char *user, const struct pcap_pkthdr *h, const u_char *p)
 		}
 		memcpy(&ep, p, sizeof(ep));
 		
-		if (ether_type = ntohs(ep.ether_type)) {
+		if ((ether_type = ntohs(ep.ether_type))) {
 			length -= sizeof(struct ether_header);
 			switch (ether_type) {
 			case ETHERTYPE_IP:
diff --git a/src/diag.c b/src/diag.c
index 70458ca..9cb6ad0 100644
--- a/src/diag.c
+++ b/src/diag.c
@@ -117,7 +117,7 @@ vdiag(int prio, const char *fmt, va_list ap)
 				*p++ = *s++;
 			*p++ = ']';
 			*p++ = ' ';
-			while (*p++ = *fmt++);
+			while ((*p++ = *fmt++));
 			vsyslog(prio, fmtbuf, ap);
 		} else			
 			vsyslog(prio, fmt, ap);
diff --git a/src/environ.c b/src/environ.c
index 1dfe74d..913124e 100644
--- a/src/environ.c
+++ b/src/environ.c
@@ -102,6 +102,7 @@ static char *defenv[] = {
 	"JUMPER_VERSION=" PACKAGE_VERSION,
 	"JUMPER_LOCUS=${file}:${line}",
 	"JUMPER_EVENT=${event}",
+	"JUMPER_PID=${pid}",
 	NULL,
 };
 
diff --git a/src/jumper.h b/src/jumper.h
index 744bb6d..98d0366 100644
--- a/src/jumper.h
+++ b/src/jumper.h
@@ -61,11 +61,16 @@
 # define DEFAULT_PIDFILE LOCALSTATEDIR "/run/jumper.pid"
 #endif
 
+#ifndef DEFAULT_HEARTBEAT
+# define DEFAULT_HEARTBEAT 60
+#endif
+
 struct jumper_config {
 	int foreground;
 	char *pidfile;
 	int debug_level;
 	unsigned shutdown_timeout;
+	unsigned heartbeat;
 	/* logging settings */
 	int facility;
 	char *tag;
@@ -126,7 +131,8 @@ enum event_type {
 	event_startup,
 	event_cleanup,
 	event_exit,
-	event_signal
+	event_signal,
+	event_heartbeat,
 };
 
 typedef struct event {
@@ -165,6 +171,7 @@ typedef struct listener {                /* Listener */
 	char *prog;                      /* Command to bring link up */
 	char **env;                      /* Environment */
 	int options;                     /* Execution options */
+
 	action_t *act_head, *act_tail;
 
 	struct grecs_locus locus;        /* Location in the config file */
@@ -175,13 +182,21 @@ typedef struct listener {                /* Listener */
 	enum listener_status status;     /* Status of this listener */
 	unsigned act_count;              /* Number of running actions in
 					    stat_onexit state */
+
 	int decommission;
 } listener_t;
 
+
 typedef struct {
 	listener_t *head;
 	listener_t *tail;
 } listener_list_t;
+
+enum process_type {
+	type_listener,
+	type_redirector,
+	type_action
+};
 
 #define OPT_NOWAIT 0x01
 #define OPT_STDOUT 0x02
@@ -223,6 +238,8 @@ void listener_print_status(listener_t *lp);
 void listener_proc_report(void);
 void listener_kill_redirector(listener_t *lp, int what);
 void listener_start(listener_t *lp, char **kve);
+int listener_run_action(listener_t *lp, int a);
+
 void onexit_reaction(listener_t *lp);
 
 #define KVE_MINSIZE 7
@@ -254,16 +271,10 @@ int envcmp(char **a, char **b);
 void envfree(char **a);
 
 
-enum process_type {
-	type_listener,
-	type_redirector,
-	type_action
-};
-
 extern int proc_stop;
 void progman_cleanup(void);
 
-void progman_terminate();
+void progman_terminate(void);
 int progman_start(int type, listener_t *lp, struct grecs_locus *loc,
 		  int options, unsigned timeout,
 		  char *prog, char **env, char **kve);
diff --git a/src/listener.c b/src/listener.c
index 9d35579..2960dab 100644
--- a/src/listener.c
+++ b/src/listener.c
@@ -319,18 +319,20 @@ listener_kve_init(char **kve, int kvn, listener_t *lp, ...)
 			     __FILE__, __LINE__);
 			abort();
 		}
-		kve[i++] = p;
+		if (p)
+			kve[i++] = p;
 	}
 	va_end(ap);
 	kve[i] = NULL;
 }
 
 static char *event_str[] = {
-	"startup",
-	"cleanup",
-	"exit",
-	"signal"
-};
+	[event_startup]   = "startup",
+	[event_cleanup]   = "cleanup",
+	[event_exit]      = "exit",
+	[event_signal]    = "signal",
+	[event_heartbeat] = "heartbeat",
+};	
 
 static event_t *
 event_match_type(action_t *act, enum event_type type)
@@ -356,18 +358,12 @@ event_match(action_t *act, enum event_type type, int code)
 
 static int runaction(listener_t *lp, action_t *act, event_t *evt);
 
-static int
+int
 listener_run_action(listener_t *lp, int a)
 {
 	action_t *act;
-	char *kve[KVE_MINSIZE+2];
 	int err = 0;
 	
-	listener_kve_init(kve, sizeof(kve)/sizeof(kve[0]),
-			  lp,
-			  "event", event_str[a],
-			  NULL);
-	
 	for (act = lp->act_head; act; act = act->next) {
 		event_t *evt = event_match_type(act, a);
 		if (evt && runaction(lp, act, evt))
@@ -481,17 +477,25 @@ listener_print_status(listener_t *lp)
 static int
 runaction(listener_t *lp, action_t *act, event_t *evt)
 {
-	char *kve[KVE_MINSIZE + 6];
+	char *kve[KVE_MINSIZE + 7];
 	char codebuf[64];
 	char statbuf[64];
-
+	char pidbuf[80], *pid;
+	
 	snprintf(statbuf, sizeof(statbuf), "%u", lp->progstat);
 	snprintf(codebuf, sizeof(codebuf), "%u", evt->code);
+	if (lp->status == stat_up) {
+		snprintf(pidbuf, sizeof(pidbuf), "%lu",
+			 (unsigned long)lp->pid);
+		pid = pidbuf;
+	} else
+		pid = NULL;
 	listener_kve_init(kve, sizeof(kve)/sizeof(kve[0]),
 			  lp,
 			  "event", event_str[evt->type],
 			  "status", statbuf,
 			  "exitcode", codebuf,
+			  "pid", pid,
 			  NULL);
 
 	if (progman_start(type_action, lp, &act->locus, act->options,
@@ -520,7 +524,7 @@ onexit_reaction(listener_t *lp)
 	}
 
 	for (act = lp->act_head; act; act = act->next) {
-		if (evt = event_match(act, type, code))
+		if ((evt = event_match(act, type, code)))
 			runaction(lp, act, evt);
 	}
 	if (lp->act_count == 0)
@@ -550,7 +554,7 @@ listener_proc_report()
 void
 listener_start(listener_t *lp, char **kve)
 {
-	progman_start(type_listener, lp, &lp->locus, lp->options, 0, 
-		      lp->prog, lp->env, kve);
+	progman_start(type_listener, lp, &lp->locus, lp->options,
+		      config.heartbeat, lp->prog, lp->env, kve);
 }
 
diff --git a/src/progman.c b/src/progman.c
index 781a764..0f48b37 100644
--- a/src/progman.c
+++ b/src/progman.c
@@ -30,7 +30,10 @@ struct process {              /* This describes a single running process */
 					 the slot is not used */
 	/* Timeout information */
 	time_t stop;                  /* stop-time for the process, if
-					 type == type_action */
+					 type == type_action;
+					 time of the next heartbeat, if
+					 type == type_listener
+				      */
 	/* Apart from the usual next and prev links, action processes are
 	   linked in a doubly-linked list ordered by their stop time.  The
 	   ts_prev member points to the process with earlier stop time, and
@@ -142,25 +145,46 @@ proc_ts_unlink(struct process *p)
 	else
 		ts_proc_list = p->ts_next;
 }
+
+static void
+proc_set_timeout(struct process *proc, unsigned timeout)
+{
+	proc->stop = time(NULL) + timeout;
+	proc_ts_unlink(proc);
+	proc_ts_link(proc);
+}
 
 /* Terminate the timed-out processes.  Return the number of seconds left
    to the expiration of the earliest available running action. */
 unsigned
-progman_expire()
+progman_expire(void)
 {
 	time_t t = time(NULL);
-	struct process *p;
+	struct process *p = ts_proc_list;
 
-	for (p = ts_proc_list; p && p->stop <= t; p = p->ts_next) {
-		debug(1, ("terminating timed out process %lu (%d,%lu)",
-			  (unsigned long) p->pid, p->type,
-			  (unsigned long) p->stop));
-		kill(p->pid, SIGKILL);
-		proc_ts_unlink(p);
-		if (p->redir_pid[REDIR_OUT])
-			kill(p->redir_pid[REDIR_OUT], SIGKILL);
-		if (p->redir_pid[REDIR_ERR])
-			kill(p->redir_pid[REDIR_ERR], SIGKILL);
+	while (p && p->stop <= t) {
+		struct process *next = p->ts_next;
+		
+		switch (p->type) {
+		case type_listener:
+			debug(1, ("%s: heartbeat for pid %lu",
+				  p->lp->id, (unsigned long) p->pid));
+			listener_run_action(p->lp, event_heartbeat);
+			proc_set_timeout(p, config.heartbeat);
+			break;
+		default:
+			debug(1, ("terminating timed out process %lu (%d,%lu)",
+				  (unsigned long) p->pid, p->type,
+				  (unsigned long) p->stop));
+			kill(p->pid, SIGKILL);
+			proc_ts_unlink(p);
+			if (p->redir_pid[REDIR_OUT])
+				kill(p->redir_pid[REDIR_OUT], SIGKILL);
+			if (p->redir_pid[REDIR_ERR])
+				kill(p->redir_pid[REDIR_ERR], SIGKILL);
+		}
+
+		p = next;
 	}
 	if (p)
 		return p->stop - t;
@@ -194,6 +218,10 @@ process_register(listener_t *lp, int type, unsigned timeout,
 		p = emalloc(sizeof(*p));
 	memset(p, 0, sizeof(*p));
 	p->lp = lp;
+	if (type == type_listener) {
+		lp->pid = pid;
+		lp->status = stat_up;
+	}
 	p->type = type;
 	p->pid = pid;
 	if (timeout == 0)
@@ -240,7 +268,7 @@ proc_kill(listener_t *lp)
 
 /* Collect terminated children. Safe to use in signal handlers. */
 void
-progman_cleanup()
+progman_cleanup(void)
 {
 	pid_t pid;
 	int status;
@@ -272,8 +300,10 @@ progman_cleanup()
 				kill(p->redir_pid[REDIR_OUT], SIGKILL);
 			if (p->redir_pid[REDIR_ERR])
 				kill(p->redir_pid[REDIR_ERR], SIGKILL);
-			if (--p->lp->act_count == 0)
-				p->lp->status = stat_down;
+			if (--p->lp->act_count == 0) {
+				if (p->lp->status == stat_onexit)
+					p->lp->status = stat_down;
+			}
 			break;
 		}
 
@@ -301,6 +331,7 @@ progman_terminate()
 	
 	start = time(NULL);
 	while (proc_list) {
+		progman_cleanup();
 		if (time(NULL) - start > config.shutdown_timeout) {
 			diag(LOG_NOTICE,
 			     "sending running processes the KILL signal");
@@ -436,7 +467,8 @@ open_redirector(const char *tag, int prio, pid_t *return_pid)
    maximum execution time (for type_action processes); 0 means no limit. */
 int
 progman_start(int type, listener_t *lp, struct grecs_locus *loc,
-	      int options, unsigned timeout, char *prog, char **env, char **kve)
+	      int options, unsigned timeout, char *prog, char **env,
+	      char **kve)
 {
 	pid_t pid;
 	int redir_fd[2] = { -1, -1 };
@@ -507,10 +539,6 @@ progman_start(int type, listener_t *lp, struct grecs_locus *loc,
 	if (pid) {
 		/* master */
 		process_register(lp, type, timeout, pid, redir_pid);
-		if (type == type_listener) {
-			lp->pid = pid;
-			lp->status = stat_up;
-		}
 		wordsplit_free(&ws);
 		return 0;
 	}