Implement snmp subagent

10 files changed, 1055 insertions, 12 deletions

diff --git a/configure.ac b/configure.ac
index 04d12a4..89029d5 100644
--- a/configure.ac
+++ b/configure.ac
@@ -10,6 +10,12 @@ AM_INIT_AUTOMAKE([1.11 foreign])
 # Checks for programs.
 AC_PROG_CC
 AC_PROG_RANLIB
+# Check for Net-SNMP
+AC_PATH_PROG([NET_SNMP_CONFIG], net-snmp-config, none, $PATH)
+if test "$NET_SNMP_CONFIG" = "none"; then
+  AC_MSG_ERROR([cannot find Net-SNMP (net-snmp-config not found)])
+fi
+AC_SUBST(NET_SNMP_CONFIG)
 
 # Checks for libraries.
 save_LIBS=$LIBS
@@ -38,5 +44,15 @@ RUNCAP_SETUP
 
 AM_CONDITIONAL([TMD_WRAP],[test x$ac_cv_lib_wrap_main = xyes && test x$ac_cv_header_tcpd_h = xyes])
 
+# Directories
+AC_SUBST([MIBDIR],['$(datarootdir)/snmp/mibs'])
+AC_ARG_WITH([mibdir],
+ [AC_HELP_STRING([--with-mibdir=DIR],
+                 [installation directory for MIB files])],
+ [case $withval in
+  /*) MIBDIR=$withval;;
+  *)  AC_MSG_ERROR([argument to --with-mibdir must be absolute pathname])
+  esac])
+
 AC_OUTPUT([Makefile
            src/Makefile])
diff --git a/src/.gitignore b/src/.gitignore
index b144557..eaf59e4 100644
--- a/src/.gitignore
+++ b/src/.gitignore
@@ -1,2 +1,3 @@
 stevedore
 tallyman
+tallyman_mib.[ch]
diff --git a/src/Makefile.am b/src/Makefile.am
index f2b9f9d..e3f5530 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -16,14 +16,30 @@ stevedore_SOURCES = \
  pidfile.c\
  runas.c\
  remoteip.c\
- servdb.c
+ servdb.c\
+ tallyman_mib.c\
+ tallyman_mib.h\
+ subagent.c
+
+
+NET_SNMP_LIBS = `$(NET_SNMP_CONFIG) --libs`
+NET_SNMP_EXLIBS = `$(NET_SNMP_CONFIG) --external-libs`
+NET_SNMP_AGENTLIBS = `$(NET_SNMP_CONFIG) --agent-libs`
+NET_SNMP_MIBDIRS = `net-snmp-config --mibdirs`
+NET_SNMP_MIBS = `net-snmp-config --default-mibs`
 
 AM_CPPFLAGS=@GRECS_INCLUDES@ @RUNCAP_INC@ -DSYSCONFDIR="\"$(sysconfdir)\""
+stevedore_LDADD=\
+ @GRECS_LDADD@\
+ @DAEMON_LIBS@
+stevedore_LDFLAGS=\
+ $(NET_SNMP_LIBS)\
+ $(NET_SNMP_EXLIBS)\
+ $(NET_SNMP_AGENTLIBS)
+
 tallyman_LDADD=@GRECS_LDADD@ @RUNCAP_LDADD@
-stevedore_LDADD=@GRECS_LDADD@ @DAEMON_LIBS@
 tallyman_LDFLAGS=-static
 
-
 noinst_HEADERS = defs.h
 
 if TMD_WRAP
@@ -31,3 +47,18 @@ if TMD_WRAP
   AM_CPPFLAGS += -DHAVE_LIBWRAP
 endif
 
+BUILT_SOURCES = \
+ tallyman_mib.c\
+ tallyman_mib.h
+
+tallyman_mib.c tallyman_mib.h: tallyman_mib.mib2c TALLYMAN-MIB.txt
+
+.mib2c.c:
+	MIBDIRS=${top_srcdir}/src:${NET_SNMP_MIBDIRS} \
+	 MIBS="+TALLYMAN-MIB" \
+	  mib2c -c $< -f $@ tallyman
+
+mibdir=@MIBDIR@
+mib_DATA = TALLYMAN-MIB.txt
+
+EXTRA_DIST = TALLYMAN-MIB.txt tallyman_mib.mib2c
diff --git a/src/TALLYMAN-MIB.txt b/src/TALLYMAN-MIB.txt
new file mode 100644
index 0000000..7f06b73
--- /dev/null
+++ b/src/TALLYMAN-MIB.txt
@@ -0,0 +1,211 @@
+TALLYMAN-MIB DEFINITIONS ::= BEGIN
+
+-- *************************************************************
+--
+-- Docker service MIBS
+--
+-- *************************************************************
+
+IMPORTS
+	MODULE-IDENTITY, OBJECT-TYPE, enterprises, Integer32, Counter64,
+		Gauge32
+		FROM SNMPv2-SMI
+	TEXTUAL-CONVENTION, TimeStamp, DateAndTime
+		FROM SNMPv2-TC
+	OBJECT-GROUP, MODULE-COMPLIANCE
+	        FROM SNMPv2-CONF;
+
+tallyman MODULE-IDENTITY
+        LAST-UPDATED "201806052216Z"
+	ORGANIZATION "Gray Software"
+	CONTACT-INFO "Sergey Poznyakoff <gray@gnu.org>"
+	DESCRIPTION
+	        "This MIB module defines objects for Docker service statistics."
+	REVISION "201806052216Z"
+	DESCRIPTION
+	        "First revision."
+	::= { enterprises 9163 103 }
+
+services      OBJECT IDENTIFIER       ::= { tallyman 1 }
+
+servicesUpTime	OBJECT-TYPE
+        SYNTAX TimeStamp
+	MAX-ACCESS read-only
+	STATUS current
+	DESCRIPTION
+	        "Total uptime of the Stevedore server."
+     	::= { services 1 }
+
+servicesTotal	OBJECT-TYPE
+        SYNTAX Gauge32
+	MAX-ACCESS read-only
+	STATUS current
+	DESCRIPTION
+	        "Number of configured services."
+     	::= { services 2 }
+
+servicesRunning	OBJECT-TYPE
+        SYNTAX Gauge32
+	MAX-ACCESS read-only
+	STATUS current
+	DESCRIPTION
+	        "Number of running services."
+     	::= { services 3 }
+
+ServiceNameString ::= TEXTUAL-CONVENTION
+	DISPLAY-HINT "128t"
+	STATUS current
+	DESCRIPTION "A string containing service name."
+        SYNTAX OCTET STRING (SIZE (0..128))
+
+InstanceNameString  ::= TEXTUAL-CONVENTION
+	DISPLAY-HINT "256t"
+	STATUS current
+	DESCRIPTION "A string containing service instance name."
+        SYNTAX OCTET STRING (SIZE (0..256))
+
+ServiceEntry ::=	     SEQUENCE {
+	serviceIndex		Integer32,
+	serviceName		ServiceNameString,
+	serviceInstances	Gauge32
+}	
+
+serviceTable   	     OBJECT-TYPE
+        SYNTAX  SEQUENCE OF ServiceEntry
+	MAX-ACCESS not-accessible
+	STATUS  current
+	DESCRIPTION
+	        "Service table."
+	::= { services 4 }
+
+serviceEntry          OBJECT-TYPE
+        SYNTAX	   ServiceEntry
+	MAX-ACCESS not-accessible
+	STATUS	   current
+	DESCRIPTION
+	        "An entry (conceptual row) describing a service."
+	INDEX { serviceIndex }
+	::= { serviceTable 1 }
+
+serviceIndex	        OBJECT-TYPE
+        SYNTAX  Integer32 (0..65535)
+	MAX-ACCESS not-accessible
+	STATUS current
+	DESCRIPTION
+	        "A number uniquely identifying each service."
+	::= { serviceEntry 1 }
+
+serviceName		OBJECT-TYPE
+        SYNTAX ServiceNameString
+	MAX-ACCESS read-only
+	STATUS current
+	DESCRIPTION
+	        "The name of the service."
+     	::= { serviceEntry 2 }
+
+serviceInstances	OBJECT-TYPE
+        SYNTAX Gauge32
+	MAX-ACCESS read-only
+	STATUS current
+	DESCRIPTION
+	        "Number of running instances in this service."
+     	::= { serviceEntry 3 }
+
+InstanceState ::= TEXTUAL-CONVENTION
+    STATUS       current
+    DESCRIPTION
+            "Represents a state of the instance."
+    SYNTAX       INTEGER { stopped(0), running(1), expired(2), error(3) }
+
+InstanceErrorMessage ::= TEXTUAL-CONVENTION
+    DISPLAY-HINT "1024t"
+    STATUS       current
+    DESCRIPTION "Error text associated with an instance."
+    SYNTAX       OCTET STRING (SIZE (0..1024))
+
+InstanceEntry ::=	     SEQUENCE {
+	instanceIndex		Integer32,
+	instanceName		InstanceNameString,
+	instanceService         ServiceNameString,
+	instanceState		InstanceState,
+	instanceTimeStamp	DateAndTime,
+	instanceErrorMessage	InstanceErrorMessage
+}	
+
+instanceTable   	     OBJECT-TYPE
+        SYNTAX  SEQUENCE OF InstanceEntry
+	MAX-ACCESS not-accessible
+	STATUS  current
+	DESCRIPTION
+	        "Instance table."
+	::= { services 5 }
+
+instanceEntry          OBJECT-TYPE
+        SYNTAX	   InstanceEntry
+	MAX-ACCESS not-accessible
+	STATUS	   current
+	DESCRIPTION
+	        "An entry (conceptual row) describing an instance."
+	INDEX { instanceIndex }
+	::= { instanceTable 1 }
+
+instanceIndex	        OBJECT-TYPE
+        SYNTAX  Integer32 (0..65535)
+	MAX-ACCESS not-accessible
+	STATUS current
+	DESCRIPTION
+	        "A number uniquely identifying each instance."
+	::= { instanceEntry 1 }
+
+instanceName		OBJECT-TYPE
+        SYNTAX InstanceNameString
+	MAX-ACCESS read-only
+	STATUS current
+	DESCRIPTION
+	        "The name of the instance."
+     	::= { instanceEntry 2 }
+
+instanceService		OBJECT-TYPE
+        SYNTAX ServiceNameString
+	MAX-ACCESS read-only
+	STATUS current
+	DESCRIPTION
+	        "The name of the instance."
+     	::= { instanceEntry 3 }
+
+instanceState	OBJECT-TYPE
+	SYNTAX InstanceState
+	MAX-ACCESS read-only
+	STATUS current
+	DESCRIPTION
+		"State of the instance"
+	::= { instanceEntry 4 }
+
+instanceTimeStamp	OBJECT-TYPE
+	SYNTAX	DateAndTime
+	MAX-ACCESS read-only
+	STATUS current
+	DESCRIPTION
+		"Time of the last successful probe"
+	::= { instanceEntry 5 }
+
+instanceErrorMessage	OBJECT-TYPE
+	SYNTAX InstanceErrorMessage
+	MAX-ACCESS read-only
+	STATUS current
+	DESCRIPTION
+		"If instanceState is error(3), error message
+	         associated with this instance"
+	::= { instanceEntry 6 }
+	
+
+END
+
+-- Local variables:
+-- eval: (add-hook 'write-file-hooks 'time-stamp)
+-- time-stamp-start: "\\(LAST-UPDATED\\|REVISION\\)  *\""
+-- time-stamp-end: "\""
+-- time-stamp-format: "%:y%02m%02d%02H%02MZ"
+-- time-stamp-line-limit: 32
+-- time-stamp-count: 2
+-- end:
diff --git a/src/config.c b/src/config.c
index 92ec3b5..77549c5 100644
--- a/src/config.c
+++ b/src/config.c
@@ -183,8 +183,12 @@ static struct grecs_keyword tallymand_kw[] = {
 	  grecs_type_string, GRECS_DFLT, &runas_group },
 	{ "syslog", NULL, "Configure syslog logging",
 	  grecs_type_section, GRECS_DFLT, NULL, 0, NULL, NULL, syslog_kw },
-	{ "service", "NAME", "Define service",
+	{ "service", "name", "Define service",
 	  grecs_type_string, GRECS_DFLT, NULL, 0, cb_service },
+	{ "instance-state-ttl", "seconds", "Sets instance state TTL",
+	  grecs_type_uint, GRECS_DFLT, &ttl_instance_state },
+	{ "snmp-table-cache-ttl", "seconds", "Sets SNMP cache TTL",
+	  grecs_type_uint, GRECS_DFLT, &ttl_snmp_table_cache },
 	{ NULL }
 };
 
diff --git a/src/servdb.c b/src/servdb.c
index 4df1157..8517f92 100644
--- a/src/servdb.c
+++ b/src/servdb.c
@@ -3,21 +3,31 @@
 #include <time.h>
 #include <pthread.h>
 #include "stevedore.h"
+#include "tallyman_mib.h"
 
 #define MAX_SERVICE 1024
 
+unsigned ttl_snmp_table_cache = 300;
+unsigned ttl_instance_state = 600;
+
 struct service {
 	char *id;
 	struct instance *head, *tail;
 	pthread_mutex_t mutex;
 };
 
+enum instance_state {
+	instance_state_stopped,
+	instance_state_running,
+	instance_state_expired,
+	instance_state_error
+};
+
 struct instance {
 	char *hostname;
 	struct service *srv;
-	int status;
+	enum instance_state state;
 	time_t timestamp;
-	int error;
 	char *error_message;
 	struct instance *prev, *next;
 	pthread_cond_t notbusy;     /* Prevent simultaneous updates */
@@ -153,6 +163,12 @@ instance_unlink(struct instance *inst)
 	inst->next = inst->prev = NULL;
 }
 
+static inline int
+instance_is_running(struct instance *inst)
+{
+	return inst->state == instance_state_running;
+}
+	
 static struct instance *
 instance_by_hostname(char const *hostname, struct service *srv)
 {
@@ -212,17 +228,267 @@ servdb_update(struct json_value *obj)
 	} else
 		instance_lock(inst);
 	
-	inst->status = status;
 	time(&inst->timestamp);
 
 	val = json_value_lookup_typed(obj, "error", json_bool);
-	inst->error = val && val->v.b;
-
-	if (inst->error) {
+	if (val && val->v.b) {
+		inst->state = instance_state_error;
 		val = json_value_lookup_typed(obj, "message", json_string);
-		inst->error_message = val ? val->v.s : NULL;
+		inst->error_message = val ? strdup(val->v.s) : NULL;
+	} else {
+		inst->state = status
+			       ? instance_state_running
+			       : instance_state_stopped;
 	}
 	instance_unlock(inst);
 	pthread_mutex_unlock(&srv->mutex);
 	return 0;
 }
+
+size_t
+servdb_servicesTotal(void)
+{
+	return service_count;
+}
+
+size_t
+servdb_servicesRunning(void)
+{
+	size_t i, result = 0;
+	for (i = 0; i < service_count; i++)
+		if (servdb_serviceInstances(i))
+			result++;
+	return result;
+}
+
+ssize_t
+servdb_serviceInstances(size_t idx)
+{
+	struct instance *inst;
+	ssize_t result = 0;
+	
+	if (idx > service_count)
+		return -1;
+	pthread_mutex_lock(&service[service_count].mutex);
+	for (inst = service[idx].head; inst; inst = inst->next) {
+		if (instance_is_running(inst)) {
+			result++;
+		}
+	}
+	pthread_mutex_unlock(&service[service_count].mutex);
+	return result;
+}
+
+static int
+service_Table_load_row(size_t idx, netsnmp_tdata *table)
+{
+	netsnmp_tdata_row *data_row;
+	struct serviceTable_entry *ent;
+	struct service *srv = &service[idx];
+		
+	ent = SNMP_MALLOC_TYPEDEF(struct serviceTable_entry);
+	if (!ent)
+		return SNMP_ERR_GENERR;
+
+	data_row = netsnmp_tdata_create_row();
+	if (!data_row) {
+		SNMP_FREE(ent);
+		return SNMP_ERR_GENERR;
+	}
+	data_row->data = ent;
+
+	ent->serviceIndex = idx;
+	ent->serviceName = srv->id;
+	ent->serviceName_len = strlen(srv->id);
+	ent->serviceInstances = servdb_serviceInstances(idx);
+
+	netsnmp_tdata_row_add_index(data_row, ASN_INTEGER,
+				    &ent->serviceIndex,
+				    sizeof(ent->serviceIndex));
+	netsnmp_tdata_add_row(table, data_row);
+
+	return SNMP_ERR_NOERROR;
+}
+	
+int
+serviceTable_load(netsnmp_cache *cache, void *vmagic)
+{
+	int rc;
+	size_t i;
+	netsnmp_tdata *table = (netsnmp_tdata *) vmagic;
+	for (i = 0; i < service_count; i++) {
+		if ((rc = service_Table_load_row(i, table)) != SNMP_ERR_NOERROR)
+			return rc;
+	}
+	return 0;
+}
+
+void
+serviceTable_entry_free(void *data)
+{
+	/* nothing */
+}
+
+#define TMSEC(t) (((t)->tm_hour * 60 + (t)->tm_min) * 60 + (t)->tm_sec)
+
+static int
+utc_offset (void)
+{
+	time_t t = time (NULL);
+	struct tm ltm = *localtime (&t);
+	struct tm gtm = *gmtime (&t);
+	int d = TMSEC (&ltm) - TMSEC (&gtm);
+	if (!(ltm.tm_year = gtm.tm_year && ltm.tm_mon == gtm.tm_mon))
+		d += 86400;
+	return d / 60;
+}
+
+static int
+setDateAndTime(time_t t, char **pbuf, size_t *plen)
+{
+	char *buf;
+	size_t len = 11;
+	struct tm *tm;
+	unsigned n;
+	
+	buf = malloc(len + 1);
+	if (!buf) 
+		return -1;
+	
+	tm = localtime(&t);
+	/*    A date-time specification.
+          
+	      field  octets  contents                  range
+	      -----  ------  --------                  -----
+	        1      1-2   year*                     0..65536
+		2       3    month                     1..12
+		3       4    day                       1..31
+		4       5    hour                      0..23
+		5       6    minutes                   0..59
+		6       7    seconds                   0..60
+		(use 60 for leap-second)
+		7       8    deci-seconds              0..9
+		8       9    direction from UTC        '+' / '-'
+		9      10    hours from UTC*           0..13
+		10      11    minutes from UTC          0..59
+		
+		* Notes:
+		- the value of year is in network-byte order
+	*/
+	n = tm->tm_year % 100;
+	buf[0] = n >> 8;
+	buf[1] = n & 0xff;
+	buf[2] = tm->tm_mon + 1;
+	buf[3] = tm->tm_mday;
+	buf[4] = tm->tm_hour; 
+	buf[5] = tm->tm_min;
+	buf[6] = tm->tm_sec;
+	buf[7] = '0';
+	n = utc_offset();
+	if (n < 0) {
+		buf[8] = '-';
+		n = - n;
+	} else 
+		buf[8] = '+';
+	buf[9] = n / 60;
+	buf[10] = n % 60;
+
+	*pbuf = buf;
+	*plen = len;
+
+	return 0;
+}
+
+static int
+instanceTable_load_row(size_t idx,
+		       struct instance *inst, char const *service_id,
+		       netsnmp_tdata *table)
+{
+	netsnmp_tdata_row *data_row;
+	struct instanceTable_entry *ent;
+		
+	ent = SNMP_MALLOC_TYPEDEF(struct instanceTable_entry);
+	if (!ent)
+		return SNMP_ERR_GENERR;
+
+	data_row = netsnmp_tdata_create_row();
+	if (!data_row) {
+		SNMP_FREE(ent);
+		return SNMP_ERR_GENERR;
+	}
+	data_row->data = ent;
+
+	ent->instanceIndex = idx;
+	ent->instanceName = inst->hostname;
+	ent->instanceName_len = strlen(inst->hostname);
+	ent->instanceService = (char*) service_id;
+	ent->instanceService_len = strlen(service_id);
+	ent->instanceState = inst->state;
+	setDateAndTime(inst->timestamp, &ent->instanceTimeStamp,
+		       &ent->instanceTimeStamp_len);
+	ent->instanceErrorMessage = inst->error_message
+		                       ? inst->error_message
+		                       : "";
+	ent->instanceErrorMessage_len = strlen(ent->instanceErrorMessage);
+		
+	netsnmp_tdata_row_add_index(data_row, ASN_INTEGER,
+				    &ent->instanceIndex,
+				    sizeof(ent->instanceIndex));
+	netsnmp_tdata_add_row(table, data_row);
+
+	return SNMP_ERR_NOERROR;
+}
+
+int
+instanceTable_load(netsnmp_cache *cache, void *vmagic)
+{
+	size_t i, idx;
+	netsnmp_tdata *table = (netsnmp_tdata *) vmagic;
+	int rc = SNMP_ERR_NOERROR;
+	
+	idx = 0;
+	for (i = 0; i < service_count && rc == SNMP_ERR_NOERROR; i++) {
+		struct instance *inst;
+		
+		pthread_mutex_lock(&service[i].mutex);
+		for (inst = service[i].head; inst; inst = inst->next) {
+			instance_lock(inst);
+			rc = instanceTable_load_row(idx, inst,
+						    service[i].id, table);
+			instance_unlock(inst);
+			if (rc != SNMP_ERR_NOERROR)
+				break;
+			idx++;
+		}
+		pthread_mutex_unlock(&service[i].mutex);
+	}
+	return rc;
+}
+
+void
+instanceTable_entry_free(void *data)
+{
+	/* nothing */
+}
+
+void *
+invalidator_thread(void *p)
+{
+	size_t i;
+	struct instance *inst;
+	while (1) {
+		time_t now = time(NULL);
+		for (i = 0; i < service_count; i++) {
+			pthread_mutex_lock(&service[i].mutex);
+			for (inst = service[i].head; inst; inst = inst->next) {
+				instance_lock(inst);
+				if (inst->state != instance_state_expired
+				    && now - inst->timestamp > ttl_instance_state)
+					inst->state = instance_state_expired;
+				instance_unlock(inst);
+			}
+			pthread_mutex_unlock(&service[i].mutex);
+		}
+		sleep(ttl_instance_state);
+	}
+}
diff --git a/src/stevedore.c b/src/stevedore.c
index 974a095..a8ffc2b 100644
--- a/src/stevedore.c
+++ b/src/stevedore.c
@@ -16,6 +16,7 @@
 #include <syslog.h>
 #include <signal.h>
 #include <time.h>
+#include <pthread.h>
 #define MHD_PLATFORM_H
 #include <microhttpd.h>
 #include "defs.h"
@@ -43,7 +44,16 @@ static char shortopts[] = "?df:Fs";
 void
 help(void)
 {
-	printf("NO HELP SO FAR\n");
+	printf("usage: %s [OPTIONS]\n", progname);
+	printf("\n");
+        printf("OPTIONS are:\n\n");
+	printf("   -f, --config-file=FILE   read configuration from FILE\n");
+	printf("                            (default %s)\n", DEFAULT_CONFIG_FILE_NAME);
+	printf("       --config-help        describe configuration file syntax and variables\n");
+	printf("   -F, --foreground         don't fork; remain in foreground\n");
+	printf("   -s, --single             don't start sentinel process\n");
+	printf("   -d, --debug              increase debug verbosity\n");
+	printf("   -?, --help               display this help text\n\n");
 }
 
 static void
@@ -365,6 +375,7 @@ http_server(int fd, struct sockaddr *server_addr)
 	struct MHD_Daemon *mhd;
 	sigset_t sigs;
 	int i;
+	pthread_t tid;
 		
 	/* Block the 'fatal signals' and SIGPIPE in the handling thread */
 	sigemptyset(&sigs);
@@ -386,6 +397,10 @@ http_server(int fd, struct sockaddr *server_addr)
 	/* Unblock only the fatal signals */
 	sigdelset(&sigs, SIGPIPE);
 	pthread_sigmask(SIG_UNBLOCK, &sigs, NULL);
+	/* Start AgentX thread */
+	pthread_create(&tid, NULL, agentx_thread, NULL);
+	/* Start invalidating thread */
+	pthread_create(&tid, NULL, invalidator_thread, NULL);
 	/* Wait for signal to arrive */
 	sigwait(&sigs, &i);
 	MHD_stop_daemon(mhd);
@@ -567,6 +582,8 @@ main(int argc, char **argv)
 			close(i);
 		syslog_enable();
 	}
+	agentx_init();
+	
 	pidfile_create();
 
 	if (single_process)
diff --git a/src/stevedore.h b/src/stevedore.h
index 33c8001..921eb05 100644
--- a/src/stevedore.h
+++ b/src/stevedore.h
@@ -9,6 +9,8 @@ extern char *pidfile;
 extern char *progname;
 extern char *runas_user;
 extern char *runas_group;
+extern unsigned ttl_snmp_table_cache;
+extern unsigned ttl_instance_state;
 
 void config_help(void);
 void readconfig(char const *file);
@@ -27,3 +29,14 @@ char *get_remote_ip(struct MHD_Connection *conn);
 
 int service_add(char const *id, grecs_locus_t *loc);
 int servdb_update(struct json_value *obj);
+
+u_long servdb_servicesUpTime(void);
+size_t servdb_servicesTotal(void);
+size_t servdb_servicesRunning(void);
+ssize_t servdb_serviceInstances(size_t idx);
+
+void agentx_init(void);
+void *agentx_thread(void *p);
+void *invalidator_thread(void *p);
+
+
diff --git a/src/subagent.c b/src/subagent.c
new file mode 100644
index 0000000..202f00c
--- /dev/null
+++ b/src/subagent.c
@@ -0,0 +1,39 @@
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+#include <pthread.h>
+#include "stevedore.h"
+#include "tallyman_mib.h"
+#include <signal.h>
+
+void
+agentx_init(void)
+{
+	if (grecs_log_to_stderr)
+		snmp_enable_calllog();
+	else
+		snmp_enable_stderrlog();
+	netsnmp_ds_set_boolean(NETSNMP_DS_APPLICATION_ID,
+			       NETSNMP_DS_AGENT_ROLE, 1);
+	SOCK_STARTUP;
+	init_agent("stevedore");
+	init_tallyman_mib();
+	init_snmp("stevedore");
+}
+
+time_t start_time;
+
+u_long
+servdb_servicesUpTime(void)
+{
+	return time(NULL) - start_time;
+}
+
+void *
+agentx_thread(void *p)
+{
+	start_time = time(NULL);
+	while (1) {
+		agent_check_and_process(1);
+	}
+}
diff --git a/src/tallyman_mib.mib2c b/src/tallyman_mib.mib2c
new file mode 100644
index 0000000..0972263
--- /dev/null
+++ b/src/tallyman_mib.mib2c
@@ -0,0 +1,445 @@
+# This file is part of Tallyman -*- c -*-
+# Copyright (C) 2018 Sergey Poznyakoff
+#
+# Tallyman is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3, or (at your option)
+# any later version.
+#
+# Tallyman is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Tallyman.  If not, see <http://www.gnu.org/licenses/>.
+
+@define ROCOM@
+/* Local variables:
+   buffer-read-only: t
+   End:
+   vi: set ro:
+*/
+@enddefine@
+@startperl@
+$vars{'tallyman_translate_table'} = {
+	servicesUpTime => qw{servdb_servicesUpTime()},
+	servicesTotal => qw{servdb_servicesTotal()},
+	servicesRunning => qw{servdb_servicesRunning()}
+};
+
+$vars{'tallyman_translate'} = sub {
+    my ($name) = @_;
+    my $r = $vars{'tallyman_translate_table'}->{$name};
+    if (!defined($r)) {
+	print STDERR "no translation for $name!\n";
+	exit(1);
+    }
+    $vars{'tallyman_get'}  = $r;
+    return 0;
+};	
+	
+$vars{'modulename'} = $vars{'name'};
+$vars{'modulename'} =~ s#.*/##;
+$vars{'modulename'} =~ s/\.c$//;
+
+0;
+@endperl@
+@open ${modulename}.h@
+/* THIS FILE IS GENERATED AUTOMATICALLY.  PLEASE DO NOT EDIT. */
+
+extern unsigned ttl_snmp_table_cache;
+
+#include <net-snmp/net-snmp-config.h>
+#include <net-snmp/net-snmp-includes.h>
+#include <net-snmp/agent/net-snmp-agent-includes.h>
+
+void init_tallyman_mib(void);
+void deinit_$modulename(void);
+
+@open ${name}@
+/* THIS FILE IS GENERATED AUTOMATICALLY.  PLEASE DO NOT EDIT. */
+
+#include "stevedore.h"
+#include "tallyman_mib.h"
+
+/* Variable handlers.
+
+   An instance handler only hands us one request at a time, unwrapping
+   any eventual GETNEXT requests.  
+*/
+
+@foreach $i scalar@
+  @startperl@
+    &{$vars{'tallyman_translate'}}($vars{'i'});
+  @endperl@
+
+static int
+handle_$i(netsnmp_mib_handler *handler,
+                          netsnmp_handler_registration *reginfo,
+                          netsnmp_agent_request_info   *reqinfo,
+                          netsnmp_request_info         *requests)
+{
+  @if $i.settable@
+    int ret;
+  @end@
+  @if $i.type eq 'ASN_OCTET_STR'@
+    @eval $tallyman_type = q{char const *};@
+  @elsif $i.type eq 'ASN_COUNTER64'@
+    @eval $tallyman_type = q{uint64_t};@
+  @elsif $i.type eq 'ASN_COUNTER'@
+    @eval $tallyman_type = q{uint32_t};@
+  @elsif $i.type eq 'ASN_INTEGER'@
+    @eval $tallyman_type = q{uint32_t};@
+  @elsif $i.type eq 'ASN_GAUGE'@
+    @eval $tallyman_type = q{uint32_t};@
+  @elsif $i.type eq 'ASN_TIMETICKS'@
+    @eval $tallyman_type = q{uint32_t};@
+  @end@
+    $tallyman_type val = $tallyman_get;
+
+    switch(reqinfo->mode) {
+    case MODE_GET:
+  @if $i.type eq 'ASN_COUNTER64'@
+    {
+        struct counter64 ctr;
+	ctr.high = val >> 32;
+	ctr.low = val & 0xffffffff;
+	snmp_set_var_typed_value(requests->requestvb, $i.type,
+				 &ctr,
+				 sizeof(ctr));
+    }				 
+  @elsif $i.type eq 'ASN_OCTET_STR'@
+    {
+       snmp_set_var_typed_value(requests->requestvb, $i.type,
+ 	                         val, strlen(val));
+    }				
+  @elsif $i.type eq 'ASN_COUNTER'@
+    {
+	snmp_set_var_typed_value(requests->requestvb, $i.type,
+				 &val,
+				 sizeof(val));
+    }
+  @elsif $i.type eq 'ASN_INTEGER'@
+    {
+	snmp_set_var_typed_value(requests->requestvb, $i.type,
+				 &val,
+				 sizeof(val));
+    }
+  @elsif $i.type eq 'ASN_GAUGE'@
+    {
+	snmp_set_var_typed_value(requests->requestvb, $i.type,
+				 &val,
+				 sizeof(val));
+    }
+  @elsif $i.type eq 'ASN_TIMETICKS'@
+    {
+	snmp_set_var_typed_value(requests->requestvb, $i.type,
+				 &val,
+				 sizeof(val));
+    }
+  @else@
+    @printf "#error \"unrecognized type %s for %s\"" $i.type $i@
+  @end@
+        break;
+
+  @if $i.settable@
+    /*
+     * SET REQUEST
+     *
+     * multiple states in the transaction.  See:
+     * http://www.net-snmp.org/tutorial-5/toolkit/mib_module/set-actions.jpg
+     */
+    case MODE_SET_RESERVE1:
+	/* or you could use netsnmp_check_vb_type_and_size instead */
+	ret = netsnmp_check_vb_type(requests->requestvb, $i.type);
+        if (ret != SNMP_ERR_NOERROR)
+	    netsnmp_set_request_error(reqinfo, requests, ret);
+	break;
+
+    case MODE_SET_RESERVE2:
+    @if $tallyman_set_reserve2 ne ''@
+        if ($tallyman_set_reserve2 (reqinfo, requests, vd)) {
+	    netsnmp_set_request_error(reqinfo, requests,
+				      SNMP_ERR_RESOURCEUNAVAILABLE);
+	}
+    @end@
+	break;
+
+    case MODE_SET_FREE:
+    @if $tallyman_set_free ne ''@
+        # Free resources allocated in RESERVE1 and/or
+	# RESERVE2.  Something failed somewhere, and the states
+	# below won't be called.
+        $tallyman_set_free(reqinfo, requests, vd);
+    @end@
+	break;
+
+    case MODE_SET_ACTION:
+    @if $tallyman_set_action ne ''@
+        ret = $tallyman_set_action(reqinfo, requests, vd);
+        if (ret != SNMP_ERR_NOERROR)
+	    netsnmp_set_request_error(reqinfo, requests, ret);
+    @end@
+	break;
+
+    case MODE_SET_COMMIT:
+    @if $tallyman_set_commit ne ''@
+	# delete temporary storage
+        if ($tallyman_set_commit(reqinfo, requests, vd))
+	    netsnmp_set_request_error(reqinfo, requests, SNMP_ERR_COMMITFAILED);
+    @end@
+	break;
+
+    case MODE_SET_UNDO:
+    @if $tallyman_set_undo ne ''@
+	# UNDO and return to previous value for the object
+        if ($tallyman_set_undo(reqinfo, requests, vd))
+	    netsnmp_set_request_error(reqinfo, requests, SNMP_ERR_UNDOFAILED);
+    @end@
+	break;
+  @end@
+
+    default:
+	/* we should never get here, so this is a really bad error */
+	snmp_log(LOG_ERR, "unknown mode (%d) in handle_${i}\n", reqinfo->mode );
+	return SNMP_ERR_GENERR;
+    }
+    
+    return SNMP_ERR_NOERROR;
+}
+@end@
+
+@foreach $i table@
+  ## Determine the first/last column names
+  @eval $first_column = "-"@
+  @eval $last_column = "-"@
+  @foreach $c column@
+    @if $c.readable@
+      @if "$first_column" eq "-"@
+        @eval $first_column = $c@
+      @end@
+      @eval $last_column = $c@
+    @end@
+  @end@
+
+@push@
+@append ${modulename}.h@
+/* column number definitions for table $i */
+  @foreach $c column@
+#define COLUMN_$c.uc		$c.subid
+  @end@
+
+struct ${i}_entry {
+    /* Index values */
+    @foreach $idx index@
+     @if $idx.needlength@
+      @startperl@
+      &{$vars{'set_field_dim'}}($vars{'idx'});
+      @endperl@
+      @if $field_dim > 0@
+    $idx.decl $idx[$field_dim];
+      @else@
+    $idx.decl *$idx;
+      @end@
+    size_t ${idx}_len;
+     @else@
+    $idx.decl $idx;
+     @end@
+    @end@
+
+    /* Column values */
+    @foreach $c nonindex@
+     @if $c.readable@
+      @if $c.needlength@
+      @startperl@
+      &{$vars{'set_field_dim'}}($vars{'c'});
+      @endperl@
+      @if $field_dim > 0@
+    $c.decl ${c}[$field_dim];
+      @else@
+    $c.decl *$c;
+      @end@
+    size_t ${c}_len;
+      @else@
+    $c.decl $c;
+      @end@
+     @end@
+    @end@
+};
+
+int ${i}_load(netsnmp_cache *cache, void *vmagic);
+void ${i}_entry_free(void *data);
+@pop@
+
+static void
+${i}_free(netsnmp_cache *cache, void *vmagic)
+{
+	netsnmp_tdata  *table = (netsnmp_tdata *) vmagic;
+	netsnmp_tdata_row *row;
+
+	while ((row = netsnmp_tdata_row_first(table))) {
+		${i}_entry_free(row->data);
+		SNMP_FREE(row->data);
+		netsnmp_tdata_remove_and_delete_row(table, row);
+	}
+}
+
+/** handles requests for the $i table */
+static int
+handle_table_${i}(
+    netsnmp_mib_handler               *handler,
+    netsnmp_handler_registration      *reginfo,
+    netsnmp_agent_request_info        *reqinfo,
+    netsnmp_request_info              *requests)
+{
+    netsnmp_request_info       *request;
+    netsnmp_table_request_info *table_info;
+    struct ${i}_entry          *table_entry;
+
+    switch (reqinfo->mode) {
+    case MODE_GET:
+        for (request = requests; request; request = request->next) {
+            table_entry = (struct ${i}_entry *)
+                           netsnmp_tdata_extract_entry(request);
+            table_info  =  netsnmp_extract_table_info(request);
+    
+            switch (table_info->colnum) {
+            @foreach $c column@
+              @if $c.readable@
+            case COLUMN_$c.uc:
+                if (!table_entry) {
+                    netsnmp_set_request_error(reqinfo, request,
+                                              SNMP_NOSUCHINSTANCE);
+                    continue;
+                }
+                @if $c.needlength@
+                snmp_set_var_typed_value(request->requestvb, $c.type,
+                                         table_entry->$c,
+                                         table_entry->${c}_len);
+                @elsif $c.type eq 'ASN_COUNTER64'@@
+	        snmp_set_var_typed_value(requests->requestvb, $c.type,
+				         &table_entry->$c,
+				         sizeof(table_entry->$c));
+                @else@
+                snmp_set_var_typed_integer(request->requestvb, $c.type,
+                                           table_entry->$c);
+                @end@
+                break;
+              @end@