Rewrite using opaque record data.

Records can carry arbitrary data, whose format is defined using
teplate strings similar to those of Perl's pack() function.  The
teplate string is stored in the logfile header and is used by
binlogcat to render a human-readable representation.

* src/.gitignore: New file.
* pack.c: New file.
* pack.h: New file.
* src/Makefile.am: Add pack.c, pack.h
* src/binlog.c: Rewrite.  Use pack routines to construct each
record.
* src/binlogcat.c: Likewise.  Use packout to unpack each record.
* src/vmod-binlog.h (struct binlog_record): Remove nid,aid.
(struct binlog_file_header): Add hdrsize.
(BINLOG_HEADER_SIZE,union binlog_header): Remove
(binlog_recnum): Remove.
(binlog_size): Rewrite.
* src/vmod.vcc (append,sappend): Remove.
(init): Change signature.
(start,commit,pack): New functions.
* tests/test01.at: Rewrite.
* tests/test02.at: Rewrite.

10 files changed, 843 insertions, 77 deletions

diff --git a/src/.gitignore b/src/.gitignore
index 7f6e438..a75fe6d 100644
--- a/src/.gitignore
+++ b/src/.gitignore
@@ -1,2 +1,3 @@
+binlogcat
 vcc_if.c
 vcc_if.h
diff --git a/src/Makefile.am b/src/Makefile.am
index c9daf9a..9ce519f 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -14,25 +14,29 @@
 # You should have received a copy of the GNU General Public License
 # along with vmod-binlog.  If not, see <http://www.gnu.org/licenses/>.
 
 AM_CPPFLAGS = -I$(VARNISHSRC)/include -I$(VARNISHSRC)
 
 bin_PROGRAMS = binlogcat
-binlogcat_SOURCES = binlogcat.c
+binlogcat_SOURCES = binlogcat.c pack.c
+binlogcat_CFLAGS = $(AM_CFLAGS)
 
 vmoddir = $(VMODDIR)
 vmod_LTLIBRARIES = libvmod_binlog.la
 
 libvmod_binlog_la_LDFLAGS = -module -export-dynamic -avoid-version
 libvmod_binlog_la_LIBADD=
 
 libvmod_binlog_la_SOURCES = \
 	binlog.c\
+	pack.c\
 	vmod-binlog.h\
 	vcc_if.c vcc_if.h
 
+noinst_HEADERS = pack.h
+
 BUILT_SOURCES = vcc_if.c vcc_if.h
 
 vcc_if.c vcc_if.h: $(VARNISHSRC)/lib/libvmod_std/vmod.py $(top_srcdir)/src/vmod.vcc
 	@PYTHON@ $(VARNISHSRC)/lib/libvmod_std/vmod.py $(top_srcdir)/src/vmod.vcc
 
 EXTRA_DIST = vmod.vcc
diff --git a/src/binlog.c b/src/binlog.c
index be32e10..6f438a8 100644
--- a/src/binlog.c
+++ b/src/binlog.c
@@ -19,43 +19,58 @@
 #include <sys/stat.h>
 #include <sys/mman.h>
 #include <fcntl.h>
 #include <unistd.h>
 #include <errno.h>
 #include <syslog.h>
+#include <stddef.h>
 #include <stdlib.h>
 #include <ctype.h>
 #include <time.h>
 #include "vrt.h"
 #include "vcc_if.h"
 #include "bin/varnishd/cache.h"
 #include "vmod-binlog.h"
+#include "pack.h"
 
 #ifndef O_SEARCH
 # define O_SEARCH 0
 #endif
 
 #define BLF_ROUNDTS 0x01
 
+enum binlog_state {
+	state_init,
+	state_start,
+	state_pack
+};
+
 struct binlog_config {
 	size_t size;         /* maximum file size */
 	unsigned interval;   /* file rotation interval */
 	char *pattern;       /* file name pattern */
 	int umask;           /* umask for new files and directories */
 	char *dir;           /* root storage directory */      
 	int dd;              /* directory descriptor */
 	char *fname;         /* current file name */
 	int fd;              /* current file descriptor */
-	union binlog_header *base;     /* mmap base */
-	struct binlog_record *recbase; /* record base */
+	struct binlog_file_header *base;     /* mmap base */
+	char *recbase;       /* record base */
 	size_t recnum;       /* number of records in recbase */
-	size_t recidx;       /* index of the next free entry in recbase */
+	size_t recsize;      /* record size */
 	time_t stoptime;     /* when to rotate the current file */
 	pthread_mutex_t mutex;
 	int debug;
 	int flags;
+
+	char *dataspec;
+	struct packinst *inst_head;
+	struct packinst *inst_cur;
+	struct packenv *env;
+	enum binlog_state state;
+	time_t timestamp;
 };
 
 static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
 
 void
 binlog_error(const char *fmt, ...)
@@ -155,30 +170,26 @@ getinterval(char *p, char **endp)
 		}
 		p++;
 	}
 }
 
 void
-vmod_init(struct sess *sp, struct vmod_priv *priv, const char *param)
+vmod_init(struct sess *sp, struct vmod_priv *priv,
+	  const char *dir, const char *dataspec, const char *param)
 {
 	struct binlog_config *conf = priv->priv;
 	struct stat st;
-	char *dir, *p, *q;
+	char *p, *q;
 	unsigned long n;
 
 	p = findparam(param, "debug");
 	if (p) {
 		conf->debug = atoi(p);
 		free(p);
 	}
-	
-	dir = findparam(param, "dir");
-	if (!dir) {
-		binlog_error("parameter \"dir\" not set");
-		abort();
-	}
+
 	if (stat(dir, &st)) {
 		if (errno == ENOENT)
 			binlog_error("logging directory does not exist");
 		else
 			binlog_error("cannot stat logging directory %s: %s",
 				     dir, strerror(errno));
@@ -191,14 +202,26 @@ vmod_init(struct sess *sp, struct vmod_priv *priv, const char *param)
 	conf->dd = open(dir, O_SEARCH | O_DIRECTORY);
 	if (conf->dd == -1) {
 		binlog_error("cannot open directory %s: %s",
 			     dir, strerror(errno));
 		abort();
 	}
-	conf->dir = dir;
+	conf->dir = strdup(dir);
+	AN(conf->dir);
 
+	conf->inst_head = packcomp(dataspec, &p);
+	if (*p) {
+		binlog_error("cannot compile data format near %s", p);
+		abort();
+	}
+	conf->recsize = packsize(conf->inst_head);
+	conf->env = packenv_create(conf->recsize);
+	conf->recsize += offsetof(struct binlog_record,data);
+	conf->dataspec = strdup(dataspec);
+	AN(conf->dataspec);
+		
 	p = findparam(param, "pattern");
 	if (!p) {
 		p = strdup(BINLOG_PATTERN);
 		AN(p);
 	}
 	conf->pattern = p;
@@ -364,29 +387,32 @@ reset(struct binlog_config *conf)
 {
 	conf->fname = NULL;
 	conf->fd = -1;
 	conf->base = NULL;
 	conf->recbase = NULL;
 	conf->recnum = 0;
-	conf->recidx = 0;
 }
 
 static int
 setstoptime(struct binlog_config *conf)
 {
 	time_t ts;
 	
 	ts = time(NULL);
 	conf->stoptime = ts - ts % conf->interval + conf->interval;
 }
 
+#define binlog_recnum(conf) \
+	(((conf)->size - (conf)->base->hdrsize) / (conf)->base->recsize)
+
 static int
 newfile(struct sess *sp, struct binlog_config *conf)
 {
 	int c;
 	void *base;	
+	size_t n;
 	
 	setstoptime(conf);
 		
 	if (createfile(sp, conf))
 		return -1;
 	if (lseek(conf->fd, conf->size, SEEK_SET) == -1) {
@@ -410,43 +436,49 @@ newfile(struct sess *sp, struct binlog_config *conf)
 		free(conf->fname);
 		reset(conf);
 		return -1;
 	}
 
 	conf->base = base;
-	memcpy(conf->base->hdr.magic, BINLOG_MAGIC_STR, BINLOG_MAGIC_LEN);
-	conf->base->hdr.version = BINLOG_VERSION;
-	conf->base->hdr.recsize = sizeof(struct binlog_record);
-	conf->base->hdr.recnum = 0;
+	memcpy(conf->base->magic, BINLOG_MAGIC_STR, BINLOG_MAGIC_LEN);
+	conf->base->version = BINLOG_VERSION;
+	conf->base->recsize = conf->recsize;
+	conf->base->recnum = 0;
+	strcpy((char*)(conf->base + 1), conf->dataspec);
 
-	conf->recbase = (struct binlog_record *) (conf->base + 1);
+	n = (sizeof(struct binlog_file_header) + strlen(conf->dataspec) +
+	     conf->recsize - 1) / conf->recsize;
+	conf->base->hdrsize = n * conf->recsize;
+
+	conf->recbase = (char *) conf->base + conf->base->hdrsize;
 	conf->recnum = binlog_recnum(conf);
-	conf->recidx = 0;
 
 	debug(conf,1,("created new log file %s",conf->fname));
 	return 0;
 }
 	
 static void
 closefile(struct sess *sp, struct binlog_config *conf)
 {
+	size_t size;
+	
 	if (conf->fd == -1)
 		return;
 	debug(conf,1,("closing log file %s",conf->fname));
+	size = binlog_size(conf->base);
 	munmap(conf->base, conf->size);
-	if (ftruncate(conf->fd, binlog_size(conf)))
+	if (ftruncate(conf->fd, size))
 		binlog_error("error truncating \"%s/%s\": %s",
 			     conf->dir, conf->fname, strerror(errno));
 	close(conf->fd);
 	free(conf->fname);
 	reset(conf);
 }
-	
 
 void
-vmod_append(struct sess *sp, struct vmod_priv *priv, int nid, int aid)
+vmod_start(struct sess *sp, struct vmod_priv *priv)
 {
 	struct binlog_config *conf = priv->priv;
 	time_t ts;
 
 	if (!conf)
 		return;
@@ -456,46 +488,103 @@ vmod_append(struct sess *sp, struct vmod_priv *priv, int nid, int aid)
 	if (ts >= conf->stoptime) {
 		AZ(pthread_mutex_lock(&conf->mutex));
 		closefile(sp, conf);
 		newfile(sp, conf);
 		AZ(pthread_mutex_unlock(&conf->mutex));
 	}
+
+	packenv_init(conf->env);
+	conf->state = state_start;
+	conf->inst_cur = conf->inst_head;
+	conf->timestamp = ts;
+}
+
+void
+vmod_pack(struct sess *sp, struct vmod_priv *priv, const char *str)
+{
+	struct binlog_config *conf = priv->priv;
+	char *argv[2];
+	
+	if (!conf)
+		return;
+
+	switch (conf->state) {
+	case state_start:
+	case state_pack:
+		break;
+	default:
+		binlog_error("pack called in wrong state (%d)", conf->state);
+		return;
+	}
+
+	if (!conf->inst_cur) {
+		binlog_error("format spec exhausted");
+		return;
+	}
+	
+	argv[0] = (char*) str;
+	argv[1] = NULL;
+	conf->env->argv = argv;
+	conf->env->argc = 2;
+	conf->env->argi = 0;
+
+	conf->inst_cur = packinnext(conf->inst_cur, conf->env);
+	
+	conf->state = state_pack;
+}
+
+void
+vmod_commit(struct sess *sp, struct vmod_priv *priv)
+{
+	struct binlog_config *conf = priv->priv;
+
+	if (!conf)
+		return;
 	if (conf->fd == -1)
 		return;
 
+	switch (conf->state) {
+	case state_start:
+		binlog_error("committing empty binlog record");
+		break;
+	case state_pack:
+		if (conf->inst_cur)
+			binlog_error("committing incomplete binlog record");
+		break;
+	default:
+		binlog_error("pack called in wrong state (%d)", conf->state);
+		return;
+	}
+
 	AZ(pthread_mutex_lock(&conf->mutex));
-	if (conf->recidx == conf->recnum) {
+	if (conf->base->recnum == conf->recnum) {
 		binlog_error("overflow of %s/%s", conf->dir, conf->fname);
 	} else {
-		struct binlog_record *p = conf->recbase + conf->recidx++;
-		p->nid = nid;
-		p->aid = aid;
-		p->ts = ts;
-		conf->base->hdr.recnum++;
+		struct binlog_record *p =
+			(struct binlog_record *)(conf->recbase +
+						 conf->base->recnum *
+						 conf->recsize);
+		p->ts = conf->timestamp;
+		memcpy(p->data, conf->env->buf_base, conf->env->buf_size);
+		conf->base->recnum++;
 	}
 	AZ(pthread_mutex_unlock(&conf->mutex));
-}
-
-void
-vmod_sappend(struct sess *sp, struct vmod_priv *priv, const char *nid,
-	     const char *aid)
-{
-	vmod_append(sp, priv, nid ? atoi(nid): 0, aid ? atoi(aid) : 0);
+	conf->state = state_init;
 }
 
 void
 vmod_sync(struct sess *sp, struct vmod_priv *priv)
 {
 	struct binlog_config *conf = priv->priv;
 
 	if (!conf)
 		return;
 
 	AZ(pthread_mutex_lock(&conf->mutex));
 	if (conf->base)
-		msync(conf->base, binlog_size(conf), 0);
+		msync(conf->base, binlog_size(conf->base), 0);
 	AZ(pthread_mutex_unlock(&conf->mutex));
 }	
 
 void
 vmod_close(struct sess *sp, struct vmod_priv *priv)
 {
diff --git a/src/binlogcat.c b/src/binlogcat.c
index 219f78c..ee8837a 100644
--- a/src/binlogcat.c
+++ b/src/binlogcat.c
@@ -14,34 +14,41 @@
    You should have received a copy of the GNU General Public License
    along with vmod-binlog.  If not, see <http://www.gnu.org/licenses/>.
 */
 
 #include <config.h>
 #include <unistd.h>
+#include <stddef.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <errno.h>
 #include <time.h>
 #include <string.h>
 #include "vmod-binlog.h"
+#include "pack.h"
 
 char *progname;
 char *timefmt = "%c";
 int number_option;
 int verbose_option;
 int timediff_option;
 
 void
 catlog(const char *fname)
 {
 	FILE *fp;
-	union binlog_header header;
-	struct binlog_record rec;
+	struct binlog_file_header header;
+	struct binlog_record *rec;
 	char timebuf[128];
 	size_t i;
 	time_t start_ts;
+	char *dataspec;
+	size_t size;
+	struct packenv *env;
+	struct packinst *inst;
+	char *p;
 	
 	if (strcmp(fname, "-") == 0)
 		fp = stdin;
 	else {
 		fp = fopen(fname, "r");
 		if (!fp) {
@@ -54,51 +61,87 @@ catlog(const char *fname)
 	if (fread(&header, sizeof(header), 1, fp) != 1) {
 		fprintf(stderr, "%s: error reading header of %s: %s\n",
 			progname, fname, strerror(errno));
 		exit(1);
 	}
 
-	if (memcmp(header.hdr.magic, BINLOG_MAGIC_STR, BINLOG_MAGIC_LEN)) {
+	if (memcmp(header.magic, BINLOG_MAGIC_STR, BINLOG_MAGIC_LEN)) {
 		fprintf(stderr, "%s: %s is not a binlog file\n",
 			progname, fname);
 		exit(1);
 	}
 
-	if (header.hdr.version != BINLOG_VERSION) {
+	if (header.version != BINLOG_VERSION) {
 		fprintf(stderr, "%s: %s: unknown version\n",
 			progname, fname);
 		exit(1);
 	}
 
-	if (header.hdr.recsize != sizeof(struct binlog_record)) {
-		fprintf(stderr, "%s: %s: record length mismatch\n",
-			progname, fname);
+	size = header.hdrsize - sizeof(header);
+	dataspec = malloc(size);
+	if (!dataspec) {
+		fprintf(stderr, "%s: not enough memory", progname);
+		abort();
+	}
+	
+	if (fread(dataspec, size, 1, fp) != 1) {
+		fprintf(stderr, "%s: error reading header of %s: %s\n",
+			progname, fname, strerror(errno));
 		exit(1);
 	}
 
 	if (verbose_option)
-		printf("# %s; %lu records\n", fname, header.hdr.recnum);
+		printf("# %s; format=%s; recsize=%lu; recnum=%lu\n",
+		       fname, dataspec, header.recsize, header.recnum);
+
+	inst = packcomp(dataspec, &p);
+	if (*p) {
+		fprintf(stderr, "%s: %s: bad dataspec near %s",
+			progname, dataspec, p);
+		exit(1);
+	}
+	free(dataspec);
+	
+	rec = malloc(header.recsize);
+	if (!rec) {
+		fprintf(stderr, "%s: not enough memory", progname);
+		abort();
+	}
+	env = packenv_create(header.recsize -
+			     offsetof(struct binlog_record,data));
+	env->fp = stdout;
 	
-	for (i = 0; i < header.hdr.recnum; i++) {
-		if (fread(&rec, sizeof(rec), 1, fp) != 1) {
+	for (i = 0; i < header.recnum; i++) {
+		if (fread(rec, header.recsize, 1, fp) != 1) {
 			fprintf(stderr, "%s: %s: unexpected eof\n",
 				progname, fname);
 			break;
 		}
 
 		if (timediff_option) {
 			if (i == 0)
-				start_ts = rec.ts;
-			rec.ts -= start_ts;
+				start_ts = rec->ts;
+			rec->ts -= start_ts;
 		}
 		
-		strftime(timebuf, sizeof timebuf, timefmt, localtime(&rec.ts));
+		strftime(timebuf, sizeof timebuf, timefmt,
+			 localtime(&rec->ts));
 		if (number_option)
 			printf("%lu ", (unsigned long) i);
-		printf("%s %ld %ld\n", timebuf, rec.nid, rec.aid);
+		printf("%s ", timebuf);
+
+		memcpy(env->buf_base, rec->data, env->buf_size);
+		env->buf_pos = 0;
+		
+		packout(inst, env);
+		fputc('\n', stdout);
 	}
+
+	free(rec);
+	packenv_free(env);
+	packfree(inst);
 	
 	fclose(fp);
 }
 
 void
 help()
diff --git a/src/pack.c b/src/pack.c
new file mode 100644
index 0000000..a6ee933
--- /dev/null
+++ b/src/pack.c
@@ -0,0 +1,588 @@
+/* This file is part of vmod-binlog
+   Copyright (C) 2013 Sergey Poznyakoff
+
+   Vmod-binlog is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3, or (at your option)
+   any later version.
+
+   Vmod-binlog is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with vmod-binlog.  If not, see <http://www.gnu.org/licenses/>.
+*/
+#include <config.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <ctype.h>
+#include "pack.h"
+
+/* Data format specification is a simplified form of Perl pack() template.
+   It consists of a series of template letters optionally followed by
+   numeric repeat count (which may be enclosed in square brackets).  The
+   valid template letters are:
+   
+   Z  A null-terminated (ASCIZ) string of at most N-1 characters, will be
+      null padded.  This letter must be followed by repeat count.
+
+   c  A signed char (8-bit) value.
+   C  An unsigned char (octet) value.
+
+   s  A signed short (16-bit) value.
+   S  An unsigned short value.
+
+   l  A signed long (32-bit) value.
+   L  An unsigned long value.
+
+   q  A signed quad (64-bit) value.
+   Q  An unsigned quad value.
+
+   i  A signed integer value.
+   I  A unsigned integer value.
+
+   x  A null byte (a.k.a ASCII NUL, "\000", chr(0))
+   X  Back up a byte.
+   @  Null-fill or truncate to absolute position, counted from the
+      current position.
+   .  Null-fill or truncate to absolute position specified by
+      the repeat count.
+*/
+
+#define F_REP 0x01
+
+struct packspec {
+	int ch;
+	size_t size;
+	int flags;
+	void (*packer)(struct packenv *, struct packspec *, int);
+	void (*unpacker)(struct packenv *, struct packspec *, int);
+};
+
+struct packinst {
+	struct packinst *next;
+	struct packspec *spec;
+	int rep;
+};
+
+static char *
+packenv_get(struct packenv *env)
+{
+	if (env->argi == env->argc)
+		return NULL;
+	return env->argv[env->argi++];
+}
+
+static void
+printunum(FILE *fp, uintmax_t num)
+{
+	char numbuf[512];
+	char *p = numbuf + sizeof(numbuf);
+	*--p = 0;
+	do
+		*--p = num % 10 + '0';
+	while (num /= 10);
+	fputs(p, fp);
+}
+
+static void
+printsnum(FILE *fp, intmax_t num)
+{
+	if (num < 0) {
+		fputc('-', fp);
+		num = - num;
+	}
+	printunum(fp, num);
+}
+
+static void
+Z_packer(struct packenv *env, struct packspec *spec, int rep)
+{
+	if (env->buf_base) {
+		char *arg = packenv_get(env);
+
+		memset(env->buf_base + env->buf_pos, 0, rep);
+		if (arg) {
+			size_t len = strlen(arg);
+			if (len > rep - 1)
+				len = rep - 1;
+			memcpy(env->buf_base + env->buf_pos, arg, len);
+		}
+	}
+	env->buf_pos += rep;
+}
+		
+static void
+Z_unpacker(struct packenv *env, struct packspec *spec, int rep)
+{
+	fprintf(env->fp, "%-*.*s", rep, rep, env->buf_base + env->buf_pos);
+ 	env->buf_pos += rep;
+}
+
+static void
+c_packer(struct packenv *env, struct packspec *spec, int rep)
+{
+	if (env->buf_base) {
+		char *arg = packenv_get(env);
+		if (arg)
+			env->buf_base[env->buf_pos] = *arg;
+	}
+	env->buf_pos++;
+}
+
+static void
+c_unpacker(struct packenv *env, struct packspec *spec, int rep)
+{
+	fprintf(env->fp, "%c", env->buf_base[env->buf_pos++]);
+}
+
+static void
+s_packer(struct packenv *env, struct packspec *spec, int rep)
+{
+	if (env->buf_base) {
+		char *arg = packenv_get(env);
+		if (arg) {
+			int16_t v = atoi(arg);
+			memcpy(env->buf_base + env->buf_pos, &v, sizeof(v));
+		}
+	}
+	env->buf_pos += spec->size;
+}
+
+static void
+s_unpacker(struct packenv *env, struct packspec *spec, int rep)
+{
+	printsnum(env->fp, *(int16_t *) (env->buf_base + env->buf_pos));
+	env->buf_pos += spec->size;
+}
+
+static void
+S_packer(struct packenv *env, struct packspec *spec, int rep)
+{
+	if (env->buf_base) {
+		char *arg = packenv_get(env);
+		if (arg) {
+			uint16_t v = atoi(arg);
+			memcpy(env->buf_base + env->buf_pos, &v, sizeof(v));
+		}
+	}
+	env->buf_pos += spec->size;
+}
+
+static void
+S_unpacker(struct packenv *env, struct packspec *spec, int rep)
+{
+	printunum(env->fp, *(uint16_t *)(env->buf_base + env->buf_pos));
+	env->buf_pos += spec->size;
+}
+
+static void
+l_packer(struct packenv *env, struct packspec *spec, int rep)
+{
+	if (env->buf_base) {
+		char *arg = packenv_get(env);
+		if (arg) {
+			int32_t v = atoi(arg);
+			memcpy(env->buf_base + env->buf_pos, &v, sizeof(v));
+		}
+	}
+	env->buf_pos += spec->size;
+}
+
+static void
+l_unpacker(struct packenv *env, struct packspec *spec, int rep)
+{
+	printsnum(env->fp, *(int32_t *)(env->buf_base + env->buf_pos));
+	env->buf_pos += spec->size;
+}
+
+static void
+L_packer(struct packenv *env, struct packspec *spec, int rep)
+{
+	if (env->buf_base) {
+		char *arg = packenv_get(env);
+		if (arg) {
+			uint32_t v = atoi(arg);
+			memcpy(env->buf_base + env->buf_pos, &v, sizeof(v));
+		}
+	}
+	env->buf_pos += spec->size;
+}
+
+static void
+L_unpacker(struct packenv *env, struct packspec *spec, int rep)
+{
+	printunum(env->fp, *(uint32_t *)(env->buf_base + env->buf_pos));
+	env->buf_pos += spec->size;
+}
+
+static void
+q_packer(struct packenv *env, struct packspec *spec, int rep)
+{
+	if (env->buf_base) {
+		char *arg = packenv_get(env);
+		if (arg) {
+			int64_t v = atoi(arg);
+			memcpy(env->buf_base + env->buf_pos, &v, sizeof(v));
+		}
+	}
+	env->buf_pos += spec->size;
+}
+
+static void
+q_unpacker(struct packenv *env, struct packspec *spec, int rep)
+{
+	printsnum(env->fp, *(int64_t *)(env->buf_base + env->buf_pos));
+	env->buf_pos += spec->size;
+}
+
+static void
+Q_packer(struct packenv *env, struct packspec *spec, int rep)
+{
+	if (env->buf_base) {
+		char *arg = packenv_get(env);
+		if (arg) {
+			uint64_t v = atoi(arg);
+			memcpy(env->buf_base + env->buf_pos, &v, sizeof(v));
+		}
+	}
+	env->buf_pos += spec->size;
+}
+
+static void
+Q_unpacker(struct packenv *env, struct packspec *spec, int rep)
+{
+	printunum(env->fp, *(uint64_t *)(env->buf_base + env->buf_pos));
+	env->buf_pos += spec->size;
+}
+
+static void
+i_packer(struct packenv *env, struct packspec *spec, int rep)
+{
+	if (env->buf_base) {
+		char *arg = packenv_get(env);
+		if (arg) {
+			int v = atoi(arg);
+			memcpy(env->buf_base + env->buf_pos, &v, sizeof(v));
+		}
+	}
+	env->buf_pos += spec->size;
+}
+
+static void
+i_unpacker(struct packenv *env, struct packspec *spec, int rep)
+{
+	printsnum(env->fp, *(int *)(env->buf_base + env->buf_pos));
+	env->buf_pos += spec->size;
+}
+
+static void
+I_packer(struct packenv *env, struct packspec *spec, int rep)
+{
+	if (env->buf_base) {
+		char *arg = packenv_get(env);
+		if (arg) {
+			unsigned int v = atoi(arg);
+			memcpy(env->buf_base + env->buf_pos, &v, sizeof(v));
+		}
+	}
+	env->buf_pos += spec->size;
+}
+
+static void
+I_unpacker(struct packenv *env, struct packspec *spec, int rep)
+{
+	printunum(env->fp, *(unsigned *)(env->buf_base + env->buf_pos));
+	env->buf_pos += spec->size;
+}
+
+static void
+x_packer(struct packenv *env, struct packspec *spec, int rep)
+{
+	if (env->buf_base)
+		env->buf_base[env->buf_pos] = 0;
+	env->buf_pos++;
+}
+
+static void
+x_unpacker(struct packenv *env, struct packspec *spec, int rep)
+{
+	fputc(0, env->fp);
+}
+
+static void
+X_packer(struct packenv *env, struct packspec *spec, int rep)
+{
+	if (env->buf_pos)
+		--env->buf_pos;
+}
+
+static void
+at_packer(struct packenv *env, struct packspec *spec, int rep)
+{
+	if (env->buf_base)
+		memset(env->buf_base + env->buf_pos, 0, rep);
+	env->buf_pos += rep;
+}
+
+static void
+at_unpacker(struct packenv *env, struct packspec *spec, int rep)
+{
+	env->buf_pos += rep;
+}
+
+static void
+dot_packer(struct packenv *env, struct packspec *spec, int rep)
+{
+	if (env->buf_base) {
+		if (rep < env->buf_pos)
+			memset(env->buf_base + rep, 0, env->buf_pos - rep);
+		else
+			memset(env->buf_base + env->buf_pos, 0,
+			       rep - env->buf_pos);
+	}
+	env->buf_pos = rep;
+}
+
+static void
+dot_unpacker(struct packenv *env, struct packspec *spec, int rep)
+{
+	env->buf_pos = rep;
+}
+
+static struct packspec packspec[] = {
+	{ 'Z', 1, F_REP, Z_packer, Z_unpacker },
+	{ 'c', 1, 0, c_packer, c_unpacker },
+	{ 's', sizeof(int16_t), 0, s_packer, s_unpacker },
+	{ 'S', sizeof(uint16_t), 0, S_packer, S_unpacker },
+	{ 'l', sizeof(int32_t), 0, l_packer, l_unpacker },
+	{ 'L', sizeof(uint32_t), 0, L_packer, L_unpacker },
+	{ 'q', sizeof(int64_t), 0, q_packer, q_unpacker },
+	{ 'Q', sizeof(uint64_t), 0, Q_packer, Q_unpacker },
+	{ 'i', sizeof(int), 0, i_packer, i_unpacker },
+	{ 'I', sizeof(unsigned), 0, I_packer, I_unpacker },
+	/* FIXME: n N v V f d */
+	{ 'x', 1, 0, x_packer, x_unpacker },
+	{ 'X', 1, 0, X_packer, X_packer },
+	{ '@', 0, F_REP, at_packer, at_unpacker },
+	{ '.', 0, F_REP, dot_packer, dot_unpacker },
+	
+	{ 0 }
+};
+
+static int
+getrep(const char *s, char const **endp, int *rep)
+{
+	int n;
+	char *p;
+
+	if (*s == '[') {
+		if (s[1]) {
+			if (isdigit(s[1])) {
+				n = strtol(s + 1, &p, 10);
+				if (n <= 0) {
+					*endp = s;
+					return -1;
+				} else if (*p != ']') {
+					*endp = p;
+					return -1;
+				}
+				*rep = n;
+				*endp = p + 1;
+			} else {
+				*endp = s;
+				return -1;
+			}
+		} else {
+			*endp = s;
+			return -1;
+		}
+	} else if (isdigit(*s)) {
+		n = strtol(s, &p, 10);
+		if (n <= 0) {
+			*endp = s;
+			return -1;
+		}
+		*rep = n;
+		*endp = p;
+	} else {
+		*rep = 1;
+		*endp = s;
+	}
+	return 0;
+}
+
+struct packinst *
+packcomp(const char *s, char **endp)
+{
+	struct packinst *head = NULL, *tail = NULL, *pi;
+	struct packspec *ps;
+	int rep;
+	
+	while (s) {
+		for (ps = packspec; ps->ch; ps++)
+			if (ps->ch == *s)
+				break;
+		if (!ps->ch)
+			break;
+		if (getrep(s + 1, &s, &rep))
+			break;
+		pi = malloc(sizeof(*pi));
+		pi->next = NULL;
+		pi->spec = ps;
+		pi->rep = rep;
+		if (tail)
+			tail->next = pi;
+		else
+			head = pi;
+		tail = pi; 
+	}
+	if (endp)
+		*endp = (char*) s;
+	return head;
+}
+
+void
+packfree(struct packinst *pi)
+{
+	while (pi) {
+		struct packinst *next = pi->next;
+		free(pi);
+		pi = next;
+	}
+}
+
+void
+packin(struct packinst *pi, struct packenv *env)
+{
+	int i;
+	
+	for (; pi; pi = pi->next) {
+		if (pi->spec->flags & F_REP)
+			pi->spec->packer(env, pi->spec, pi->rep);
+		else
+			for (i = 0; i < pi->rep; i++)
+				pi->spec->packer(env, pi->spec, 1);
+	}
+}
+
+struct packinst *
+packinnext(struct packinst *pi, struct packenv *env)
+{
+	int i;
+
+	if (!pi)
+		return NULL;
+	if (pi->spec->flags & F_REP)
+		pi->spec->packer(env, pi->spec, pi->rep);
+	else
+		for (i = 0; i < pi->rep; i++)
+			pi->spec->packer(env, pi->spec, 1);
+	return pi->next;
+}
+
+void
+packout(struct packinst *pi, struct packenv *env)
+{
+	int i;
+
+	for (; pi; pi = pi->next) {
+		if (pi->spec->flags & F_REP)
+			pi->spec->unpacker(env, pi->spec, pi->rep);
+		else
+			for (i = 0; i < pi->rep; i++)
+				pi->spec->unpacker(env, pi->spec, 1);
+		if (pi->next)
+			fputc(' ', env->fp);
+	}
+}
+
+size_t
+packsize(struct packinst *pi)
+{
+	struct packenv env;
+
+	memset(&env, 0, sizeof env);
+	packin(pi, &env);
+	return env.buf_pos;
+}
+
+struct packenv *
+packenv_create(size_t size)
+{
+	struct packenv *env = calloc(1, sizeof(*env));
+	env->buf_base = calloc(1, size);
+	env->buf_size = size;
+	return env;
+}
+
+void
+packenv_free(struct packenv *env)
+{
+	if (!env)
+		return;
+	free(env->buf_base);
+	free(env);
+}
+
+void
+packenv_init(struct packenv *env)
+{
+	memset(env->buf_base, 0, env->buf_size);
+	env->buf_pos = 0;
+}
+
+#ifdef STANDALONE
+#include <unistd.h>
+
+int
+main(int argc, char **argv)
+{
+	void (*fn)(struct packinst *pi, struct packenv *env) = packin;
+	struct packinst *pi;
+	struct packenv *env;
+	char *end;
+	int c;
+	
+	while ((c = getopt(argc, argv, "d")) != EOF) {
+		switch (c) {
+		case 'd':
+			fn = packout;
+			break;
+		default:
+			exit(1);
+		}
+	}
+
+	argc -= optind;
+	argv += optind;
+
+	if (argc == 0)
+		abort();
+
+	pi = packcomp(argv[0], &end);
+	if (*end) {
+		fprintf(stderr, "compile error near %s\n", end);
+		exit(1);
+	}
+	env = packenv_create(packsize(pi));
+	env->fp = stdout;
+	env->argv = argv + 1;
+	env->argc = argc - 1;
+
+	if (fn == packout)
+		fread(env->buf_base, env->buf_size, 1, stdin);
+	
+	fn(pi, env);
+	if (fn == packin) {
+		fwrite(env->buf_base, env->buf_size, 1, stdout);
+	}
+	
+	packenv_free(env);
+	packfree(pi);
+}
+#endif
diff --git a/src/pack.h b/src/pack.h
new file mode 100644
index 0000000..70a2488
--- /dev/null
+++ b/src/pack.h
@@ -0,0 +1,47 @@
+/* This file is part of vmod-binlog
+   Copyright (C) 2013 Sergey Poznyakoff
+
+   Vmod-binlog is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3, or (at your option)
+   any later version.
+
+   Vmod-binlog is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with vmod-binlog.  If not, see <http://www.gnu.org/licenses/>.
+*/
+#include <stdio.h>
+
+struct packenv {
+	char *buf_base;
+	size_t buf_size;
+	size_t buf_pos;
+	char **argv;
+	int argc;
+	int argi;
+	FILE *fp;
+};
+
+struct packinst;
+
+struct packinst *packcomp(const char *s, char **endp);
+void packfree(struct packinst *pi);
+void packin(struct packinst *pi, struct packenv *env);