Rewrite using opaque record data.

Records can carry arbitrary data, whose format is defined using
teplate strings similar to those of Perl's pack() function.  The
teplate string is stored in the logfile header and is used by
binlogcat to render a human-readable representation.

* src/.gitignore: New file.
* pack.c: New file.
* pack.h: New file.
* src/Makefile.am: Add pack.c, pack.h
* src/binlog.c: Rewrite.  Use pack routines to construct each
record.
* src/binlogcat.c: Likewise.  Use packout to unpack each record.
* src/vmod-binlog.h (struct binlog_record): Remove nid,aid.
(struct binlog_file_header): Add hdrsize.
(BINLOG_HEADER_SIZE,union binlog_header): Remove
(binlog_recnum): Remove.
(binlog_size): Rewrite.
* src/vmod.vcc (append,sappend): Remove.
(init): Change signature.
(start,commit,pack): New functions.
* tests/test01.at: Rewrite.
* tests/test02.at: Rewrite.

1 files changed, 588 insertions, 0 deletions

diff --git a/src/pack.c b/src/pack.c
new file mode 100644
index 0000000..a6ee933
--- /dev/null
+++ b/src/pack.c
@@ -0,0 +1,588 @@
+/* This file is part of vmod-binlog
+   Copyright (C) 2013 Sergey Poznyakoff
+
+   Vmod-binlog is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3, or (at your option)
+   any later version.
+
+   Vmod-binlog is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with vmod-binlog.  If not, see <http://www.gnu.org/licenses/>.
+*/
+#include <config.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <ctype.h>
+#include "pack.h"
+
+/* Data format specification is a simplified form of Perl pack() template.
+   It consists of a series of template letters optionally followed by
+   numeric repeat count (which may be enclosed in square brackets).  The
+   valid template letters are:
+   
+   Z  A null-terminated (ASCIZ) string of at most N-1 characters, will be
+      null padded.  This letter must be followed by repeat count.
+
+   c  A signed char (8-bit) value.
+   C  An unsigned char (octet) value.
+
+   s  A signed short (16-bit) value.
+   S  An unsigned short value.
+
+   l  A signed long (32-bit) value.
+   L  An unsigned long value.
+
+   q  A signed quad (64-bit) value.
+   Q  An unsigned quad value.
+
+   i  A signed integer value.
+   I  A unsigned integer value.
+
+   x  A null byte (a.k.a ASCII NUL, "\000", chr(0))
+   X  Back up a byte.
+   @  Null-fill or truncate to absolute position, counted from the
+      current position.
+   .  Null-fill or truncate to absolute position specified by
+      the repeat count.
+*/
+
+#define F_REP 0x01
+
+struct packspec {
+	int ch;
+	size_t size;
+	int flags;
+	void (*packer)(struct packenv *, struct packspec *, int);
+	void (*unpacker)(struct packenv *, struct packspec *, int);
+};
+
+struct packinst {
+	struct packinst *next;
+	struct packspec *spec;
+	int rep;
+};
+
+static char *
+packenv_get(struct packenv *env)
+{
+	if (env->argi == env->argc)
+		return NULL;
+	return env->argv[env->argi++];
+}
+
+static void
+printunum(FILE *fp, uintmax_t num)
+{
+	char numbuf[512];
+	char *p = numbuf + sizeof(numbuf);
+	*--p = 0;
+	do
+		*--p = num % 10 + '0';
+	while (num /= 10);
+	fputs(p, fp);
+}
+
+static void
+printsnum(FILE *fp, intmax_t num)
+{
+	if (num < 0) {
+		fputc('-', fp);
+		num = - num;
+	}
+	printunum(fp, num);
+}
+
+static void
+Z_packer(struct packenv *env, struct packspec *spec, int rep)
+{
+	if (env->buf_base) {
+		char *arg = packenv_get(env);
+
+		memset(env->buf_base + env->buf_pos, 0, rep);
+		if (arg) {
+			size_t len = strlen(arg);
+			if (len > rep - 1)
+				len = rep - 1;
+			memcpy(env->buf_base + env->buf_pos, arg, len);
+		}
+	}
+	env->buf_pos += rep;
+}
+		
+static void
+Z_unpacker(struct packenv *env, struct packspec *spec, int rep)
+{
+	fprintf(env->fp, "%-*.*s", rep, rep, env->buf_base + env->buf_pos);
+ 	env->buf_pos += rep;
+}
+
+static void
+c_packer(struct packenv *env, struct packspec *spec, int rep)
+{
+	if (env->buf_base) {
+		char *arg = packenv_get(env);
+		if (arg)
+			env->buf_base[env->buf_pos] = *arg;
+	}
+	env->buf_pos++;
+}
+
+static void
+c_unpacker(struct packenv *env, struct packspec *spec, int rep)
+{
+	fprintf(env->fp, "%c", env->buf_base[env->buf_pos++]);
+}
+
+static void
+s_packer(struct packenv *env, struct packspec *spec, int rep)
+{
+	if (env->buf_base) {
+		char *arg = packenv_get(env);
+		if (arg) {
+			int16_t v = atoi(arg);
+			memcpy(env->buf_base + env->buf_pos, &v, sizeof(v));
+		}
+	}
+	env->buf_pos += spec->size;
+}
+
+static void
+s_unpacker(struct packenv *env, struct packspec *spec, int rep)
+{
+	printsnum(env->fp, *(int16_t *) (env->buf_base + env->buf_pos));
+	env->buf_pos += spec->size;
+}
+
+static void
+S_packer(struct packenv *env, struct packspec *spec, int rep)
+{
+	if (env->buf_base) {
+		char *arg = packenv_get(env);
+		if (arg) {
+			uint16_t v = atoi(arg);
+			memcpy(env->buf_base + env->buf_pos, &v, sizeof(v));
+		}
+	}
+	env->buf_pos += spec->size;
+}
+
+static void
+S_unpacker(struct packenv *env, struct packspec *spec, int rep)
+{
+	printunum(env->fp, *(uint16_t *)(env->buf_base + env->buf_pos));
+	env->buf_pos += spec->size;
+}
+
+static void
+l_packer(struct packenv *env, struct packspec *spec, int rep)
+{
+	if (env->buf_base) {
+		char *arg = packenv_get(env);
+		if (arg) {
+			int32_t v = atoi(arg);
+			memcpy(env->buf_base + env->buf_pos, &v, sizeof(v));
+		}
+	}
+	env->buf_pos += spec->size;
+}
+
+static void
+l_unpacker(struct packenv *env, struct packspec *spec, int rep)
+{
+	printsnum(env->fp, *(int32_t *)(env->buf_base + env->buf_pos));
+	env->buf_pos += spec->size;
+}
+
+static void
+L_packer(struct packenv *env, struct packspec *spec, int rep)
+{
+	if (env->buf_base) {
+		char *arg = packenv_get(env);
+		if (arg) {
+			uint32_t v = atoi(arg);
+			memcpy(env->buf_base + env->buf_pos, &v, sizeof(v));
+		}
+	}
+	env->buf_pos += spec->size;
+}
+
+static void
+L_unpacker(struct packenv *env, struct packspec *spec, int rep)
+{
+	printunum(env->fp, *(uint32_t *)(env->buf_base + env->buf_pos));
+	env->buf_pos += spec->size;
+}
+
+static void
+q_packer(struct packenv *env, struct packspec *spec, int rep)
+{
+	if (env->buf_base) {
+		char *arg = packenv_get(env);
+		if (arg) {
+			int64_t v = atoi(arg);
+			memcpy(env->buf_base + env->buf_pos, &v, sizeof(v));
+		}
+	}
+	env->buf_pos += spec->size;
+}
+
+static void
+q_unpacker(struct packenv *env, struct packspec *spec, int rep)
+{
+	printsnum(env->fp, *(int64_t *)(env->buf_base + env->buf_pos));
+	env->buf_pos += spec->size;
+}
+
+static void
+Q_packer(struct packenv *env, struct packspec *spec, int rep)
+{
+	if (env->buf_base) {
+		char *arg = packenv_get(env);
+		if (arg) {
+			uint64_t v = atoi(arg);
+			memcpy(env->buf_base + env->buf_pos, &v, sizeof(v));
+		}
+	}
+	env->buf_pos += spec->size;
+}
+
+static void
+Q_unpacker(struct packenv *env, struct packspec *spec, int rep)
+{
+	printunum(env->fp, *(uint64_t *)(env->buf_base + env->buf_pos));
+	env->buf_pos += spec->size;
+}
+
+static void
+i_packer(struct packenv *env, struct packspec *spec, int rep)
+{
+	if (env->buf_base) {
+		char *arg = packenv_get(env);
+		if (arg) {
+			int v = atoi(arg);
+			memcpy(env->buf_base + env->buf_pos, &v, sizeof(v));
+		}
+	}
+	env->buf_pos += spec->size;
+}
+
+static void
+i_unpacker(struct packenv *env, struct packspec *spec, int rep)
+{
+	printsnum(env->fp, *(int *)(env->buf_base + env->buf_pos));
+	env->buf_pos += spec->size;
+}
+
+static void
+I_packer(struct packenv *env, struct packspec *spec, int rep)
+{
+	if (env->buf_base) {
+		char *arg = packenv_get(env);
+		if (arg) {
+			unsigned int v = atoi(arg);
+			memcpy(env->buf_base + env->buf_pos, &v, sizeof(v));
+		}
+	}
+	env->buf_pos += spec->size;
+}
+
+static void
+I_unpacker(struct packenv *env, struct packspec *spec, int rep)
+{
+	printunum(env->fp, *(unsigned *)(env->buf_base + env->buf_pos));
+	env->buf_pos += spec->size;
+}
+
+static void
+x_packer(struct packenv *env, struct packspec *spec, int rep)
+{
+	if (env->buf_base)
+		env->buf_base[env->buf_pos] = 0;
+	env->buf_pos++;
+}
+
+static void
+x_unpacker(struct packenv *env, struct packspec *spec, int rep)
+{
+	fputc(0, env->fp);
+}
+
+static void
+X_packer(struct packenv *env, struct packspec *spec, int rep)
+{
+	if (env->buf_pos)
+		--env->buf_pos;
+}
+
+static void
+at_packer(struct packenv *env, struct packspec *spec, int rep)
+{
+	if (env->buf_base)
+		memset(env->buf_base + env->buf_pos, 0, rep);
+	env->buf_pos += rep;
+}
+
+static void
+at_unpacker(struct packenv *env, struct packspec *spec, int rep)
+{
+	env->buf_pos += rep;
+}
+
+static void
+dot_packer(struct packenv *env, struct packspec *spec, int rep)
+{
+	if (env->buf_base) {
+		if (rep < env->buf_pos)
+			memset(env->buf_base + rep, 0, env->buf_pos - rep);
+		else
+			memset(env->buf_base + env->buf_pos, 0,
+			       rep - env->buf_pos);
+	}
+	env->buf_pos = rep;
+}
+
+static void
+dot_unpacker(struct packenv *env, struct packspec *spec, int rep)
+{
+	env->buf_pos = rep;
+}
+
+static struct packspec packspec[] = {
+	{ 'Z', 1, F_REP, Z_packer, Z_unpacker },
+	{ 'c', 1, 0, c_packer, c_unpacker },
+	{ 's', sizeof(int16_t), 0, s_packer, s_unpacker },
+	{ 'S', sizeof(uint16_t), 0, S_packer, S_unpacker },
+	{ 'l', sizeof(int32_t), 0, l_packer, l_unpacker },
+	{ 'L', sizeof(uint32_t), 0, L_packer, L_unpacker },
+	{ 'q', sizeof(int64_t), 0, q_packer, q_unpacker },
+	{ 'Q', sizeof(uint64_t), 0, Q_packer, Q_unpacker },
+	{ 'i', sizeof(int), 0, i_packer, i_unpacker },
+	{ 'I', sizeof(unsigned), 0, I_packer, I_unpacker },
+	/* FIXME: n N v V f d */
+	{ 'x', 1, 0, x_packer, x_unpacker },
+	{ 'X', 1, 0, X_packer, X_packer },
+	{ '@', 0, F_REP, at_packer, at_unpacker },
+	{ '.', 0, F_REP, dot_packer, dot_unpacker },
+	
+	{ 0 }
+};
+
+static int
+getrep(const char *s, char const **endp, int *rep)
+{
+	int n;
+	char *p;
+
+	if (*s == '[') {
+		if (s[1]) {
+			if (isdigit(s[1])) {
+				n = strtol(s + 1, &p, 10);
+				if (n <= 0) {
+					*endp = s;
+					return -1;
+				} else if (*p != ']') {
+					*endp = p;
+					return -1;
+				}
+				*rep = n;
+				*endp = p + 1;
+			} else {
+				*endp = s;
+				return -1;
+			}
+		} else {
+			*endp = s;
+			return -1;
+		}
+	} else if (isdigit(*s)) {
+		n = strtol(s, &p, 10);
+		if (n <= 0) {
+			*endp = s;
+			return -1;
+		}
+		*rep = n;
+		*endp = p;
+	} else {
+		*rep = 1;
+		*endp = s;
+	}
+	return 0;
+}
+
+struct packinst *
+packcomp(const char *s, char **endp)
+{
+	struct packinst *head = NULL, *tail = NULL, *pi;
+	struct packspec *ps;
+	int rep;
+	
+	while (s) {
+		for (ps = packspec; ps->ch; ps++)
+			if (ps->ch == *s)
+				break;
+		if (!ps->ch)
+			break;
+		if (getrep(s + 1, &s, &rep))
+			break;
+		pi = malloc(sizeof(*pi));
+		pi->next = NULL;
+		pi->spec = ps;
+		pi->rep = rep;
+		if (tail)
+			tail->next = pi;
+		else
+			head = pi;
+		tail = pi; 
+	}
+	if (endp)
+		*endp = (char*) s;
+	return head;
+}
+
+void
+packfree(struct packinst *pi)
+{
+	while (pi) {
+		struct packinst *next = pi->next;
+		free(pi);
+		pi = next;
+	}
+}
+
+void
+packin(struct packinst *pi, struct packenv *env)
+{
+	int i;
+	
+	for (; pi; pi = pi->next) {
+		if (pi->spec->flags & F_REP)
+			pi->spec->packer(env, pi->spec, pi->rep);
+		else
+			for (i = 0; i < pi->rep; i++)
+				pi->spec->packer(env, pi->spec, 1);
+	}
+}
+
+struct packinst *
+packinnext(struct packinst *pi, struct packenv *env)
+{
+	int i;
+
+	if (!pi)
+		return NULL;
+	if (pi->spec->flags & F_REP)
+		pi->spec->packer(env, pi->spec, pi->rep);
+	else
+		for (i = 0; i < pi->rep; i++)
+			pi->spec->packer(env, pi->spec, 1);
+	return pi->next;
+}
+
+void
+packout(struct packinst *pi, struct packenv *env)
+{
+	int i;
+
+	for (; pi; pi = pi->next) {
+		if (pi->spec->flags & F_REP)
+			pi->spec->unpacker(env, pi->spec, pi->rep);
+		else
+			for (i = 0; i < pi->rep; i++)
+				pi->spec->unpacker(env, pi->spec, 1);
+		if (pi->next)
+			fputc(' ', env->fp);
+	}
+}
+
+size_t
+packsize(struct packinst *pi)
+{
+	struct packenv env;
+
+	memset(&env, 0, sizeof env);
+	packin(pi, &env);
+	return env.buf_pos;
+}
+
+struct packenv *
+packenv_create(size_t size)
+{
+	struct packenv *env = calloc(1, sizeof(*env));
+	env->buf_base = calloc(1, size);
+	env->buf_size = size;
+	return env;
+}
+
+void
+packenv_free(struct packenv *env)
+{
+	if (!env)
+		return;
+	free(env->buf_base);
+	free(env);
+}
+
+void
+packenv_init(struct packenv *env)
+{
+	memset(env->buf_base, 0, env->buf_size);
+	env->buf_pos = 0;
+}
+
+#ifdef STANDALONE
+#include <unistd.h>
+
+int
+main(int argc, char **argv)
+{
+	void (*fn)(struct packinst *pi, struct packenv *env) = packin;
+	struct packinst *pi;
+	struct packenv *env;
+	char *end;
+	int c;
+	
+	while ((c = getopt(argc, argv, "d")) != EOF) {
+		switch (c) {
+		case 'd':
+			fn = packout;
+			break;
+		default:
+			exit(1);
+		}
+	}
+
+	argc -= optind;
+	argv += optind;
+
+	if (argc == 0)
+		abort();
+
+	pi = packcomp(argv[0], &end);
+	if (*end) {
+		fprintf(stderr, "compile error near %s\n", end);
+		exit(1);
+	}
+	env = packenv_create(packsize(pi));
+	env->fp = stdout;
+	env->argv = argv + 1;
+	env->argc = argc - 1;
+
+	if (fn == packout)
+		fread(env->buf_base, env->buf_size, 1, stdin);
+	
+	fn(pi, env);
+	if (fn == packin) {
+		fwrite(env->buf_base, env->buf_size, 1, stdout);
+	}
+	
+	packenv_free(env);
+	packfree(pi);
+}
+#endif