Initial commit

6 files changed, 771 insertions, 0 deletions

diff --git a/src/.gitignore b/src/.gitignore
new file mode 100644
index 0000000..7f6e438
--- /dev/null
+++ b/src/.gitignore
@@ -0,0 +1,2 @@
+vcc_if.c
+vcc_if.h
diff --git a/src/Makefile.am b/src/Makefile.am
new file mode 100644
index 0000000..c9daf9a
--- /dev/null
+++ b/src/Makefile.am
@@ -0,0 +1,41 @@
+# This file is part of vmod-binlog
+# Copyright (C) 2013 Sergey Poznyakoff
+#
+# Vmod-binlog is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3, or (at your option)
+# any later version.
+#
+# Vmod-binlog is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with vmod-binlog.  If not, see <http://www.gnu.org/licenses/>.
+
+AM_CPPFLAGS = -I$(VARNISHSRC)/include -I$(VARNISHSRC)
+
+bin_PROGRAMS = binlogcat
+binlogcat_SOURCES = binlogcat.c
+
+vmoddir = $(VMODDIR)
+vmod_LTLIBRARIES = libvmod_binlog.la
+
+libvmod_binlog_la_LDFLAGS = -module -export-dynamic -avoid-version
+libvmod_binlog_la_LIBADD=
+
+libvmod_binlog_la_SOURCES = \
+	binlog.c\
+	vmod-binlog.h\
+	vcc_if.c vcc_if.h
+
+BUILT_SOURCES = vcc_if.c vcc_if.h
+
+vcc_if.c vcc_if.h: $(VARNISHSRC)/lib/libvmod_std/vmod.py $(top_srcdir)/src/vmod.vcc
+	@PYTHON@ $(VARNISHSRC)/lib/libvmod_std/vmod.py $(top_srcdir)/src/vmod.vcc
+
+EXTRA_DIST = vmod.vcc
+
+CLEANFILES = $(builddir)/vcc_if.c $(builddir)/vcc_if.h
+
diff --git a/src/binlog.c b/src/binlog.c
new file mode 100644
index 0000000..f896803
--- /dev/null
+++ b/src/binlog.c
@@ -0,0 +1,516 @@
+/* This file is part of vmod-binlog
+   Copyright (C) 2013 Sergey Poznyakoff
+
+   Vmod-binlog is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3, or (at your option)
+   any later version.
+
+   Vmod-binlog is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with vmod-binlog.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include <config.h>
+#include <sys/stat.h>
+#include <sys/mman.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <errno.h>
+#include <syslog.h>
+#include <stdlib.h>
+#include <ctype.h>
+#include <time.h>
+#include "vrt.h"
+#include "vcc_if.h"
+#include "bin/varnishd/cache.h"
+#include "vmod-binlog.h"
+
+#ifndef O_SEARCH
+# define O_SEARCH 0
+#endif
+
+#define BLF_ROUNDTS 0x01
+
+struct binlog_config {
+	size_t size;         /* maximum file size */
+	unsigned interval;   /* file rotation interval */
+	char *pattern;       /* file name pattern */
+	int umask;           /* umask for new files and directories */
+	char *dir;           /* root storage directory */      
+	int dd;              /* directory descriptor */
+	char *fname;         /* current file name */
+	int fd;              /* current file descriptor */
+	union binlog_header *base;     /* mmap base */
+	struct binlog_record *recbase; /* record base */
+	size_t recnum;       /* number of records in recbase */
+	size_t recidx;       /* index of the next free entry in recbase */
+	time_t stoptime;     /* when to rotate the current file */
+	pthread_mutex_t mutex;
+	int debug;
+	int flags;
+};
+
+static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
+
+void
+binlog_error(const char *fmt, ...)
+{
+	va_list ap;
+	va_start(ap, fmt);
+	vsyslog(LOG_DAEMON|LOG_ERR, fmt, ap);
+	va_end(ap);
+}
+
+void
+binlog_debug(const char *fmt, ...)
+{
+	va_list ap;
+	va_start(ap, fmt);
+	vsyslog(LOG_DAEMON|LOG_DEBUG, fmt, ap);
+	va_end(ap);
+}
+
+#define debug(c,l,s) do { if ((c)->debug>=(l)) binlog_debug s; } while(0)
+
+int
+module_init(struct vmod_priv *priv, const struct VCL_conf *vclconf)
+{
+	struct binlog_config *conf = calloc(1, sizeof(*conf));
+	AN(conf);
+	priv->priv = conf;
+	return 0;
+}
+
+static char *
+findparam(const char *param, char *name)
+{
+	const char *p;
+	char *q;
+
+	while (*param) {
+		for (p = param, q = name; *p && *q && *p == *q; p++, q++);
+		for (param = p; *param && *param != ';'; param++);
+		if (*q == 0 && *p == '=') {
+			size_t len = param - p - 1;
+			q = malloc(len + 1);
+			AN(q);
+			memcpy(q, p + 1, len);
+			q[len] = 0;
+			return q;
+		}
+		if (*param)
+			++param;
+	}
+	return NULL;
+}
+
+static unsigned long
+getinterval(char *p, char **endp)
+{
+	int n;
+	unsigned long hours = 0, minutes = 0, seconds = 0;
+    
+	for (;;) {
+		n = 0;
+		while (*p && isdigit(*p)) {
+			n = 10*n + *p - '0';
+			p++;
+		}
+    
+		switch (*p) {
+		case 'h':
+		case 'H':
+			if (hours) {
+				*endp = p;
+				return -1;
+			}
+			hours = n;
+			break;
+		case 'm':
+		case 'M':
+			if (minutes) {
+				*endp = p;
+				return -1;
+			}
+			minutes = n;
+			break;
+		case 's':
+		case 'S':
+			if (seconds) {
+				*endp = p;
+				return -1;
+			}
+			seconds = n;
+			break;
+		default:
+			*endp = p;
+			if (!hours && !minutes && !seconds)
+				return n;
+			return (hours*60 + minutes)*60 + seconds;
+		}
+		p++;
+	}
+}
+
+void
+vmod_init(struct sess *sp, struct vmod_priv *priv, const char *param)
+{
+	struct binlog_config *conf = priv->priv;
+	struct stat st;
+	char *dir, *p, *q;
+	unsigned long n;
+
+	p = findparam(param, "debug");
+	if (p) {
+		conf->debug = atoi(p);
+		free(p);
+	}
+	
+	dir = findparam(param, "dir");
+	if (!dir) {
+		binlog_error("parameter \"dir\" not set");
+		abort();
+	}
+	if (stat(dir, &st)) {
+		if (errno == ENOENT)
+			binlog_error("logging directory does not exist");
+		else
+			binlog_error("cannot stat logging directory %s: %s",
+				     dir, strerror(errno));
+		abort();
+	}
+	if (!S_ISDIR(st.st_mode)) {
+		binlog_error("%s exists, but is not a directory");
+		abort();
+	}
+	conf->dd = open(dir, O_SEARCH | O_DIRECTORY);
+	if (conf->dd == -1) {
+		binlog_error("cannot open directory %s: %s",
+			     dir, strerror(errno));
+		abort();
+	}
+	conf->dir = dir;
+
+	p = findparam(param, "pattern");
+	if (!p) {
+		p = strdup(BINLOG_PATTERN);
+		AN(p);
+	}
+	conf->pattern = p;
+	
+	p = findparam(param, "size");
+	if (p) {
+		uintmax_t u;
+		
+		errno = 0;
+		u = strtoul(p, &q, 10);
+		if (errno) {
+			binlog_error("invalid size value");
+			abort();
+		}
+		free(p);
+		switch (*q) {
+		case 'g':
+		case 'G':
+			u <<= 10;
+		case 'm':
+		case 'M':
+			u <<= 10;
+		case 'k':
+		case 'K':
+			u <<= 10;
+		}
+		if (u > SIZE_T_MAX) {
+			binlog_error("invalid size value");
+			abort();
+		}
+		conf->size = u;
+	} else
+		conf->size = BINLOG_SIZE;
+
+	p = findparam(param, "interval");
+	if (p) {
+		conf->interval = getinterval(p, &q);
+		free(p);
+		if (*q) {
+			binlog_error("invalid interval (near \"%s\")", q);
+			abort();
+		}
+	} else
+		conf->interval = BINLOG_INTERVAL;
+
+	p = findparam(param, "umask");
+	if (p) {
+		n = strtoul(p, &q, 8);
+		free(p);
+		if (n & ~0777) {
+			binlog_error("umask out of range");
+			abort();
+		}
+		if (*q) {
+			binlog_error("invalid umask (near \"%s\")", q);
+			abort();
+		}
+	} else
+		conf->umask = BINLOG_UMASK;
+	
+	p = findparam(param, "roundts");
+	if (p) {
+		if (atoi(p))
+			conf->flags |= BLF_ROUNDTS;
+		else
+			conf->flags &= ~BLF_ROUNDTS;
+		free(p);
+	}
+	
+	conf->fd = -1;
+	conf->base = NULL;
+	conf->stoptime = time(NULL);
+	pthread_mutex_init(&conf->mutex, NULL);
+}
+
+static char *
+mkfilename(struct sess *sp, struct binlog_config *conf)
+{
+	time_t ts = time(NULL);
+	size_t u, n;
+	char *p, *q;
+
+	if (conf->flags & BLF_ROUNDTS)
+		ts -= ts % conf->interval;
+	u = WS_Reserve(sp->wrk->ws, 0);
+        p = sp->wrk->ws->f;
+        n = strftime(p, u, conf->pattern, gmtime(&ts));
+	if (n == 0) {
+		WS_Release(sp->wrk->ws, 0);
+		return NULL;
+	}
+	q = strdup(p);
+	AN(q);
+	WS_Release(sp->wrk->ws, 0);
+	return q;
+}
+
+static int
+mkdir_p(struct binlog_config *conf, char *dir)
+{
+	int rc = 0;
+	char *p;
+	struct stat st;
+	
+	for (p = dir; rc == 0 && *p; p++) {
+		if (*p != '/')
+			continue;
+		*p = 0;
+		rc = fstatat(conf->dd, dir, &st, 0);
+		if (rc) {
+			if (errno == ENOENT) {
+				rc = mkdirat(conf->dd, dir,
+					     0777 & ~conf->umask);
+				if (rc)
+					binlog_error("cannot create %s: %s",
+						     dir,
+						     strerror(errno));
+			} else
+				binlog_error("cannot stat %s: %s", dir,
+					     strerror(errno));
+		} else if (!S_ISDIR(st.st_mode)) {
+			binlog_error("component \"%s\" is not a directory",
+				     dir);
+			rc = -1;
+		}
+		*p = '/';
+	}
+	return rc;
+}
+
+static int
+createfile(struct sess *sp, struct binlog_config *conf)
+{
+	char *fname;
+	int fd;
+
+	conf->fname = NULL;
+	conf->fd = -1;
+	
+	fname = mkfilename(sp, conf);
+	if (!fname)
+		return -1;
+	if (mkdir_p(conf, fname)) {
+		free(fname);
+		return -1;
+	}
+
+	fd = openat(conf->dd, fname, O_CREAT|O_RDWR|O_TRUNC,
+		    0666 & ~conf->umask);
+	if (fd == -1) {
+		binlog_error("cannot create log file %s/%s: %s",
+			     conf->dir, fname, strerror(errno));
+		free(fname);
+	}
+
+	conf->fname = fname;
+	conf->fd = fd;
+	return 0;
+}
+
+static void
+reset(struct binlog_config *conf)
+{
+	conf->fname = NULL;
+	conf->fd = -1;
+	conf->base = NULL;
+	conf->recbase = NULL;
+	conf->recnum = 0;
+	conf->recidx = 0;
+}
+
+static int
+setstoptime(struct binlog_config *conf)
+{
+	time_t ts;
+	
+	ts = time(NULL);
+	conf->stoptime = ts - ts % conf->interval + conf->interval;
+}
+
+static int
+newfile(struct sess *sp, struct binlog_config *conf)
+{
+	int c;
+	void *base;	
+	
+	setstoptime(conf);
+		
+	if (createfile(sp, conf))
+		return -1;
+	if (lseek(conf->fd, conf->size, SEEK_SET) == -1) {
+		binlog_error("seek in log file %s/%s failed: %s",
+			     conf->dir, conf->fname, strerror(errno));
+		unlinkat(conf->dd, conf->fname, 0);
+		close(conf->fd);
+		free(conf->fname);
+		reset(conf);
+		return -1;
+	}
+	c = 0;
+	write(conf->fd, &c, 1);
+	base = mmap((caddr_t)0, conf->size,
+		    PROT_READ|PROT_WRITE, MAP_SHARED,
+		    conf->fd, 0);
+	if (base == MAP_FAILED) {
+		binlog_error("mmap: %s", strerror(errno));
+		unlinkat(conf->dd, conf->fname, 0);
+		close(conf->fd);
+		free(conf->fname);
+		reset(conf);
+		return -1;
+	}
+
+	conf->base = base;
+	memcpy(conf->base->hdr.magic, BINLOG_MAGIC_STR, BINLOG_MAGIC_LEN);
+	conf->base->hdr.version = BINLOG_VERSION;
+	conf->base->hdr.recsize = sizeof(struct binlog_record);
+	conf->base->hdr.recnum = 0;
+
+	conf->recbase = (struct binlog_record *) (conf->base + 1);
+	conf->recnum = binlog_recnum(conf);
+	conf->recidx = 0;
+
+	debug(conf,1,("created new log file %s",conf->fname));
+	return 0;
+}
+	
+static void
+closefile(struct sess *sp, struct binlog_config *conf)
+{
+	if (conf->fd == -1)
+		return;
+	debug(conf,1,("closing log file %s",conf->fname));
+	munmap(conf->base, conf->size);
+	if (ftruncate(conf->fd, binlog_size(conf)))
+		binlog_error("error truncating \"%s/%s\": %s",
+			     conf->dir, conf->fname, strerror(errno));
+	close(conf->fd);
+	free(conf->fname);
+	reset(conf);
+}
+	
+
+void
+vmod_append(struct sess *sp, struct vmod_priv *priv, int nid, int aid)
+{
+	struct binlog_config *conf = priv->priv;
+	time_t ts;
+
+	if (!conf)
+		return;
+
+	ts = time(NULL);
+	
+	if (ts >= conf->stoptime) {
+		AZ(pthread_mutex_lock(&conf->mutex));
+		closefile(sp, conf);
+		newfile(sp, conf);
+		AZ(pthread_mutex_unlock(&conf->mutex));
+	}
+	if (conf->fd == -1)
+		return;
+
+	AZ(pthread_mutex_lock(&conf->mutex));
+	if (conf->recidx == conf->recnum) {
+		binlog_error("overflow of %s/%s", conf->dir, conf->fname);
+	} else {
+		struct binlog_record *p = conf->recbase + conf->recidx++;
+		p->nid = nid;
+		p->aid = aid;
+		p->ts = ts;
+		conf->base->hdr.recnum++;
+	}
+	AZ(pthread_mutex_unlock(&conf->mutex));
+}
+
+void
+vmod_sappend(struct sess *sp, struct vmod_priv *priv, const char *nid,
+	     const char *aid)
+{
+	vmod_append(sp, priv, nid ? atoi(nid): 0, aid ? atoi(aid) : 0);
+}
+
+void
+vmod_sync(struct sess *sp, struct vmod_priv *priv)
+{
+	struct binlog_config *conf = priv->priv;
+
+	if (!conf)
+		return;
+
+	AZ(pthread_mutex_lock(&conf->mutex));
+	if (conf->base)
+		msync(conf->base, binlog_size(conf), 0);
+	AZ(pthread_mutex_unlock(&conf->mutex));
+}	
+
+void
+vmod_close(struct sess *sp, struct vmod_priv *priv)
+{
+	struct binlog_config *conf = priv->priv;
+	
+	if (!conf)
+		return;
+
+	mutex = conf->mutex;
+	AZ(pthread_mutex_lock(&mutex));
+	closefile(sp, conf);
+	close(conf->dd);
+	free(conf->dir);
+	free(conf->pattern);
+	free(conf);
+	AZ(pthread_mutex_unlock(&mutex));
+}	
+	
diff --git a/src/binlogcat.c b/src/binlogcat.c
new file mode 100644
index 0000000..c51125b
--- /dev/null
+++ b/src/binlogcat.c
@@ -0,0 +1,134 @@
+/* This file is part of vmod-binlog
+   Copyright (C) 2013 Sergey Poznyakoff
+
+   Vmod-binlog is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3, or (at your option)
+   any later version.
+
+   Vmod-binlog is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with vmod-binlog.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include <config.h>
+#include <unistd.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <errno.h>
+#include <time.h>
+#include <string.h>
+#include "vmod-binlog.h"
+
+char *progname;
+char *timefmt = "%c";
+int number_option;
+int verbose_option;
+
+void
+catlog(const char *fname)
+{
+	FILE *fp;
+	union binlog_header header;
+	struct binlog_record rec;
+	char timebuf[128];
+	size_t i;
+		
+	if (strcmp(fname, "-") == 0)
+		fp = stdin;
+	else {
+		fp = fopen(fname, "r");
+		if (!fp) {
+			fprintf(stderr, "%s: cannot open %s: %s\n",
+				progname, fname, strerror(errno));
+			exit(1);
+		}
+	}
+
+	if (fread(&header, sizeof(header), 1, fp) != 1) {
+		fprintf(stderr, "%s: error reading header of %s: %s\n",
+			progname, fname, strerror(errno));
+		exit(1);
+	}
+
+	if (memcmp(header.hdr.magic, BINLOG_MAGIC_STR, BINLOG_MAGIC_LEN)) {
+		fprintf(stderr, "%s: %s is not a binlog file\n",
+			progname, fname);
+		exit(1);
+	}
+
+	if (header.hdr.version != BINLOG_VERSION) {
+		fprintf(stderr, "%s: %s: unknown version\n",
+			progname, fname);
+		exit(1);
+	}
+
+	if (header.hdr.recsize != sizeof(struct binlog_record)) {
+		fprintf(stderr, "%s: %s: record length mismatch\n",
+			progname, fname);
+		exit(1);
+	}
+
+	if (verbose_option)
+		printf("# %s; %lu records\n", fname, header.hdr.recnum);
+	
+	for (i = 0; i < header.hdr.recnum; i++) {
+		if (fread(&rec, sizeof(rec), 1, fp) != 1) {
+			fprintf(stderr, "%s: %s: unexpected eof\n",
+				progname, fname);
+			break;
+		}
+
+		strftime(timebuf, sizeof timebuf, timefmt, localtime(&rec.ts));
+		if (number_option)
+			printf("%lu ", (unsigned long) i);
+		printf("%s %ld %ld\n", timebuf, rec.nid, rec.aid);
+	}
+	
+	fclose(fp);
+}
+
+void
+help()
+{
+	printf("usage: %s [-hnv] [t FORMAT] [FILE...]\n");
+}
+
+int
+main(int argc, char **argv)
+{
+	progname = argv[0];
+	int c;
+	
+	while ((c = getopt(argc, argv, "ht:nv")) != EOF)
+		switch (c) {
+		case 'h':
+			help();
+			return 0;
+		case 't':
+			timefmt = optarg;
+			break;
+		case 'n':
+			number_option = 1;
+			break;
+		case 'v':
+			verbose_option = 1;
+			break;
+		default:
+			exit(1);
+		}
+
+	argc -= optind;
+	argv += optind;
+	
+	if (argc == 0)
+		catlog("-");
+	else while (argc--)
+		     catlog(*(argv++));
+	return 0;
+}
+
diff --git a/src/vmod-binlog.h b/src/vmod-binlog.h
new file mode 100644
index 0000000..db6c7f0
--- /dev/null
+++ b/src/vmod-binlog.h
@@ -0,0 +1,71 @@
+/* This file is part of vmod-binlog
+   Copyright (C) 2013 Sergey Poznyakoff
+
+   Vmod-binlog is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3, or (at your option)
+   any later version.
+
+   Vmod-binlog is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with vmod-binlog.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include <stdint.h>
+
+#define SIZE_T_MAX ((size_t)~0)
+
+#ifndef BINLOG_SIZE
+# define BINLOG_SIZE (1024*1024*1024)
+#endif
+
+#ifndef BINLOG_PATTERN
+# define BINLOG_PATTERN "%Y/%m/%d.log"
+#endif
+
+#ifndef BINLOG_INTERVAL
+# define BINLOG_INTERVAL 86400
+#endif
+
+#ifndef BINLOG_UMASK
+# define BINLOG_UMASK 0077
+#endif
+
+struct binlog_record {
+	long nid;            /* node ID */
+	long aid;            /* article ID */  
+	time_t ts;           /* timestamp */
+};
+
+#define BINLOG_MAGIC_STR "NXCBINLOG"
+#define BINLOG_MAGIC_LEN (sizeof(BINLOG_MAGIC_STR) - 1)
+#define BINLOG_VERSION 0x00010000UL
+
+struct binlog_file_header {
+	char magic[BINLOG_MAGIC_LEN];
+	char pad[16 - BINLOG_MAGIC_LEN];
+	uint32_t version;
+	size_t recsize;
+	size_t recnum;
+};
+
+#define BINLOG_HEADER_SIZE \
+   ((sizeof(struct binlog_file_header) + sizeof(struct binlog_record) - 1) / \
+    sizeof(struct binlog_record))
+
+union binlog_header {
+	struct binlog_file_header hdr;
+	struct binlog_record pad[BINLOG_HEADER_SIZE];
+};
+
+#define binlog_size(conf) \
+	(sizeof(union binlog_header) + \
+	 (conf)->recidx * sizeof(struct binlog_record))
+#define binlog_recnum(conf) \
+	(((conf)->size - sizeof(union binlog_header)) / \
+	 sizeof(struct binlog_record))
+
diff --git a/src/vmod.vcc b/src/vmod.vcc
new file mode 100644
index 0000000..def0ec3
--- /dev/null
+++ b/src/vmod.vcc
@@ -0,0 +1,7 @@
+Module binlog
+Init module_init
+Function VOID init(PRIV_VCL, STRING)
+Function VOID append(PRIV_VCL, INT, INT)
+Function VOID sappend(PRIV_VCL, STRING, STRING)
+Function VOID sync(PRIV_VCL)
+Function VOID close(PRIV_VCL)