Change default log naming.

* configure.ac: Call AM_PROG_CC_C_O
* src/binlog.c (BLF_TRUNCATE): New flag.
(vmod_init): Change pattern initialization.
New parameter "reuselog".
(createfile): Remove O_TRUNC.
(checkheader): New function.
(newfile): Reuse existing file, if it is the first file
to be opened after varnish startup and its header matches
exactly our data.
* src/binlogsel.c: Use indexed directory structure to speed up
searches.
* src/vmod-binlog.h (BINLOG_PATTERN): Change pattern.
(BINLOG_GLOB_PATTERN,BINLOG_INDEX): New defines.

1 files changed, 164 insertions, 17 deletions

diff --git a/src/binlog.c b/src/binlog.c
index 765b945..e7ac7e5 100644
--- a/src/binlog.c
+++ b/src/binlog.c
@@ -34,13 +34,14 @@
 #include "pack.h"
 
 #ifndef O_SEARCH
 # define O_SEARCH 0
 #endif
 
-#define BLF_ROUNDTS 0x01
+#define BLF_ROUNDTS  0x01
+#define BLF_TRUNCATE 0x02
 
 enum binlog_state {
 	state_init,
 	state_start,
 	state_pack
 };
@@ -180,21 +181,45 @@ getinterval(char *p, char **endp)
 			return (hours*60 + minutes)*60 + seconds;
 		}
 		p++;
 	}
 }
 
+static struct indexdef {
+	char *name;
+	char *pat;
+} indextab[] = {
+	{ "year",   "%Y" },
+	{ "0",      "%Y" },
+	{ "month",  "%Y/%m" },
+	{ "1",      "%Y/%m" },
+	{ "day",    "%Y/%m/%d" },
+	{ "2",      "%Y/%m/%d" },
+	{ NULL }
+};
+
+static char *
+getindexpat(const char *name)
+{
+	struct indexdef *p;
+	for (p = indextab; p->name; p++)
+		if (strcmp(p->name, name) == 0)
+			return p->pat;
+	return NULL;
+}
+
 void
 vmod_init(struct sess *sp, struct vmod_priv *priv,
 	  const char *dir, const char *dataspec, const char *param)
 {
 	struct binlog_config *conf = priv->priv;
 	struct stat st;
 	char *p, *q;
 	unsigned long n;
-
+	int user_pattern = 0;
+	
 	p = findparam(param, "debug");
 	if (p) {
 		conf->debug = atoi(p);
 		free(p);
 	}
 
@@ -235,14 +260,38 @@ vmod_init(struct sess *sp, struct vmod_priv *priv,
 	AN(conf->dataspec);
 		
 	p = findparam(param, "pattern");
 	if (!p) {
 		p = strdup(BINLOG_PATTERN);
 		AN(p);
-	}
+	} else
+		user_pattern = 1;
 	conf->pattern = p;
+
+	p = findparam(param, "index");
+	if (p) {
+		q = getindexpat(p);
+		if (!q) {
+			binlog_error("invalid index type");
+			abort();
+		}
+	} else if (!user_pattern) {
+		q = getindexpat(BINLOG_INDEX);
+		AN(q);
+	} else
+		q = NULL;
+	
+	if (q) {
+		p = malloc(strlen(q) + strlen(conf->pattern) + 2);
+		AN(p);
+		strcpy(p, q);
+		strcat(p, "/");
+		strcat(p, conf->pattern);
+		free(conf->pattern);
+		conf->pattern = p;
+	}
 	
 	p = findparam(param, "size");
 	if (p) {
 		uintmax_t u;
 		
 		errno = 0;
@@ -302,12 +351,21 @@ vmod_init(struct sess *sp, struct vmod_priv *priv,
 		if (atoi(p))
 			conf->flags |= BLF_ROUNDTS;
 		else
 			conf->flags &= ~BLF_ROUNDTS;
 		free(p);
 	}
+
+	p = findparam(param, "reuselog");
+	if (p) {
+		if (atoi(p))
+			conf->flags &= ~BLF_TRUNCATE;
+		else
+			conf->flags |= BLF_TRUNCATE;
+		free(p);
+	}
 	
 	conf->fd = -1;
 	conf->base = NULL;
 	conf->stoptime = time(NULL);
 	pthread_mutex_init(&conf->mutex, NULL);
 }
@@ -381,13 +439,13 @@ createfile(struct sess *sp, struct binlog_config *conf)
 		return -1;
 	if (mkdir_p(conf, fname)) {
 		free(fname);
 		return -1;
 	}
 
-	fd = openat(conf->dd, fname, O_CREAT|O_RDWR|O_TRUNC,
+	fd = openat(conf->dd, fname, O_CREAT|O_RDWR,
 		    0666 & ~conf->umask);
 	if (fd == -1) {
 		binlog_error("cannot create log file %s/%s: %s",
 			     conf->dir, fname, strerror(errno));
 		free(fname);
 	}
@@ -417,60 +475,149 @@ setstoptime(struct binlog_config *conf)
 }
 
 #define binlog_recnum(conf) \
 	(((conf)->size - (conf)->base->hdrsize) / (conf)->base->recsize)
 
 static int
+checkheader(struct binlog_config *conf, size_t hdrsize)
+{
+	struct binlog_file_header header;
+	int c;
+	ssize_t rc;
+	char *p;
+	
+	rc = read(conf->fd, &header, sizeof(header));
+	if (rc == -1) {
+		binlog_error("error reading header of %s/%s: %s",
+			     conf->dir, conf->fname, strerror(errno));
+		return -1;
+	} else if (rc != sizeof(header)) {
+		binlog_error("error reading header of %s/%s: %s",
+			     conf->dir, conf->fname, "hit eof");
+		return -1;
+	}
+
+	if (memcmp(header.magic, BINLOG_MAGIC_STR, BINLOG_MAGIC_LEN)) {
+		binlog_error("%s/%s is not a binlog file",
+			     conf->dir, conf->fname);
+		return -1;
+	}
+
+	if (header.version != BINLOG_VERSION) {
+		binlog_error("%s/%s: unknown version", conf->dir, conf->fname);
+		return -1;
+	}
+
+	if (header.hdrsize != hdrsize) {
+		debug(conf,1,("%s/%s: header size mismatch",
+			      conf->dir, conf->fname));
+		return 1;
+	}
+	if (header.recsize != conf->recsize) {
+		debug(conf,1,("%s/%s: record size mismatch",
+			      conf->dir, conf->fname));
+		return 1;
+	}
+	
+	p = conf->dataspec;
+	while (*p) {
+		if (read(conf->fd, &c, 1) != 1 || c != *p) {
+			debug(conf,1,("%s/%s: dataspec mismatch near %s: %c",
+				      conf->dir, conf->fname, p, c));
+			return 1;
+		}
+		++p;
+	}
+	if (read(conf->fd, &c, 1) != 1 || c != 0) {
+		debug(conf,1,("%s/%s: dataspec mismatch at the end: %c",
+				      conf->dir, conf->fname, c));
+		return 1;
+	}
+	return 0;
+}
+
+static int
 newfile(struct sess *sp, struct binlog_config *conf)
 {
 	int c;
 	void *base;	
-	size_t n;
+	size_t hdrsize;
+	struct stat st;
+	int reuse = 0;
 	
 	setstoptime(conf);
 		
 	if (createfile(sp, conf))
 		return -1;
+
+	hdrsize = ((sizeof(struct binlog_file_header) +
+		    strlen(conf->dataspec) +
+		    conf->recsize - 1) / conf->recsize) * conf->recsize;
+
+	if (fstat(conf->fd, &st) == 0) {
+		/* File already exists */
+		if (st.st_size > 0 &&
+		    !(conf->flags & BLF_TRUNCATE) &&
+		    checkheader(conf, hdrsize) == 0) {
+			reuse = 1;
+		} else {
+			binlog_error("truncating existing file %s/%s",
+				     conf->dir, conf->fname);
+			ftruncate(conf->fd, 0);
+		}
+	} else {
+		binlog_error("can't stat %s/%s: %s",
+			     conf->dir, conf->fname, strerror(errno));
+		/* try to continue anyway */
+	}
+	conf->flags |= BLF_TRUNCATE;
+	
 	if (lseek(conf->fd, conf->size, SEEK_SET) == -1) {
 		binlog_error("seek in log file %s/%s failed: %s",
 			     conf->dir, conf->fname, strerror(errno));
-		unlinkat(conf->dd, conf->fname, 0);
+		if (!reuse)
+			unlinkat(conf->dd, conf->fname, 0);
 		close(conf->fd);
 		free(conf->fname);
 		reset(conf);
 		return -1;
 	}
 	c = 0;
 	write(conf->fd, &c, 1);
 	base = mmap((caddr_t)0, conf->size,
 		    PROT_READ|PROT_WRITE, MAP_SHARED,
 		    conf->fd, 0);
 	if (base == MAP_FAILED) {
 		binlog_error("mmap: %s", strerror(errno));
-		unlinkat(conf->dd, conf->fname, 0);
+		if (!reuse)
+			unlinkat(conf->dd, conf->fname, 0);
 		close(conf->fd);
 		free(conf->fname);
 		reset(conf);
 		return -1;
 	}
 
 	conf->base = base;
-	memcpy(conf->base->magic, BINLOG_MAGIC_STR, BINLOG_MAGIC_LEN);
-	conf->base->version = BINLOG_VERSION;
-	conf->base->recsize = conf->recsize;
-	conf->base->recnum = 0;
-	strcpy((char*)(conf->base + 1), conf->dataspec);
-
-	n = (sizeof(struct binlog_file_header) + strlen(conf->dataspec) +
-	     conf->recsize - 1) / conf->recsize;
-	conf->base->hdrsize = n * conf->recsize;
 
+	if (reuse) {
+		debug(conf,1,("reusing log file %s, recnum=%lu",
+			      conf->fname, (unsigned long)conf->base->recnum));
+	} else {
+		debug(conf,1,("created new log file %s",conf->fname));
+		memcpy(conf->base->magic, BINLOG_MAGIC_STR, BINLOG_MAGIC_LEN);
+		conf->base->version = BINLOG_VERSION;
+		conf->base->recsize = conf->recsize;
+		conf->base->recnum = 0;
+		strcpy((char*)(conf->base + 1), conf->dataspec);
+
+		conf->base->hdrsize = hdrsize;
+	}
+	
 	conf->recbase = (char *) conf->base + conf->base->hdrsize;
 	conf->recnum = binlog_recnum(conf);
 
-	debug(conf,1,("created new log file %s",conf->fname));
 	return 0;
 }
 	
 static void
 closefile(struct sess *sp, struct binlog_config *conf)
 {