diff options
-rw-r--r-- | src/.gitignore | 1 | ||||
-rw-r--r-- | src/Makefile.am | 6 | ||||
-rw-r--r-- | src/binlog.c | 161 | ||||
-rw-r--r-- | src/binlogcat.c | 71 | ||||
-rw-r--r-- | src/pack.c | 588 | ||||
-rw-r--r-- | src/pack.h | 47 | ||||
-rw-r--r-- | src/vmod-binlog.h | 21 | ||||
-rw-r--r-- | src/vmod.vcc | 7 | ||||
-rw-r--r-- | tests/test01.at | 9 | ||||
-rw-r--r-- | tests/test02.at | 9 |
10 files changed, 843 insertions, 77 deletions
diff --git a/src/.gitignore b/src/.gitignore index 7f6e438..a75fe6d 100644 --- a/src/.gitignore +++ b/src/.gitignore @@ -1,2 +1,3 @@ +binlogcat vcc_if.c vcc_if.h diff --git a/src/Makefile.am b/src/Makefile.am index c9daf9a..9ce519f 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -18,5 +18,6 @@ AM_CPPFLAGS = -I$(VARNISHSRC)/include -I$(VARNISHSRC) bin_PROGRAMS = binlogcat -binlogcat_SOURCES = binlogcat.c +binlogcat_SOURCES = binlogcat.c pack.c +binlogcat_CFLAGS = $(AM_CFLAGS) vmoddir = $(VMODDIR) @@ -28,7 +29,10 @@ libvmod_binlog_la_LIBADD= libvmod_binlog_la_SOURCES = \ binlog.c\ + pack.c\ vmod-binlog.h\ vcc_if.c vcc_if.h +noinst_HEADERS = pack.h + BUILT_SOURCES = vcc_if.c vcc_if.h diff --git a/src/binlog.c b/src/binlog.c index be32e10..6f438a8 100644 --- a/src/binlog.c +++ b/src/binlog.c @@ -23,4 +23,5 @@ #include <errno.h> #include <syslog.h> +#include <stddef.h> #include <stdlib.h> #include <ctype.h> @@ -30,4 +31,5 @@ #include "bin/varnishd/cache.h" #include "vmod-binlog.h" +#include "pack.h" #ifndef O_SEARCH @@ -37,4 +39,10 @@ #define BLF_ROUNDTS 0x01 +enum binlog_state { + state_init, + state_start, + state_pack +}; + struct binlog_config { size_t size; /* maximum file size */ @@ -46,12 +54,19 @@ struct binlog_config { char *fname; /* current file name */ int fd; /* current file descriptor */ - union binlog_header *base; /* mmap base */ - struct binlog_record *recbase; /* record base */ + struct binlog_file_header *base; /* mmap base */ + char *recbase; /* record base */ size_t recnum; /* number of records in recbase */ - size_t recidx; /* index of the next free entry in recbase */ + size_t recsize; /* record size */ time_t stoptime; /* when to rotate the current file */ pthread_mutex_t mutex; int debug; int flags; + + char *dataspec; + struct packinst *inst_head; + struct packinst *inst_cur; + struct packenv *env; + enum binlog_state state; + time_t timestamp; }; @@ -159,9 +174,10 @@ getinterval(char *p, char **endp) void -vmod_init(struct sess *sp, struct vmod_priv *priv, const char *param) +vmod_init(struct sess *sp, struct vmod_priv *priv, + const char *dir, const char *dataspec, const char *param) { struct binlog_config *conf = priv->priv; struct stat st; - char *dir, *p, *q; + char *p, *q; unsigned long n; @@ -171,10 +187,5 @@ vmod_init(struct sess *sp, struct vmod_priv *priv, const char *param) free(p); } - - dir = findparam(param, "dir"); - if (!dir) { - binlog_error("parameter \"dir\" not set"); - abort(); - } + if (stat(dir, &st)) { if (errno == ENOENT) @@ -195,6 +206,18 @@ vmod_init(struct sess *sp, struct vmod_priv *priv, const char *param) abort(); } - conf->dir = dir; + conf->dir = strdup(dir); + AN(conf->dir); + conf->inst_head = packcomp(dataspec, &p); + if (*p) { + binlog_error("cannot compile data format near %s", p); + abort(); + } + conf->recsize = packsize(conf->inst_head); + conf->env = packenv_create(conf->recsize); + conf->recsize += offsetof(struct binlog_record,data); + conf->dataspec = strdup(dataspec); + AN(conf->dataspec); + p = findparam(param, "pattern"); if (!p) { @@ -368,5 +391,4 @@ reset(struct binlog_config *conf) conf->recbase = NULL; conf->recnum = 0; - conf->recidx = 0; } @@ -380,4 +402,7 @@ setstoptime(struct binlog_config *conf) } +#define binlog_recnum(conf) \ + (((conf)->size - (conf)->base->hdrsize) / (conf)->base->recsize) + static int newfile(struct sess *sp, struct binlog_config *conf) @@ -385,4 +410,5 @@ newfile(struct sess *sp, struct binlog_config *conf) int c; void *base; + size_t n; setstoptime(conf); @@ -414,12 +440,16 @@ newfile(struct sess *sp, struct binlog_config *conf) conf->base = base; - memcpy(conf->base->hdr.magic, BINLOG_MAGIC_STR, BINLOG_MAGIC_LEN); - conf->base->hdr.version = BINLOG_VERSION; - conf->base->hdr.recsize = sizeof(struct binlog_record); - conf->base->hdr.recnum = 0; + memcpy(conf->base->magic, BINLOG_MAGIC_STR, BINLOG_MAGIC_LEN); + conf->base->version = BINLOG_VERSION; + conf->base->recsize = conf->recsize; + conf->base->recnum = 0; + strcpy((char*)(conf->base + 1), conf->dataspec); - conf->recbase = (struct binlog_record *) (conf->base + 1); + n = (sizeof(struct binlog_file_header) + strlen(conf->dataspec) + + conf->recsize - 1) / conf->recsize; + conf->base->hdrsize = n * conf->recsize; + + conf->recbase = (char *) conf->base + conf->base->hdrsize; conf->recnum = binlog_recnum(conf); - conf->recidx = 0; debug(conf,1,("created new log file %s",conf->fname)); @@ -430,9 +460,12 @@ static void closefile(struct sess *sp, struct binlog_config *conf) { + size_t size; + if (conf->fd == -1) return; debug(conf,1,("closing log file %s",conf->fname)); + size = binlog_size(conf->base); munmap(conf->base, conf->size); - if (ftruncate(conf->fd, binlog_size(conf))) + if (ftruncate(conf->fd, size)) binlog_error("error truncating \"%s/%s\": %s", conf->dir, conf->fname, strerror(errno)); @@ -441,8 +474,7 @@ closefile(struct sess *sp, struct binlog_config *conf) reset(conf); } - void -vmod_append(struct sess *sp, struct vmod_priv *priv, int nid, int aid) +vmod_start(struct sess *sp, struct vmod_priv *priv) { struct binlog_config *conf = priv->priv; @@ -460,25 +492,82 @@ vmod_append(struct sess *sp, struct vmod_priv *priv, int nid, int aid) AZ(pthread_mutex_unlock(&conf->mutex)); } + + packenv_init(conf->env); + conf->state = state_start; + conf->inst_cur = conf->inst_head; + conf->timestamp = ts; +} + +void +vmod_pack(struct sess *sp, struct vmod_priv *priv, const char *str) +{ + struct binlog_config *conf = priv->priv; + char *argv[2]; + + if (!conf) + return; + + switch (conf->state) { + case state_start: + case state_pack: + break; + default: + binlog_error("pack called in wrong state (%d)", conf->state); + return; + } + + if (!conf->inst_cur) { + binlog_error("format spec exhausted"); + return; + } + + argv[0] = (char*) str; + argv[1] = NULL; + conf->env->argv = argv; + conf->env->argc = 2; + conf->env->argi = 0; + + conf->inst_cur = packinnext(conf->inst_cur, conf->env); + + conf->state = state_pack; +} + +void +vmod_commit(struct sess *sp, struct vmod_priv *priv) +{ + struct binlog_config *conf = priv->priv; + + if (!conf) + return; if (conf->fd == -1) return; + switch (conf->state) { + case state_start: + binlog_error("committing empty binlog record"); + break; + case state_pack: + if (conf->inst_cur) + binlog_error("committing incomplete binlog record"); + break; + default: + binlog_error("pack called in wrong state (%d)", conf->state); + return; + } + AZ(pthread_mutex_lock(&conf->mutex)); - if (conf->recidx == conf->recnum) { + if (conf->base->recnum == conf->recnum) { binlog_error("overflow of %s/%s", conf->dir, conf->fname); } else { - struct binlog_record *p = conf->recbase + conf->recidx++; - p->nid = nid; - p->aid = aid; - p->ts = ts; - conf->base->hdr.recnum++; + struct binlog_record *p = + (struct binlog_record *)(conf->recbase + + conf->base->recnum * + conf->recsize); + p->ts = conf->timestamp; + memcpy(p->data, conf->env->buf_base, conf->env->buf_size); + conf->base->recnum++; } AZ(pthread_mutex_unlock(&conf->mutex)); -} - -void -vmod_sappend(struct sess *sp, struct vmod_priv *priv, const char *nid, - const char *aid) -{ - vmod_append(sp, priv, nid ? atoi(nid): 0, aid ? atoi(aid) : 0); + conf->state = state_init; } @@ -493,5 +582,5 @@ vmod_sync(struct sess *sp, struct vmod_priv *priv) AZ(pthread_mutex_lock(&conf->mutex)); if (conf->base) - msync(conf->base, binlog_size(conf), 0); + msync(conf->base, binlog_size(conf->base), 0); AZ(pthread_mutex_unlock(&conf->mutex)); } diff --git a/src/binlogcat.c b/src/binlogcat.c index 219f78c..ee8837a 100644 --- a/src/binlogcat.c +++ b/src/binlogcat.c @@ -18,4 +18,5 @@ #include <config.h> #include <unistd.h> +#include <stddef.h> #include <stdio.h> #include <stdlib.h> @@ -24,4 +25,5 @@ #include <string.h> #include "vmod-binlog.h" +#include "pack.h" char *progname; @@ -35,9 +37,14 @@ catlog(const char *fname) { FILE *fp; - union binlog_header header; - struct binlog_record rec; + struct binlog_file_header header; + struct binlog_record *rec; char timebuf[128]; size_t i; time_t start_ts; + char *dataspec; + size_t size; + struct packenv *env; + struct packinst *inst; + char *p; if (strcmp(fname, "-") == 0) @@ -58,5 +65,5 @@ catlog(const char *fname) } - if (memcmp(header.hdr.magic, BINLOG_MAGIC_STR, BINLOG_MAGIC_LEN)) { + if (memcmp(header.magic, BINLOG_MAGIC_STR, BINLOG_MAGIC_LEN)) { fprintf(stderr, "%s: %s is not a binlog file\n", progname, fname); @@ -64,5 +71,5 @@ catlog(const char *fname) } - if (header.hdr.version != BINLOG_VERSION) { + if (header.version != BINLOG_VERSION) { fprintf(stderr, "%s: %s: unknown version\n", progname, fname); @@ -70,15 +77,40 @@ catlog(const char *fname) } - if (header.hdr.recsize != sizeof(struct binlog_record)) { - fprintf(stderr, "%s: %s: record length mismatch\n", - progname, fname); + size = header.hdrsize - sizeof(header); + dataspec = malloc(size); + if (!dataspec) { + fprintf(stderr, "%s: not enough memory", progname); + abort(); + } + + if (fread(dataspec, size, 1, fp) != 1) { + fprintf(stderr, "%s: error reading header of %s: %s\n", + progname, fname, strerror(errno)); exit(1); } if (verbose_option) - printf("# %s; %lu records\n", fname, header.hdr.recnum); + printf("# %s; format=%s; recsize=%lu; recnum=%lu\n", + fname, dataspec, header.recsize, header.recnum); + + inst = packcomp(dataspec, &p); + if (*p) { + fprintf(stderr, "%s: %s: bad dataspec near %s", + progname, dataspec, p); + exit(1); + } + free(dataspec); + + rec = malloc(header.recsize); + if (!rec) { + fprintf(stderr, "%s: not enough memory", progname); + abort(); + } + env = packenv_create(header.recsize - + offsetof(struct binlog_record,data)); + env->fp = stdout; - for (i = 0; i < header.hdr.recnum; i++) { - if (fread(&rec, sizeof(rec), 1, fp) != 1) { + for (i = 0; i < header.recnum; i++) { + if (fread(rec, header.recsize, 1, fp) != 1) { fprintf(stderr, "%s: %s: unexpected eof\n", progname, fname); @@ -88,13 +120,24 @@ catlog(const char *fname) if (timediff_option) { if (i == 0) - start_ts = rec.ts; - rec.ts -= start_ts; + start_ts = rec->ts; + rec->ts -= start_ts; } - strftime(timebuf, sizeof timebuf, timefmt, localtime(&rec.ts)); + strftime(timebuf, sizeof timebuf, timefmt, + localtime(&rec->ts)); if (number_option) printf("%lu ", (unsigned long) i); - printf("%s %ld %ld\n", timebuf, rec.nid, rec.aid); + printf("%s ", timebuf); + + memcpy(env->buf_base, rec->data, env->buf_size); + env->buf_pos = 0; + + packout(inst, env); + fputc('\n', stdout); } + + free(rec); + packenv_free(env); + packfree(inst); fclose(fp); diff --git a/src/pack.c b/src/pack.c new file mode 100644 index 0000000..a6ee933 --- /dev/null +++ b/src/pack.c @@ -0,0 +1,588 @@ +/* This file is part of vmod-binlog + Copyright (C) 2013 Sergey Poznyakoff + + Vmod-binlog is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + Vmod-binlog is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with vmod-binlog. If not, see <http://www.gnu.org/licenses/>. +*/ +#include <config.h> +#include <stdint.h> +#include <stdlib.h> +#include <stdio.h> +#include <string.h> +#include <ctype.h> +#include "pack.h" + +/* Data format specification is a simplified form of Perl pack() template. + It consists of a series of template letters optionally followed by + numeric repeat count (which may be enclosed in square brackets). The + valid template letters are: + + Z A null-terminated (ASCIZ) string of at most N-1 characters, will be + null padded. This letter must be followed by repeat count. + + c A signed char (8-bit) value. + C An unsigned char (octet) value. + + s A signed short (16-bit) value. + S An unsigned short value. + + l A signed long (32-bit) value. + L An unsigned long value. + + q A signed quad (64-bit) value. + Q An unsigned quad value. + + i A signed integer value. + I A unsigned integer value. + + x A null byte (a.k.a ASCII NUL, "\000", chr(0)) + X Back up a byte. + @ Null-fill or truncate to absolute position, counted from the + current position. + . Null-fill or truncate to absolute position specified by + the repeat count. +*/ + +#define F_REP 0x01 + +struct packspec { + int ch; + size_t size; + int flags; + void (*packer)(struct packenv *, struct packspec *, int); + void (*unpacker)(struct packenv *, struct packspec *, int); +}; + +struct packinst { + struct packinst *next; + struct packspec *spec; + int rep; +}; + +static char * +packenv_get(struct packenv *env) +{ + if (env->argi == env->argc) + return NULL; + return env->argv[env->argi++]; +} + +static void +printunum(FILE *fp, uintmax_t num) +{ + char numbuf[512]; + char *p = numbuf + sizeof(numbuf); + *--p = 0; + do + *--p = num % 10 + '0'; + while (num /= 10); + fputs(p, fp); +} + +static void +printsnum(FILE *fp, intmax_t num) +{ + if (num < 0) { + fputc('-', fp); + num = - num; + } + printunum(fp, num); +} + +static void +Z_packer(struct packenv *env, struct packspec *spec, int rep) +{ + if (env->buf_base) { + char *arg = packenv_get(env); + + memset(env->buf_base + env->buf_pos, 0, rep); + if (arg) { + size_t len = strlen(arg); + if (len > rep - 1) + len = rep - 1; + memcpy(env->buf_base + env->buf_pos, arg, len); + } + } + env->buf_pos += rep; +} + +static void +Z_unpacker(struct packenv *env, struct packspec *spec, int rep) +{ + fprintf(env->fp, "%-*.*s", rep, rep, env->buf_base + env->buf_pos); + env->buf_pos += rep; +} + +static void +c_packer(struct packenv *env, struct packspec *spec, int rep) +{ + if (env->buf_base) { + char *arg = packenv_get(env); + if (arg) + env->buf_base[env->buf_pos] = *arg; + } + env->buf_pos++; +} + +static void +c_unpacker(struct packenv *env, struct packspec *spec, int rep) +{ + fprintf(env->fp, "%c", env->buf_base[env->buf_pos++]); +} + +static void +s_packer(struct packenv *env, struct packspec *spec, int rep) +{ + if (env->buf_base) { + char *arg = packenv_get(env); + if (arg) { + int16_t v = atoi(arg); + memcpy(env->buf_base + env->buf_pos, &v, sizeof(v)); + } + } + env->buf_pos += spec->size; +} + +static void +s_unpacker(struct packenv *env, struct packspec *spec, int rep) +{ + printsnum(env->fp, *(int16_t *) (env->buf_base + env->buf_pos)); + env->buf_pos += spec->size; +} + +static void +S_packer(struct packenv *env, struct packspec *spec, int rep) +{ + if (env->buf_base) { + char *arg = packenv_get(env); + if (arg) { + uint16_t v = atoi(arg); + memcpy(env->buf_base + env->buf_pos, &v, sizeof(v)); + } + } + env->buf_pos += spec->size; +} + +static void +S_unpacker(struct packenv *env, struct packspec *spec, int rep) +{ + printunum(env->fp, *(uint16_t *)(env->buf_base + env->buf_pos)); + env->buf_pos += spec->size; +} + +static void +l_packer(struct packenv *env, struct packspec *spec, int rep) +{ + if (env->buf_base) { + char *arg = packenv_get(env); + if (arg) { + int32_t v = atoi(arg); + memcpy(env->buf_base + env->buf_pos, &v, sizeof(v)); + } + } + env->buf_pos += spec->size; +} + +static void +l_unpacker(struct packenv *env, struct packspec *spec, int rep) +{ + printsnum(env->fp, *(int32_t *)(env->buf_base + env->buf_pos)); + env->buf_pos += spec->size; +} + +static void +L_packer(struct packenv *env, struct packspec *spec, int rep) +{ + if (env->buf_base) { + char *arg = packenv_get(env); + if (arg) { + uint32_t v = atoi(arg); + memcpy(env->buf_base + env->buf_pos, &v, sizeof(v)); + } + } + env->buf_pos += spec->size; +} + +static void +L_unpacker(struct packenv *env, struct packspec *spec, int rep) +{ + printunum(env->fp, *(uint32_t *)(env->buf_base + env->buf_pos)); + env->buf_pos += spec->size; +} + +static void +q_packer(struct packenv *env, struct packspec *spec, int rep) +{ + if (env->buf_base) { + char *arg = packenv_get(env); + if (arg) { + int64_t v = atoi(arg); + memcpy(env->buf_base + env->buf_pos, &v, sizeof(v)); + } + } + env->buf_pos += spec->size; +} + +static void +q_unpacker(struct packenv *env, struct packspec *spec, int rep) +{ + printsnum(env->fp, *(int64_t *)(env->buf_base + env->buf_pos)); + env->buf_pos += spec->size; +} + +static void +Q_packer(struct packenv *env, struct packspec *spec, int rep) +{ + if (env->buf_base) { + char *arg = packenv_get(env); + if (arg) { + uint64_t v = atoi(arg); + memcpy(env->buf_base + env->buf_pos, &v, sizeof(v)); + } + } + env->buf_pos += spec->size; +} + +static void +Q_unpacker(struct packenv *env, struct packspec *spec, int rep) +{ + printunum(env->fp, *(uint64_t *)(env->buf_base + env->buf_pos)); + env->buf_pos += spec->size; +} + +static void +i_packer(struct packenv *env, struct packspec *spec, int rep) +{ + if (env->buf_base) { + char *arg = packenv_get(env); + if (arg) { + int v = atoi(arg); + memcpy(env->buf_base + env->buf_pos, &v, sizeof(v)); + } + } + env->buf_pos += spec->size; +} + +static void +i_unpacker(struct packenv *env, struct packspec *spec, int rep) +{ + printsnum(env->fp, *(int *)(env->buf_base + env->buf_pos)); + env->buf_pos += spec->size; +} + +static void +I_packer(struct packenv *env, struct packspec *spec, int rep) +{ + if (env->buf_base) { + char *arg = packenv_get(env); + if (arg) { + unsigned int v = atoi(arg); + memcpy(env->buf_base + env->buf_pos, &v, sizeof(v)); + } + } + env->buf_pos += spec->size; +} + +static void +I_unpacker(struct packenv *env, struct packspec *spec, int rep) +{ + printunum(env->fp, *(unsigned *)(env->buf_base + env->buf_pos)); + env->buf_pos += spec->size; +} + +static void +x_packer(struct packenv *env, struct packspec *spec, int rep) +{ + if (env->buf_base) + env->buf_base[env->buf_pos] = 0; + env->buf_pos++; +} + +static void +x_unpacker(struct packenv *env, struct packspec *spec, int rep) +{ + fputc(0, env->fp); +} + +static void +X_packer(struct packenv *env, struct packspec *spec, int rep) +{ + if (env->buf_pos) + --env->buf_pos; +} + +static void +at_packer(struct packenv *env, struct packspec *spec, int rep) +{ + if (env->buf_base) + memset(env->buf_base + env->buf_pos, 0, rep); + env->buf_pos += rep; +} + +static void +at_unpacker(struct packenv *env, struct packspec *spec, int rep) +{ + env->buf_pos += rep; +} + +static void +dot_packer(struct packenv *env, struct packspec *spec, int rep) +{ + if (env->buf_base) { + if (rep < env->buf_pos) + memset(env->buf_base + rep, 0, env->buf_pos - rep); + else + memset(env->buf_base + env->buf_pos, 0, + rep - env->buf_pos); + } + env->buf_pos = rep; +} + +static void +dot_unpacker(struct packenv *env, struct packspec *spec, int rep) +{ + env->buf_pos = rep; +} + +static struct packspec packspec[] = { + { 'Z', 1, F_REP, Z_packer, Z_unpacker }, + { 'c', 1, 0, c_packer, c_unpacker }, + { 's', sizeof(int16_t), 0, s_packer, s_unpacker }, + { 'S', sizeof(uint16_t), 0, S_packer, S_unpacker }, + { 'l', sizeof(int32_t), 0, l_packer, l_unpacker }, + { 'L', sizeof(uint32_t), 0, L_packer, L_unpacker }, + { 'q', sizeof(int64_t), 0, q_packer, q_unpacker }, + { 'Q', sizeof(uint64_t), 0, Q_packer, Q_unpacker }, + { 'i', sizeof(int), 0, i_packer, i_unpacker }, + { 'I', sizeof(unsigned), 0, I_packer, I_unpacker }, + /* FIXME: n N v V f d */ + { 'x', 1, 0, x_packer, x_unpacker }, + { 'X', 1, 0, X_packer, X_packer }, + { '@', 0, F_REP, at_packer, at_unpacker }, + { '.', 0, F_REP, dot_packer, dot_unpacker }, + + { 0 } +}; + +static int +getrep(const char *s, char const **endp, int *rep) +{ + int n; + char *p; + + if (*s == '[') { + if (s[1]) { + if (isdigit(s[1])) { + n = strtol(s + 1, &p, 10); + if (n <= 0) { + *endp = s; + return -1; + } else if (*p != ']') { + *endp = p; + return -1; + } + *rep = n; + *endp = p + 1; + } else { + *endp = s; + return -1; + } + } else { + *endp = s; + return -1; + } + } else if (isdigit(*s)) { + n = strtol(s, &p, 10); + if (n <= 0) { + *endp = s; + return -1; + } + *rep = n; + *endp = p; + } else { + *rep = 1; + *endp = s; + } + return 0; +} + +struct packinst * +packcomp(const char *s, char **endp) +{ + struct packinst *head = NULL, *tail = NULL, *pi; + struct packspec *ps; + int rep; + + while (s) { + for (ps = packspec; ps->ch; ps++) + if (ps->ch == *s) + break; + if (!ps->ch) + break; + if (getrep(s + 1, &s, &rep)) + break; + pi = malloc(sizeof(*pi)); + pi->next = NULL; + pi->spec = ps; + pi->rep = rep; + if (tail) + tail->next = pi; + else + head = pi; + tail = pi; + } + if (endp) + *endp = (char*) s; + return head; +} + +void +packfree(struct packinst *pi) +{ + while (pi) { + struct packinst *next = pi->next; + free(pi); + pi = next; + } +} + +void +packin(struct packinst *pi, struct packenv *env) +{ + int i; + + for (; pi; pi = pi->next) { + if (pi->spec->flags & F_REP) + pi->spec->packer(env, pi->spec, pi->rep); + else + for (i = 0; i < pi->rep; i++) + pi->spec->packer(env, pi->spec, 1); + } +} + +struct packinst * +packinnext(struct packinst *pi, struct packenv *env) +{ + int i; + + if (!pi) + return NULL; + if (pi->spec->flags & F_REP) + pi->spec->packer(env, pi->spec, pi->rep); + else + for (i = 0; i < pi->rep; i++) + pi->spec->packer(env, pi->spec, 1); + return pi->next; +} + +void +packout(struct packinst *pi, struct packenv *env) +{ + int i; + + for (; pi; pi = pi->next) { + if (pi->spec->flags & F_REP) + pi->spec->unpacker(env, pi->spec, pi->rep); + else + for (i = 0; i < pi->rep; i++) + pi->spec->unpacker(env, pi->spec, 1); + if (pi->next) + fputc(' ', env->fp); + } +} + +size_t +packsize(struct packinst *pi) +{ + struct packenv env; + + memset(&env, 0, sizeof env); + packin(pi, &env); + return env.buf_pos; +} + +struct packenv * +packenv_create(size_t size) +{ + struct packenv *env = calloc(1, sizeof(*env)); + env->buf_base = calloc(1, size); + env->buf_size = size; + return env; +} + +void +packenv_free(struct packenv *env) +{ + if (!env) + return; + free(env->buf_base); + free(env); +} + +void +packenv_init(struct packenv *env) +{ + memset(env->buf_base, 0, env->buf_size); + env->buf_pos = 0; +} + +#ifdef STANDALONE +#include <unistd.h> + +int +main(int argc, char **argv) +{ + void (*fn)(struct packinst *pi, struct packenv *env) = packin; + struct packinst *pi; + struct packenv *env; + char *end; + int c; + + while ((c = getopt(argc, argv, "d")) != EOF) { + switch (c) { + case 'd': + fn = packout; + break; + default: + exit(1); + } + } + + argc -= optind; + argv += optind; + + if (argc == 0) + abort(); + + pi = packcomp(argv[0], &end); + if (*end) { + fprintf(stderr, "compile error near %s\n", end); + exit(1); + } + env = packenv_create(packsize(pi)); + env->fp = stdout; + env->argv = argv + 1; + env->argc = argc - 1; + + if (fn == packout) + fread(env->buf_base, env->buf_size, 1, stdin); + + fn(pi, env); + if (fn == packin) { + fwrite(env->buf_base, env->buf_size, 1, stdout); + } + + packenv_free(env); + packfree(pi); +} +#endif diff --git a/src/pack.h b/src/pack.h new file mode 100644 index 0000000..70a2488 --- /dev/null +++ b/src/pack.h @@ -0,0 +1,47 @@ +/* This file is part of vmod-binlog + Copyright (C) 2013 Sergey Poznyakoff + + Vmod-binlog is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + Vmod-binlog is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with vmod-binlog. If not, see <http://www.gnu.org/licenses/>. +*/ +#include <stdio.h> + +struct packenv { + char *buf_base; + size_t buf_size; + size_t buf_pos; + char **argv; + int argc; + int argi; + FILE *fp; +}; + +struct packinst; + +struct packinst *packcomp(const char *s, char **endp); +void packfree(struct packinst *pi); +void packin(struct packinst *pi, struct packenv *env); +void packout(struct packinst *pi, struct packenv *env); +struct packinst *packinnext(struct packinst *pi, struct packenv *env); + +size_t packsize(struct packinst *pi); +struct packenv *packenv_create(size_t size); +void packenv_init(struct packenv *env); +void packenv_free(struct packenv *env); + + + + + + + diff --git a/src/vmod-binlog.h b/src/vmod-binlog.h index db6c7f0..06d517d 100644 --- a/src/vmod-binlog.h +++ b/src/vmod-binlog.h @@ -37,7 +37,6 @@ struct binlog_record { - long nid; /* node ID */ - long aid; /* article ID */ time_t ts; /* timestamp */ + char data[1]; /* payload */ }; @@ -50,22 +49,10 @@ struct binlog_file_header { char pad[16 - BINLOG_MAGIC_LEN]; uint32_t version; + size_t hdrsize; size_t recsize; size_t recnum; +/* char dataspec[X];*/ }; -#define BINLOG_HEADER_SIZE \ - ((sizeof(struct binlog_file_header) + sizeof(struct binlog_record) - 1) / \ - sizeof(struct binlog_record)) - -union binlog_header { - struct binlog_file_header hdr; - struct binlog_record pad[BINLOG_HEADER_SIZE]; -}; - -#define binlog_size(conf) \ - (sizeof(union binlog_header) + \ - (conf)->recidx * sizeof(struct binlog_record)) -#define binlog_recnum(conf) \ - (((conf)->size - sizeof(union binlog_header)) / \ - sizeof(struct binlog_record)) +#define binlog_size(hdr) ((hdr)->hdrsize + (hdr)->recnum * (hdr)->recsize) diff --git a/src/vmod.vcc b/src/vmod.vcc index def0ec3..45da354 100644 --- a/src/vmod.vcc +++ b/src/vmod.vcc @@ -1,7 +1,8 @@ Module binlog Init module_init -Function VOID init(PRIV_VCL, STRING) -Function VOID append(PRIV_VCL, INT, INT) -Function VOID sappend(PRIV_VCL, STRING, STRING) +Function VOID init(PRIV_VCL, STRING, STRING, STRING) +Function VOID start(PRIV_VCL) +Function VOID commit(PRIV_VCL) +Function VOID pack(PRIV_VCL, STRING) Function VOID sync(PRIV_VCL) Function VOID close(PRIV_VCL) diff --git a/tests/test01.at b/tests/test01.at index 116c8f1..62eb4fb 100644 --- a/tests/test01.at +++ b/tests/test01.at @@ -16,5 +16,5 @@ AT_SETUP([appends]) -AT_KEYWORDS([append]) +AT_KEYWORDS([test01 append]) AT_CHECK([ @@ -33,5 +33,5 @@ varnish v1 -vcl+backend { import binlog from "$abs_top_builddir/src/.libs/libvmod_binlog.so"; sub vcl_init { - binlog.init("dir=$cwd/log;size=1M;pattern=logfile"); + binlog.init("$cwd/log", "LL", "size=1M;pattern=logfile"); } sub vcl_fini { @@ -39,5 +39,8 @@ varnish v1 -vcl+backend { } sub vcl_recv { - binlog.sappend(req.http.X-nid, req.http.X-aid); + binlog.start(); + binlog.pack(req.http.X-nid); + binlog.pack(req.http.X-aid); + binlog.commit(); return (lookup); } diff --git a/tests/test02.at b/tests/test02.at index df639a5..459a3cc 100644 --- a/tests/test02.at +++ b/tests/test02.at @@ -16,5 +16,5 @@ AT_SETUP([log rotation]) -AT_KEYWORDS([rotation]) +AT_KEYWORDS([test02 rotation]) AT_CHECK([ @@ -33,5 +33,5 @@ varnish v1 -vcl+backend { import binlog from "$abs_top_builddir/src/.libs/libvmod_binlog.so"; sub vcl_init { - binlog.init("dir=$cwd/log;size=1M;interval=10;roundts=1;pattern=%S.log"); + binlog.init("$cwd/log","LL","size=1M;interval=10;roundts=1;pattern=%S.log"); } sub vcl_fini { @@ -39,5 +39,8 @@ varnish v1 -vcl+backend { } sub vcl_recv { - binlog.sappend(req.http.X-nid, req.http.X-aid); + binlog.start(); + binlog.pack(req.http.X-nid); + binlog.pack(req.http.X-aid); + binlog.commit(); return (lookup); } |