diff options
author | Sergey Poznyakoff <gray@gnu.org.ua> | 2013-10-17 22:27:16 +0300 |
---|---|---|
committer | Sergey Poznyakoff <gray@gnu.org.ua> | 2013-10-17 22:27:16 +0300 |
commit | 803128db75e3efea8cd7be29732a7b4315205d13 (patch) | |
tree | 713699a3724f905ca3dd740c386c9ffdb75e94a7 | |
parent | aad0ead6ae1bb56953642eac584d9a5f1dc6d72e (diff) | |
download | vmod-binlog-803128db75e3efea8cd7be29732a7b4315205d13.tar.gz vmod-binlog-803128db75e3efea8cd7be29732a7b4315205d13.tar.bz2 |
binlogsel: add loadable module support
* Makefile.am (SUBDIRS): Add libltdl
* configure.ac: Configure ltdl
* src/Makefile.am (binlogsel_LDADD): Add LIBLTDL.
(binlogsel_CPPFLAGS): New variable.
* src/binlogcat.c (catlog): Bugfix.
* src/binlogsel.c: Allow for multiple intervals. Support loadable
modules.
* doc/binlogcat.1: Update.
* doc/binlogsel.1: Update.
* doc/vmod-binlog.3: Update.
-rw-r--r-- | Makefile.am | 2 | ||||
-rw-r--r-- | configure.ac | 10 | ||||
-rw-r--r-- | doc/binlogcat.1 | 6 | ||||
-rw-r--r-- | doc/binlogsel.1 | 142 | ||||
-rw-r--r-- | doc/vmod-binlog.3 | 4 | ||||
-rw-r--r-- | src/Makefile.am | 4 | ||||
-rw-r--r-- | src/binlogcat.c | 2 | ||||
-rw-r--r-- | src/binlogsel.c | 182 |
8 files changed, 314 insertions, 38 deletions
diff --git a/Makefile.am b/Makefile.am index 73cb11c..917f2e5 100644 --- a/Makefile.am +++ b/Makefile.am @@ -15,7 +15,7 @@ # along with vmod-binlog. If not, see <http://www.gnu.org/licenses/>. ACLOCAL_AMFLAGS = -I m4 -SUBDIRS = src tests doc +SUBDIRS = libltdl src tests doc EXTRA_DIST=git2chg.awk diff --git a/configure.ac b/configure.ac index 5c74cb7..c4c9872 100644 --- a/configure.ac +++ b/configure.ac @@ -23,7 +23,7 @@ AM_CONFIG_HEADER(config.h) AC_CANONICAL_SYSTEM AC_LANG(C) -AM_INIT_AUTOMAKE([gnu tar-ustar]) +AM_INIT_AUTOMAKE([gnu tar-ustar subdir-objects]) AC_GNU_SOURCE AC_PROG_CC @@ -38,6 +38,12 @@ AC_PROG_INSTALL AC_PROG_LIBTOOL AC_PROG_MAKE_SET +LT_PREREQ(2.2.5a) +LT_CONFIG_LTDL_DIR([libltdl]) +LT_INIT([dlopen]) +LTDL_INIT([recursive]) +AC_CONFIG_FILES([libltdl/Makefile]) + # Check for pkg-config PKG_PROG_PKG_CONFIG @@ -97,6 +103,8 @@ if test -z "$VMODDIR"; then VMODDIR='$(libdir)/varnish/mods' fi +AC_SUBST(BINLOGSEL_MODDIR,'$(libdir)/$(PACKAGE)') + # Initialize the test suite. AC_CONFIG_TESTDIR(tests) AC_CONFIG_FILES([tests/Makefile tests/atlocal]) diff --git a/doc/binlogcat.1 b/doc/binlogcat.1 index 5707860..886d842 100644 --- a/doc/binlogcat.1 +++ b/doc/binlogcat.1 @@ -13,7 +13,7 @@ .\" .\" You should have received a copy of the GNU General Public License .\" along with vmod-binlog. If not, see <http://www.gnu.org/licenses/>. -.TH BINLOGCAT 1 "October 16, 2013" "BINLOGCAT" "User Reference" +.TH BINLOGCAT 1 "October 17, 2013" "BINLOGCAT" "User Reference" .SH NAME binlogcat \- print binary log files in human-readable form .SH SYNOPSIS @@ -22,7 +22,7 @@ binlogcat \- print binary log files in human-readable form The .B binlogcat utility displays on standard output binary log files created by -.BR vmod-binlog (3). +.BR vmod\-binlog (3). If no \fIFILE\fR is given on the command line, the utility will read the binary data from its standard input. .PP @@ -48,7 +48,7 @@ Print a short help summary. .BR \-V Show program version and exit. .SH "SEE ALSO" -.BR vmod-binlog (3), +.BR vmod\-binlog (3), .BR binlogsel (1), .BR vcl (7), .BR varnishd (1). diff --git a/doc/binlogsel.1 b/doc/binlogsel.1 index cebb98c..ec7bd24 100644 --- a/doc/binlogsel.1 +++ b/doc/binlogsel.1 @@ -13,13 +13,16 @@ .\" .\" You should have received a copy of the GNU General Public License .\" along with vmod-binlog. If not, see <http://www.gnu.org/licenses/>. -.TH BINLOGSEL 1 "October 16, 2013" "BINLOGSEL" "User Reference" +.TH BINLOGSEL 1 "October 17, 2013" "BINLOGSEL" "User Reference" .SH NAME binlogsel \- select records from binary logs .SH SYNOPSIS -\fBbinlogsel\fR [\fB\-dnv\fR] [\fB\-t\fR \fIFORMAT\fR] [\fB\-F\fR -\fITIME\fR] [\fB\-T\fR \fITIME\fR] [\fB\-p\fR \fIPATTERN\fR]\ - [\fB\-D\fR \fIDIR\fR] +\fBbinlogsel\fR [\fB\-dnv\fR] [\fB\-D\fR \fIDIR\fR]\ + [\fB\-p\fR \fIPATTERN\fR]\ [\fB\-t\fR \fIFORMAT\fR] + [\fB\-I\fR \fITAG\fR] [\fB\-F\fR \fITIME\fR] [\fB\-T\fR \fITIME\fR]\ + + [\fB\-L\fR \fIDIR\fR] [\fB\-P\fR \fIDIR\fR]\ + [\fB\-m\fR '\fIMODULE\fR[ \fIARGS\fR]'] [\fIFILE\fR...] \fBbinlogsel\fR [\fB\-hV\fR] @@ -65,6 +68,20 @@ the following command: .B info coreutils 'Date input formats' .EE .PP +Several time intervals can be specified, provided that each of them is +preceded by the \fB\-I\fR option, which introduces the tag to mark +records falling within the time interval that follows it. This tag is +output before each record. For example: +.PP +.EX +binlogsel -I 1h -F '1 hour ago' -I 2d -F '2 days ago' +.EE +.PP +This command selects two intervals: records added within the last hour, +which will be prefixed on the output with the string \fB1h\fR, and +records added within the last 2 days, which are prefixed by the string +\fB2d\fR. +.PP The log files are searched in the directory specified with the \fB\-D\fR command line option. The \fB\-i\fR option can be used to define directory indexing level. @@ -75,47 +92,132 @@ If files are listed in the command line, the and .B \-p options are ignored. +.PP +The default action of +.B binlogsel +is to print matching records on the standard output. This can be +changed by the use of loadable modules. A module is a dynamic library +that is loaded at program startup and provides functions for handling +records in a specific way. +.PP +The module to be loaded is supplied with the \fB\-m\fR option. For +example, the option \fB-m modname\fR instructs +.B binlogsel +to load library \fBmodname.so\fR. Additional arguments for the module +initialization function can be supplied in the same option: +.EX +.B binlogsel -m 'modname -n arg' +.EE +.PP +(note quoting). +.PP +The module to be loaded is searched in the library path, which +consists initially of the single directory +\fI$prefix\fR\fB/lib/vmod\-binlog\fR. This path can be modified using +the +.BI \-L " DIR" +option, which adds \fIDIR\fR to its and, or by the +.BI \-P " DIR" +option, which inserts its argument immediately before the default +library directory. +.PP +See the section \fBLOADABLE MODULES\fR for the discussion of how to +write loadable modules. .SH OPTIONS .TP .BI \-D " DIR" Set base log directory. .TP -.BI \-i " N" -Set directory indexing level. Valid values for \fIN\fR are \fB0\fR, -\fB1\fR and \fB2\fR. +.B \-d +Print timestamps relative to first record in the file. .TP .BI \-F " TIME" Select records newer than \fITIME\fR. See the section \fBDESCRIPTION\fR for a discussion of valid time formats. .TP -.BI \-T " TIME" -Select records older than \fITIME\fR. +.BI \-I " TAG" +Defines the tag for the \fB\-F\fR and \fB\-T\fR options that +immediately follow it. +.TP +.BI \-i " N" +Set directory indexing level. Valid values for \fIN\fR are \fB0\fR, +\fB1\fR and \fB2\fR. +.TP +.BR \-h +Print a short help summary and exit. +.TP +.BI \-L " DIR" +Add \fIDIR\fR to the end of the library search path. +.TP +\fB\-m\fR \fB'\fR\fIMODULE\fR[ \fIARGS\fR]\fB'\fR +Load \fIMODULE\fR. +.TP +.B \-n +Precede each record by its number in the file (0-based). +.TP +.BI \-P " DIR" +Insert \fIDIR\fR into the library search path immediately before the +default library directory. .TP .BI \-p " PATTERN" Set pattern for log file names. \fIPATTERN\fR must be a valid .BR strftime (3) format string. -.B \-d -Print timestamps relative to first record in the file. .TP -.B \-n -Precede each record by its number in the file (0-based). -.TP -.B \-v -Print information about each file before dumpng it. +.BI \-T " TIME" +Select records older than \fITIME\fR. .TP .BI \-t " FORMAT" Format timestamps according to \fIFORMAT\fR (see .BR strftime (3)). Default is \fB%c\fR. .TP -.BR \-h -Print a short help summary. -.TP .BR \-V Show program version and exit. +.TP +.B \-v +Print information about each file before dumping it. +.SH LOADABLE MODULES +.TP +.BI "void init(char *" param ", void (*" addfn ")(const char *, const char *, const char *));" +.B [OPTIONAL] +Initializes the module. The first argument points to the arguments +supplied to the module in the command line. The \fBaddfn\fR function +can be used to add new time intervals. Its usage is: + +.EX +.BI "void addfn(const char *" tag ", const char *" start ", const char *" end ")" +.EE + +where \fItag\fR is the interval tag, and \fIstart\fR and \fIend\fR +supply interval start and end times, correspondingly. +.TP +.BI "void done(" void ");" +.B [OPTIONAL] +Called immediately before +.B binlogsel +terminates. +.TP +.BI "int open(const char *" file ", size_t " recsize ", const char *" format ");" +.B [OPTIONAL] +This function is called immediately after opening the new log file. +The name of the file is given by the \fIfile\fR argument. The +\fIrecsize\fR argument gives the record size, and \fIformat\fR points +to the data format specification. + +The function must return 0 to indicate success and non-null value on errors. +.TP +.BI "void close(" void ");" +.B [OPTIONAL] +Called before closing the log file. +.TP +.BI "void record(const char *" tag ", time_t " ts ", void * " data ");" +.B [MANDATORY] +This function processes the record pointed to by \fIdata\fR. The +\fItag\fR argument points to the interval tag (can be \fBNULL\fR). +Timestamp of the record is given by the \fIts\fR argument. .SH "SEE ALSO" -.BR vmod-binlog (3), +.BR vmod\-binlog (3), .BR binlogcat (1), .BR vcl (7), .BR varnishd (1). diff --git a/doc/vmod-binlog.3 b/doc/vmod-binlog.3 index f6eb94f..2d6dfdb 100644 --- a/doc/vmod-binlog.3 +++ b/doc/vmod-binlog.3 @@ -13,9 +13,9 @@ .\" .\" You should have received a copy of the GNU General Public License .\" along with vmod-binlog. If not, see <http://www.gnu.org/licenses/>. -.TH VMOD-BINLOG 1 "October 16, 2013" "VMOD-BINLOG" "User Reference" +.TH VMOD-BINLOG 1 "October 17, 2013" "VMOD-BINLOG" "User Reference" .SH NAME -vmod-binlog \- binary log file support for Varnish Cache. +vmod\-binlog \- binary log file support for Varnish Cache. .SH SYNOPSIS .B import binlog; diff --git a/src/Makefile.am b/src/Makefile.am index 565444a..a20f9cb 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -25,7 +25,9 @@ binlogcat_SOURCES = binlogcat.c binlogcat_LDADD = ./libbinlog.a binlogsel_SOURCES = binlogsel.c -binlogsel_LDADD = ./libbinlog.a +binlogsel_LDADD = ./libbinlog.a @LIBLTDL@ +binlogsel_CPPFLAGS =\ + -DBINLOGSEL_MODDIR=\"$(BINLOGSEL_MODDIR)\" vmoddir = $(VMODDIR) vmod_LTLIBRARIES = libvmod_binlog.la diff --git a/src/binlogcat.c b/src/binlogcat.c index bc7df6c..dd337bb 100644 --- a/src/binlogcat.c +++ b/src/binlogcat.c @@ -54,7 +54,7 @@ catlog(const char *fname) else { fp = fopen(fname, "r"); if (!fp) { - error("cannot open %s: %s", strerror(errno)); + error("cannot open %s: %s", fname, strerror(errno)); exit(1); } } diff --git a/src/binlogsel.c b/src/binlogsel.c index 5238215..10630e6 100644 --- a/src/binlogsel.c +++ b/src/binlogsel.c @@ -28,6 +28,7 @@ #include <time.h> #include <string.h> #include <glob.h> +#include <ltdl.h> #include "vmod-binlog.h" #include "pack.h" #include "err.h" @@ -42,6 +43,14 @@ char *directory; char *pattern; enum binlog_index_type index_type = index_year; +char *module_name; +char *module_args; +void (*module_init)(char *, void (*)(const char *, const char *, const char *)); +int (*module_open)(const char *, size_t, const char *); +void (*module_close)(void); +void (*module_done)(void); +void (*module_record)(const char *, time_t, void *); + /* Time mask flags indicate which timestamp is set. */ #define START_TIME 0x01 /* start time is set */ #define STOP_TIME 0x02 /* stop time is set */ @@ -96,22 +105,65 @@ interval_add(const char *name, int tmask, time_t start, time_t end) } void +interval_add_str(const char *id, const char *from, const char *to) +{ + struct timespec ts; + time_t start, end; + int tmask = 0; + + if (from) { + if (!parse_datetime(&ts, from, NULL)) { + error("invalid timespec: %s", from); + exit(1); + } + start = ts.tv_sec; + tmask |= START_TIME; + } + + if (to) { + if (!parse_datetime(&ts, to, NULL)) { + error("invalid timespec: %s", to); + exit(1); + } + end = ts.tv_sec; + tmask |= STOP_TIME; + } + + if (tmask) + interval_add(id, tmask, start, end); +} + + +void help() { - printf("usage: %s [-dhnVv] [-t FORMAT] [-F TIME] [-T TIME] [-p PATTERN] [-D DIR] [-i 0|1|2] [FILE...]\n", progname); + printf("usage: %s [OPTIONS] [FILE...]\n", progname); printf("Select records from binary logs\n"); - printf("\nOptions are:\n\n"); + printf("\nOPTIONS are:\n\n"); + printf("File selection:\n"); printf(" -D DIR log file storage directory\n"); printf(" -i 0|1|2 select directory indexing level\n"); + printf(" -p PATTERN select files matching PATTERN\n"); + printf("\n"); + printf("Time intervals:\n"); + printf(" -I TAG set tag for the subsequent -F and -T option pair\n"); printf(" -F TIME print records starting from TIME\n"); printf(" -T TIME print records starting up to TIME\n"); - printf(" -p PATTERN select files matching PATTERN\n"); printf("\n"); + printf("Loadable module control\n"); + printf(" -L DIR append DIR to the loadable module search path\n"); + printf(" -P DIR add DIR to the loadable module search path before\n"); + printf(" the default module directory\n"); + printf(" -m 'MODULE[ ARGS]'\n"); + printf(" load given MODULE\n"); + printf("\n"); + printf("Output control:\n"); printf(" -d print timestamps relative to first record in the file\n"); printf(" -n output record numbers\n"); printf(" -v print information about each file\n"); printf(" -t FMT format timestamps according to FMT\n"); printf("\n"); + printf("Informational options:\n"); printf(" -h print this help summary\n"); printf(" -V show program version\n"); printf("\n"); @@ -208,6 +260,7 @@ selmem(const char *name, void *base) char timebuf[128]; char *format; struct interval *ip; + size_t datasize; hdr = base; @@ -228,6 +281,13 @@ selmem(const char *name, void *base) name, format, (unsigned long) hdr->recsize, (unsigned long) hdr->recnum); + datasize = hdr->recsize - offsetof(struct binlog_record,data); + + if (module_open && module_open(name, datasize, format)) { + error("%s: rejected by module %s", name, module_name); + return; + } + inst = packcomp(format, &p); if (!inst) { if (errno == EINVAL) { @@ -240,8 +300,7 @@ selmem(const char *name, void *base) return; } - env = packenv_create(hdr->recsize - - offsetof(struct binlog_record,data)); + env = packenv_create(datasize); env->fp = stdout; base = (char*)base + hdr->hdrsize; @@ -286,6 +345,9 @@ selmem(const char *name, void *base) continue; if ((ip->timemask & STOP_TIME) && ip->end < rec->ts) continue; + if (module_record) + module_record(ip->name, rec->ts, env->buf_base); + else { if (ip->name) printf("%s ", ip->name); else if (ip == interval_head && ip->next) @@ -300,6 +362,10 @@ selmem(const char *name, void *base) } } + if (module_close) + module_close(); +} + static int fchecktime(FILE *fp, const char *fname, time_t *ts) { @@ -679,6 +745,78 @@ selglob(const char *dir, const char *pattern) globfree(&gl); } +#define LP_PREPEND 0 +#define LP_APPEND 1 + +struct libdir { + struct libdir *next; + struct libdir *prev; + const char *dir; +}; + +struct libdir *libdir_head, *libdir_mid, *libdir_tail; + +void +add_load_path(const char *dir, int where) +{ + struct libdir *ld = xmalloc(sizeof(ld[0])); + ld->dir = dir; + if (!libdir_mid) { + ld->next = ld->prev = NULL; + libdir_mid = libdir_head = libdir_tail = ld; + } else if (where == LP_PREPEND) { + ld->next = libdir_mid; + ld->prev = libdir_mid->prev; + if (ld->prev) + ld->prev->next = libdir_mid; + else + libdir_head = ld; + libdir_mid->prev = ld; + } else /* if (where == LT_APPEND) */ { + ld->next = NULL; + ld->prev = libdir_tail; + libdir_tail->next = ld; + libdir_tail = ld; + } +} + +void +loader_init(void) +{ + struct libdir *ld; + lt_dlhandle handle = NULL; + lt_dladvise advise = NULL; + + if (!module_name) + return; + + lt_dlinit(); + for (ld = libdir_head; ld; ld = ld->next) + lt_dladdsearchdir(ld->dir); + + if (!lt_dladvise_init(&advise) && !lt_dladvise_ext(&advise) + && !lt_dladvise_global(&advise)) + handle = lt_dlopenadvise(module_name, advise); + lt_dladvise_destroy(&advise); + + if (!handle) { + error("cannot load module %s: %s", module_name, + lt_dlerror()); + exit(1); + } + + module_record = lt_dlsym(handle, "record"); + if (!module_record) { + error("%s: faulty module: function record() not defined", + module_name); + exit(1); + } + module_init = lt_dlsym(handle, "init"); + module_open = lt_dlsym(handle, "open"); + module_close = lt_dlsym(handle, "close"); + module_done = lt_dlsym(handle, "done"); +} + int main(int argc, char **argv) { @@ -687,9 +825,11 @@ main(int argc, char **argv) const char *id; int tmask = 0; time_t start, stop; + char *p; setprogname(argv[0]); - while ((c = getopt(argc, argv, "D:dF:hi:I:p:T:t:nV")) != EOF) + add_load_path(BINLOGSEL_MODDIR, LP_APPEND); + while ((c = getopt(argc, argv, "D:dF:hi:I:L:m:p:P:T:t:nV")) != EOF) switch (c) { case 'D': directory = optarg; @@ -722,6 +862,23 @@ main(int argc, char **argv) exit(1); } break; + case 'L': + add_load_path(optarg, LP_APPEND); + break; + case 'm': + p = strchr(optarg, ' '); + if (p) { + *p++ = 0; + while (*p == ' ') + p++; + if (*p) + module_args = p; + } + module_name = optarg; + break; + case 'P': + add_load_path(optarg, LP_PREPEND); + break; case 'p': pattern = optarg; break; @@ -752,14 +909,19 @@ main(int argc, char **argv) if (tmask) interval_add(id, tmask, start, stop); + argc -= optind; + argv += optind; + + loader_init(); + + if (module_init) + module_init(module_args, interval_add_str); + if (timemask & CLEAR_START_TIME) timemask &= ~START_TIME; if (timemask & CLEAR_STOP_TIME) timemask &= ~STOP_TIME; - argc -= optind; - argv += optind; - if (argc) { if (pattern) { error("either files or pattern (-p) must be given, " @@ -772,6 +934,8 @@ main(int argc, char **argv) } else { selidx_year(directory); } + if (module_done) + module_done(); exit(0); } |