summaryrefslogtreecommitdiffabout
authorSergey Poznyakoff <gray@gnu.org.ua>2013-10-17 19:27:16 (GMT)
committer Sergey Poznyakoff <gray@gnu.org.ua>2013-10-17 19:27:16 (GMT)
commit803128db75e3efea8cd7be29732a7b4315205d13 (patch) (side-by-side diff)
tree713699a3724f905ca3dd740c386c9ffdb75e94a7
parentaad0ead6ae1bb56953642eac584d9a5f1dc6d72e (diff)
downloadvmod-binlog-803128db75e3efea8cd7be29732a7b4315205d13.tar.gz
vmod-binlog-803128db75e3efea8cd7be29732a7b4315205d13.tar.bz2
binlogsel: add loadable module support
* Makefile.am (SUBDIRS): Add libltdl * configure.ac: Configure ltdl * src/Makefile.am (binlogsel_LDADD): Add LIBLTDL. (binlogsel_CPPFLAGS): New variable. * src/binlogcat.c (catlog): Bugfix. * src/binlogsel.c: Allow for multiple intervals. Support loadable modules. * doc/binlogcat.1: Update. * doc/binlogsel.1: Update. * doc/vmod-binlog.3: Update.
Diffstat (more/less context) (ignore whitespace changes)
-rw-r--r--Makefile.am2
-rw-r--r--configure.ac10
-rw-r--r--doc/binlogcat.16
-rw-r--r--doc/binlogsel.1142
-rw-r--r--doc/vmod-binlog.34
-rw-r--r--src/Makefile.am4
-rw-r--r--src/binlogcat.c2
-rw-r--r--src/binlogsel.c202
8 files changed, 324 insertions, 48 deletions
diff --git a/Makefile.am b/Makefile.am
index 73cb11c..917f2e5 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -15,7 +15,7 @@
# along with vmod-binlog. If not, see <http://www.gnu.org/licenses/>.
ACLOCAL_AMFLAGS = -I m4
-SUBDIRS = src tests doc
+SUBDIRS = libltdl src tests doc
EXTRA_DIST=git2chg.awk
diff --git a/configure.ac b/configure.ac
index 5c74cb7..c4c9872 100644
--- a/configure.ac
+++ b/configure.ac
@@ -23,7 +23,7 @@ AM_CONFIG_HEADER(config.h)
AC_CANONICAL_SYSTEM
AC_LANG(C)
-AM_INIT_AUTOMAKE([gnu tar-ustar])
+AM_INIT_AUTOMAKE([gnu tar-ustar subdir-objects])
AC_GNU_SOURCE
AC_PROG_CC
@@ -38,6 +38,12 @@ AC_PROG_INSTALL
AC_PROG_LIBTOOL
AC_PROG_MAKE_SET
+LT_PREREQ(2.2.5a)
+LT_CONFIG_LTDL_DIR([libltdl])
+LT_INIT([dlopen])
+LTDL_INIT([recursive])
+AC_CONFIG_FILES([libltdl/Makefile])
+
# Check for pkg-config
PKG_PROG_PKG_CONFIG
@@ -97,6 +103,8 @@ if test -z "$VMODDIR"; then
VMODDIR='$(libdir)/varnish/mods'
fi
+AC_SUBST(BINLOGSEL_MODDIR,'$(libdir)/$(PACKAGE)')
+
# Initialize the test suite.
AC_CONFIG_TESTDIR(tests)
AC_CONFIG_FILES([tests/Makefile tests/atlocal])
diff --git a/doc/binlogcat.1 b/doc/binlogcat.1
index 5707860..886d842 100644
--- a/doc/binlogcat.1
+++ b/doc/binlogcat.1
@@ -13,7 +13,7 @@
.\"
.\" You should have received a copy of the GNU General Public License
.\" along with vmod-binlog. If not, see <http://www.gnu.org/licenses/>.
-.TH BINLOGCAT 1 "October 16, 2013" "BINLOGCAT" "User Reference"
+.TH BINLOGCAT 1 "October 17, 2013" "BINLOGCAT" "User Reference"
.SH NAME
binlogcat \- print binary log files in human-readable form
.SH SYNOPSIS
@@ -22,7 +22,7 @@ binlogcat \- print binary log files in human-readable form
The
.B binlogcat
utility displays on standard output binary log files created by
-.BR vmod-binlog (3).
+.BR vmod\-binlog (3).
If no \fIFILE\fR is given on the command line, the utility will read
the binary data from its standard input.
.PP
@@ -48,7 +48,7 @@ Print a short help summary.
.BR \-V
Show program version and exit.
.SH "SEE ALSO"
-.BR vmod-binlog (3),
+.BR vmod\-binlog (3),
.BR binlogsel (1),
.BR vcl (7),
.BR varnishd (1).
diff --git a/doc/binlogsel.1 b/doc/binlogsel.1
index cebb98c..ec7bd24 100644
--- a/doc/binlogsel.1
+++ b/doc/binlogsel.1
@@ -13,13 +13,16 @@
.\"
.\" You should have received a copy of the GNU General Public License
.\" along with vmod-binlog. If not, see <http://www.gnu.org/licenses/>.
-.TH BINLOGSEL 1 "October 16, 2013" "BINLOGSEL" "User Reference"
+.TH BINLOGSEL 1 "October 17, 2013" "BINLOGSEL" "User Reference"
.SH NAME
binlogsel \- select records from binary logs
.SH SYNOPSIS
-\fBbinlogsel\fR [\fB\-dnv\fR] [\fB\-t\fR \fIFORMAT\fR] [\fB\-F\fR
-\fITIME\fR] [\fB\-T\fR \fITIME\fR] [\fB\-p\fR \fIPATTERN\fR]\
- [\fB\-D\fR \fIDIR\fR]
+\fBbinlogsel\fR [\fB\-dnv\fR] [\fB\-D\fR \fIDIR\fR]\
+ [\fB\-p\fR \fIPATTERN\fR]\ [\fB\-t\fR \fIFORMAT\fR]
+ [\fB\-I\fR \fITAG\fR] [\fB\-F\fR \fITIME\fR] [\fB\-T\fR \fITIME\fR]\
+
+ [\fB\-L\fR \fIDIR\fR] [\fB\-P\fR \fIDIR\fR]\
+ [\fB\-m\fR '\fIMODULE\fR[ \fIARGS\fR]']
[\fIFILE\fR...]
\fBbinlogsel\fR [\fB\-hV\fR]
@@ -65,6 +68,20 @@ the following command:
.B info coreutils 'Date input formats'
.EE
.PP
+Several time intervals can be specified, provided that each of them is
+preceded by the \fB\-I\fR option, which introduces the tag to mark
+records falling within the time interval that follows it. This tag is
+output before each record. For example:
+.PP
+.EX
+binlogsel -I 1h -F '1 hour ago' -I 2d -F '2 days ago'
+.EE
+.PP
+This command selects two intervals: records added within the last hour,
+which will be prefixed on the output with the string \fB1h\fR, and
+records added within the last 2 days, which are prefixed by the string
+\fB2d\fR.
+.PP
The log files are searched in the directory specified with the
\fB\-D\fR command line option. The \fB\-i\fR option can be used to
define directory indexing level.
@@ -75,47 +92,132 @@ If files are listed in the command line, the
and
.B \-p
options are ignored.
+.PP
+The default action of
+.B binlogsel
+is to print matching records on the standard output. This can be
+changed by the use of loadable modules. A module is a dynamic library
+that is loaded at program startup and provides functions for handling
+records in a specific way.
+.PP
+The module to be loaded is supplied with the \fB\-m\fR option. For
+example, the option \fB-m modname\fR instructs
+.B binlogsel
+to load library \fBmodname.so\fR. Additional arguments for the module
+initialization function can be supplied in the same option:
+.EX
+.B binlogsel -m 'modname -n arg'
+.EE
+.PP
+(note quoting).
+.PP
+The module to be loaded is searched in the library path, which
+consists initially of the single directory
+\fI$prefix\fR\fB/lib/vmod\-binlog\fR. This path can be modified using
+the
+.BI \-L " DIR"
+option, which adds \fIDIR\fR to its and, or by the
+.BI \-P " DIR"
+option, which inserts its argument immediately before the default
+library directory.
+.PP
+See the section \fBLOADABLE MODULES\fR for the discussion of how to
+write loadable modules.
.SH OPTIONS
.TP
.BI \-D " DIR"
Set base log directory.
.TP
-.BI \-i " N"
-Set directory indexing level. Valid values for \fIN\fR are \fB0\fR,
-\fB1\fR and \fB2\fR.
+.B \-d
+Print timestamps relative to first record in the file.
.TP
.BI \-F " TIME"
Select records newer than \fITIME\fR. See the section
\fBDESCRIPTION\fR for a discussion of valid time formats.
.TP
-.BI \-T " TIME"
-Select records older than \fITIME\fR.
+.BI \-I " TAG"
+Defines the tag for the \fB\-F\fR and \fB\-T\fR options that
+immediately follow it.
+.TP
+.BI \-i " N"
+Set directory indexing level. Valid values for \fIN\fR are \fB0\fR,
+\fB1\fR and \fB2\fR.
+.TP
+.BR \-h
+Print a short help summary and exit.
+.TP
+.BI \-L " DIR"
+Add \fIDIR\fR to the end of the library search path.
+.TP
+\fB\-m\fR \fB'\fR\fIMODULE\fR[ \fIARGS\fR]\fB'\fR
+Load \fIMODULE\fR.
+.TP
+.B \-n
+Precede each record by its number in the file (0-based).
+.TP
+.BI \-P " DIR"
+Insert \fIDIR\fR into the library search path immediately before the
+default library directory.
.TP
.BI \-p " PATTERN"
Set pattern for log file names. \fIPATTERN\fR must be a valid
.BR strftime (3)
format string.
-.B \-d
-Print timestamps relative to first record in the file.
.TP
-.B \-n
-Precede each record by its number in the file (0-based).
-.TP
-.B \-v
-Print information about each file before dumpng it.
+.BI \-T " TIME"
+Select records older than \fITIME\fR.
.TP
.BI \-t " FORMAT"
Format timestamps according to \fIFORMAT\fR (see
.BR strftime (3)).
Default is \fB%c\fR.
.TP
-.BR \-h
-Print a short help summary.
-.TP
.BR \-V
Show program version and exit.
+.TP
+.B \-v
+Print information about each file before dumping it.
+.SH LOADABLE MODULES
+.TP
+.BI "void init(char *" param ", void (*" addfn ")(const char *, const char *, const char *));"
+.B [OPTIONAL]
+Initializes the module. The first argument points to the arguments
+supplied to the module in the command line. The \fBaddfn\fR function
+can be used to add new time intervals. Its usage is:
+
+.EX
+.BI "void addfn(const char *" tag ", const char *" start ", const char *" end ")"
+.EE
+
+where \fItag\fR is the interval tag, and \fIstart\fR and \fIend\fR
+supply interval start and end times, correspondingly.
+.TP
+.BI "void done(" void ");"
+.B [OPTIONAL]
+Called immediately before
+.B binlogsel
+terminates.
+.TP
+.BI "int open(const char *" file ", size_t " recsize ", const char *" format ");"
+.B [OPTIONAL]
+This function is called immediately after opening the new log file.
+The name of the file is given by the \fIfile\fR argument. The
+\fIrecsize\fR argument gives the record size, and \fIformat\fR points
+to the data format specification.
+
+The function must return 0 to indicate success and non-null value on errors.
+.TP
+.BI "void close(" void ");"
+.B [OPTIONAL]
+Called before closing the log file.
+.TP
+.BI "void record(const char *" tag ", time_t " ts ", void * " data ");"
+.B [MANDATORY]
+This function processes the record pointed to by \fIdata\fR. The
+\fItag\fR argument points to the interval tag (can be \fBNULL\fR).
+Timestamp of the record is given by the \fIts\fR argument.
.SH "SEE ALSO"
-.BR vmod-binlog (3),
+.BR vmod\-binlog (3),
.BR binlogcat (1),
.BR vcl (7),
.BR varnishd (1).
diff --git a/doc/vmod-binlog.3 b/doc/vmod-binlog.3
index f6eb94f..2d6dfdb 100644
--- a/doc/vmod-binlog.3
+++ b/doc/vmod-binlog.3
@@ -13,9 +13,9 @@
.\"
.\" You should have received a copy of the GNU General Public License
.\" along with vmod-binlog. If not, see <http://www.gnu.org/licenses/>.
-.TH VMOD-BINLOG 1 "October 16, 2013" "VMOD-BINLOG" "User Reference"
+.TH VMOD-BINLOG 1 "October 17, 2013" "VMOD-BINLOG" "User Reference"
.SH NAME
-vmod-binlog \- binary log file support for Varnish Cache.
+vmod\-binlog \- binary log file support for Varnish Cache.
.SH SYNOPSIS
.B import binlog;
diff --git a/src/Makefile.am b/src/Makefile.am
index 565444a..a20f9cb 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -25,7 +25,9 @@ binlogcat_SOURCES = binlogcat.c
binlogcat_LDADD = ./libbinlog.a
binlogsel_SOURCES = binlogsel.c
-binlogsel_LDADD = ./libbinlog.a
+binlogsel_LDADD = ./libbinlog.a @LIBLTDL@
+binlogsel_CPPFLAGS =\
+ -DBINLOGSEL_MODDIR=\"$(BINLOGSEL_MODDIR)\"
vmoddir = $(VMODDIR)
vmod_LTLIBRARIES = libvmod_binlog.la
diff --git a/src/binlogcat.c b/src/binlogcat.c
index bc7df6c..dd337bb 100644
--- a/src/binlogcat.c
+++ b/src/binlogcat.c
@@ -54,7 +54,7 @@ catlog(const char *fname)
else {
fp = fopen(fname, "r");
if (!fp) {
- error("cannot open %s: %s", strerror(errno));
+ error("cannot open %s: %s", fname, strerror(errno));
exit(1);
}
}
diff --git a/src/binlogsel.c b/src/binlogsel.c
index 5238215..10630e6 100644
--- a/src/binlogsel.c
+++ b/src/binlogsel.c
@@ -28,6 +28,7 @@
#include <time.h>
#include <string.h>
#include <glob.h>
+#include <ltdl.h>
#include "vmod-binlog.h"
#include "pack.h"
#include "err.h"
@@ -42,6 +43,14 @@ char *directory;
char *pattern;
enum binlog_index_type index_type = index_year;
+char *module_name;
+char *module_args;
+void (*module_init)(char *, void (*)(const char *, const char *, const char *));
+int (*module_open)(const char *, size_t, const char *);
+void (*module_close)(void);
+void (*module_done)(void);
+void (*module_record)(const char *, time_t, void *);
+
/* Time mask flags indicate which timestamp is set. */
#define START_TIME 0x01 /* start time is set */
#define STOP_TIME 0x02 /* stop time is set */
@@ -96,22 +105,65 @@ interval_add(const char *name, int tmask, time_t start, time_t end)
}
void
+interval_add_str(const char *id, const char *from, const char *to)
+{
+ struct timespec ts;
+ time_t start, end;
+ int tmask = 0;
+
+ if (from) {
+ if (!parse_datetime(&ts, from, NULL)) {
+ error("invalid timespec: %s", from);
+ exit(1);
+ }
+ start = ts.tv_sec;
+ tmask |= START_TIME;
+ }
+
+ if (to) {
+ if (!parse_datetime(&ts, to, NULL)) {
+ error("invalid timespec: %s", to);
+ exit(1);
+ }
+ end = ts.tv_sec;
+ tmask |= STOP_TIME;
+ }
+
+ if (tmask)
+ interval_add(id, tmask, start, end);
+}
+
+
+void
help()
{
- printf("usage: %s [-dhnVv] [-t FORMAT] [-F TIME] [-T TIME] [-p PATTERN] [-D DIR] [-i 0|1|2] [FILE...]\n", progname);
+ printf("usage: %s [OPTIONS] [FILE...]\n", progname);
printf("Select records from binary logs\n");
- printf("\nOptions are:\n\n");
+ printf("\nOPTIONS are:\n\n");
+ printf("File selection:\n");
printf(" -D DIR log file storage directory\n");
printf(" -i 0|1|2 select directory indexing level\n");
+ printf(" -p PATTERN select files matching PATTERN\n");
+ printf("\n");
+ printf("Time intervals:\n");
+ printf(" -I TAG set tag for the subsequent -F and -T option pair\n");
printf(" -F TIME print records starting from TIME\n");
printf(" -T TIME print records starting up to TIME\n");
- printf(" -p PATTERN select files matching PATTERN\n");
printf("\n");
+ printf("Loadable module control\n");
+ printf(" -L DIR append DIR to the loadable module search path\n");
+ printf(" -P DIR add DIR to the loadable module search path before\n");
+ printf(" the default module directory\n");
+ printf(" -m 'MODULE[ ARGS]'\n");
+ printf(" load given MODULE\n");
+ printf("\n");
+ printf("Output control:\n");
printf(" -d print timestamps relative to first record in the file\n");
printf(" -n output record numbers\n");
printf(" -v print information about each file\n");
printf(" -t FMT format timestamps according to FMT\n");
printf("\n");
+ printf("Informational options:\n");
printf(" -h print this help summary\n");
printf(" -V show program version\n");
printf("\n");
@@ -208,6 +260,7 @@ selmem(const char *name, void *base)
char timebuf[128];
char *format;
struct interval *ip;
+ size_t datasize;
hdr = base;
@@ -228,6 +281,13 @@ selmem(const char *name, void *base)
name, format, (unsigned long) hdr->recsize,
(unsigned long) hdr->recnum);
+ datasize = hdr->recsize - offsetof(struct binlog_record,data);
+
+ if (module_open && module_open(name, datasize, format)) {
+ error("%s: rejected by module %s", name, module_name);
+ return;
+ }
+
inst = packcomp(format, &p);
if (!inst) {
if (errno == EINVAL) {
@@ -240,8 +300,7 @@ selmem(const char *name, void *base)
return;
}
- env = packenv_create(hdr->recsize -
- offsetof(struct binlog_record,data));
+ env = packenv_create(datasize);
env->fp = stdout;
base = (char*)base + hdr->hdrsize;
@@ -286,18 +345,25 @@ selmem(const char *name, void *base)
continue;
if ((ip->timemask & STOP_TIME) && ip->end < rec->ts)
continue;
- if (ip->name)
- printf("%s ", ip->name);
- else if (ip == interval_head && ip->next)
- printf("default ");
- if (number_option)
- printf("%lu ", (unsigned long) start);
- printf("%s ", timebuf);
- env->buf_pos = 0;
- packout(inst, env);
- fputc('\n', stdout);
+ if (module_record)
+ module_record(ip->name, rec->ts, env->buf_base);
+ else {
+ if (ip->name)
+ printf("%s ", ip->name);
+ else if (ip == interval_head && ip->next)
+ printf("default ");
+ if (number_option)
+ printf("%lu ", (unsigned long) start);
+ printf("%s ", timebuf);
+ env->buf_pos = 0;
+ packout(inst, env);
+ fputc('\n', stdout);
+ }
}
}
+
+ if (module_close)
+ module_close();
}
static int
@@ -679,6 +745,78 @@ selglob(const char *dir, const char *pattern)
globfree(&gl);
}
+#define LP_PREPEND 0
+#define LP_APPEND 1
+
+struct libdir {
+ struct libdir *next;
+ struct libdir *prev;
+ const char *dir;
+};
+
+struct libdir *libdir_head, *libdir_mid, *libdir_tail;
+
+void
+add_load_path(const char *dir, int where)
+{
+ struct libdir *ld = xmalloc(sizeof(ld[0]));
+ ld->dir = dir;
+ if (!libdir_mid) {
+ ld->next = ld->prev = NULL;
+ libdir_mid = libdir_head = libdir_tail = ld;
+ } else if (where == LP_PREPEND) {
+ ld->next = libdir_mid;
+ ld->prev = libdir_mid->prev;
+ if (ld->prev)
+ ld->prev->next = libdir_mid;
+ else
+ libdir_head = ld;
+ libdir_mid->prev = ld;
+ } else /* if (where == LT_APPEND) */ {
+ ld->next = NULL;
+ ld->prev = libdir_tail;
+ libdir_tail->next = ld;
+ libdir_tail = ld;
+ }
+}
+
+void
+loader_init(void)
+{
+ struct libdir *ld;
+ lt_dlhandle handle = NULL;
+ lt_dladvise advise = NULL;
+
+ if (!module_name)
+ return;
+
+ lt_dlinit();
+ for (ld = libdir_head; ld; ld = ld->next)
+ lt_dladdsearchdir(ld->dir);
+
+ if (!lt_dladvise_init(&advise) && !lt_dladvise_ext(&advise)
+ && !lt_dladvise_global(&advise))
+ handle = lt_dlopenadvise(module_name, advise);
+ lt_dladvise_destroy(&advise);
+
+ if (!handle) {
+ error("cannot load module %s: %s", module_name,
+ lt_dlerror());
+ exit(1);
+ }
+
+ module_record = lt_dlsym(handle, "record");
+ if (!module_record) {
+ error("%s: faulty module: function record() not defined",
+ module_name);
+ exit(1);
+ }
+ module_init = lt_dlsym(handle, "init");
+ module_open = lt_dlsym(handle, "open");
+ module_close = lt_dlsym(handle, "close");
+ module_done = lt_dlsym(handle, "done");
+}
+
int
main(int argc, char **argv)
{
@@ -687,9 +825,11 @@ main(int argc, char **argv)
const char *id;
int tmask = 0;
time_t start, stop;
+ char *p;
setprogname(argv[0]);
- while ((c = getopt(argc, argv, "D:dF:hi:I:p:T:t:nV")) != EOF)
+ add_load_path(BINLOGSEL_MODDIR, LP_APPEND);
+ while ((c = getopt(argc, argv, "D:dF:hi:I:L:m:p:P:T:t:nV")) != EOF)
switch (c) {
case 'D':
directory = optarg;
@@ -722,6 +862,23 @@ main(int argc, char **argv)
exit(1);
}
break;
+ case 'L':
+ add_load_path(optarg, LP_APPEND);
+ break;
+ case 'm':
+ p = strchr(optarg, ' ');
+ if (p) {
+ *p++ = 0;
+ while (*p == ' ')
+ p++;
+ if (*p)
+ module_args = p;
+ }
+ module_name = optarg;
+ break;
+ case 'P':
+ add_load_path(optarg, LP_PREPEND);
+ break;
case 'p':
pattern = optarg;
break;
@@ -752,13 +909,18 @@ main(int argc, char **argv)
if (tmask)
interval_add(id, tmask, start, stop);
+ argc -= optind;
+ argv += optind;
+
+ loader_init();
+
+ if (module_init)
+ module_init(module_args, interval_add_str);
+
if (timemask & CLEAR_START_TIME)
timemask &= ~START_TIME;
if (timemask & CLEAR_STOP_TIME)
timemask &= ~STOP_TIME;
-
- argc -= optind;
- argv += optind;
if (argc) {
if (pattern) {
@@ -772,6 +934,8 @@ main(int argc, char **argv)
} else {
selidx_year(directory);
}
+ if (module_done)
+ module_done();
exit(0);
}

Return to:

Send suggestions and report system problems to the System administrator.