diff options
-rw-r--r-- | Makefile.am | 2 | ||||
-rw-r--r-- | configure.ac | 2 | ||||
-rw-r--r-- | doc/Makefile.am | 18 | ||||
-rw-r--r-- | doc/binlogcat.1 | 76 | ||||
-rw-r--r-- | doc/binlogsel.1 | 143 | ||||
-rw-r--r-- | doc/magic | 7 | ||||
-rw-r--r-- | doc/vmod-binlog.3 | 287 | ||||
-rw-r--r-- | doc/vmod_binlog.3 | 1 | ||||
-rw-r--r-- | src/binlogsel.c | 3 |
9 files changed, 536 insertions, 3 deletions
diff --git a/Makefile.am b/Makefile.am index d11b042..73cb11c 100644 --- a/Makefile.am +++ b/Makefile.am @@ -15,7 +15,7 @@ # along with vmod-binlog. If not, see <http://www.gnu.org/licenses/>. ACLOCAL_AMFLAGS = -I m4 -SUBDIRS = src tests #doc +SUBDIRS = src tests doc EXTRA_DIST=git2chg.awk diff --git a/configure.ac b/configure.ac index 56e0411..5c74cb7 100644 --- a/configure.ac +++ b/configure.ac @@ -102,10 +102,10 @@ AC_CONFIG_TESTDIR(tests) AC_CONFIG_FILES([tests/Makefile tests/atlocal]) AM_MISSING_PROG([AUTOM4TE], [autom4te]) -dnl FIXME: doc/Makefile AC_CONFIG_FILES([ Makefile src/Makefile + doc/Makefile ]) AC_OUTPUT diff --git a/doc/Makefile.am b/doc/Makefile.am new file mode 100644 index 0000000..1808f7d --- /dev/null +++ b/doc/Makefile.am @@ -0,0 +1,18 @@ +# This file is part of vmod-binlog +# Copyright (C) 2013 Sergey Poznyakoff +# +# Vmod-binlog is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3, or (at your option) +# any later version. +# +# Vmod-binlog is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with vmod-binlog. If not, see <http://www.gnu.org/licenses/>. + +dist_man_MANS = vmod-binlog.3 vmod_binlog.3 binlogcat.1 binlogsel.1 +EXTRA_DIST = magic diff --git a/doc/binlogcat.1 b/doc/binlogcat.1 new file mode 100644 index 0000000..5707860 --- /dev/null +++ b/doc/binlogcat.1 @@ -0,0 +1,76 @@ +.\" This file is part of Vmod-binlog -*- nroff -*- +.\" Copyright (C) 2013 Sergey Poznyakoff +.\" +.\" Vmod-binlog is free software; you can redistribute it and/or modify +.\" it under the terms of the GNU General Public License as published by +.\" the Free Software Foundation; either version 3, or (at your option) +.\" any later version. +.\" +.\" Vmod-binlog is distributed in the hope that it will be useful, +.\" but WITHOUT ANY WARRANTY; without even the implied warranty of +.\" MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +.\" GNU General Public License for more details. +.\" +.\" You should have received a copy of the GNU General Public License +.\" along with vmod-binlog. If not, see <http://www.gnu.org/licenses/>. +.TH BINLOGCAT 1 "October 16, 2013" "BINLOGCAT" "User Reference" +.SH NAME +binlogcat \- print binary log files in human-readable form +.SH SYNOPSIS +\fBbinlogcat\fR [\fB\-dhnVv\fR] [\fB\-t\fR \fIFORMAT\fR] [\fIFILE\fR...] +.SH DESCRIPTION +The +.B binlogcat +utility displays on standard output binary log files created by +.BR vmod-binlog (3). +If no \fIFILE\fR is given on the command line, the utility will read +the binary data from its standard input. +.PP +Valid command line options are: +.TP +.B \-d +Print timestamps relative to first record in the file. +.TP +.B \-n +Precede each record by its number in the file (0-based). +.TP +.B \-v +Print information about each file before dumpng it. +.TP +.BI \-t FORMAT +Format timestamps according to \fIFORMAT\fR (see +.BR strftime (3)). +Default is \fB%c\fR. +.TP +.BR \-h +Print a short help summary. +.TP +.BR \-V +Show program version and exit. +.SH "SEE ALSO" +.BR vmod-binlog (3), +.BR binlogsel (1), +.BR vcl (7), +.BR varnishd (1). +.SH AUTHORS +Sergey Poznyakoff +.SH "BUG REPORTS" +Report bugs to <gray@gnu.org>. +.SH COPYRIGHT +Copyright \(co 2013 Sergey Poznyakoff +.br +.na +License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html> +.br +.ad +This is free software: you are free to change and redistribute it. +There is NO WARRANTY, to the extent permitted by law. +.\" Local variables: +.\" eval: (add-hook 'write-file-hooks 'time-stamp) +.\" time-stamp-start: ".TH [A-Z_][A-Z0-9_.-]* [0-9] \"" +.\" time-stamp-format: "%:B %:d, %:y" +.\" time-stamp-end: "\"" +.\" time-stamp-line-limit: 20 +.\" end: + + diff --git a/doc/binlogsel.1 b/doc/binlogsel.1 new file mode 100644 index 0000000..cebb98c --- /dev/null +++ b/doc/binlogsel.1 @@ -0,0 +1,143 @@ +.\" This file is part of Vmod-binlog -*- nroff -*- +.\" Copyright (C) 2013 Sergey Poznyakoff +.\" +.\" Vmod-binlog is free software; you can redistribute it and/or modify +.\" it under the terms of the GNU General Public License as published by +.\" the Free Software Foundation; either version 3, or (at your option) +.\" any later version. +.\" +.\" Vmod-binlog is distributed in the hope that it will be useful, +.\" but WITHOUT ANY WARRANTY; without even the implied warranty of +.\" MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +.\" GNU General Public License for more details. +.\" +.\" You should have received a copy of the GNU General Public License +.\" along with vmod-binlog. If not, see <http://www.gnu.org/licenses/>. +.TH BINLOGSEL 1 "October 16, 2013" "BINLOGSEL" "User Reference" +.SH NAME +binlogsel \- select records from binary logs +.SH SYNOPSIS +\fBbinlogsel\fR [\fB\-dnv\fR] [\fB\-t\fR \fIFORMAT\fR] [\fB\-F\fR +\fITIME\fR] [\fB\-T\fR \fITIME\fR] [\fB\-p\fR \fIPATTERN\fR]\ + [\fB\-D\fR \fIDIR\fR] + [\fIFILE\fR...] + +\fBbinlogsel\fR [\fB\-hV\fR] +.SH DESCRIPTION +.B Binlosel +selects from the binary logs the records that fall within the given +time interval. The interval is specified using the +.BI \-F " FROMTIME" +and +.BI \-T " TOTIME" +options. If both options are used, only records falling within the +interval \fIFROMTIME\fR..\fITOTIME\fR will be returned. If +.B \-F +is used, but +.B \-T +is omitted, \fITOTIME\fR is set to the current time. If +.B \-T +is used, but +.B \-F +is omitted, the starting time is set to the time of the earliest +available log file. Finally, if neither of options is used, all +records will be output. In this case, the operation is equivalent +to that of the +.B binlogcat +command. +.PP +The argument to either option is a mostly free format human readable +date string such as "Sun, 29 Feb 2004 16:21:42 -0800" or "2004-02-29 +16:21:42" or even "2 days ago". It may contain items indicating +calendar date, time of day, time zone, day of week, relative time, +relative date, and numbers. An empty string indicates the beginning +of the day. For a detailed description of date input formats, see +section +.B Date input formats +in +.BR "GNU Coreutils manual" . +If the +.B info +program and coreutils documentation are properly installed at your site, use +the following command: +.PP +.EX +.B info coreutils 'Date input formats' +.EE +.PP +The log files are searched in the directory specified with the +\fB\-D\fR command line option. The \fB\-i\fR option can be used to +define directory indexing level. +.PP +If files are listed in the command line, the +.BR \-D , +.BR \-i , +and +.B \-p +options are ignored. +.SH OPTIONS +.TP +.BI \-D " DIR" +Set base log directory. +.TP +.BI \-i " N" +Set directory indexing level. Valid values for \fIN\fR are \fB0\fR, +\fB1\fR and \fB2\fR. +.TP +.BI \-F " TIME" +Select records newer than \fITIME\fR. See the section +\fBDESCRIPTION\fR for a discussion of valid time formats. +.TP +.BI \-T " TIME" +Select records older than \fITIME\fR. +.TP +.BI \-p " PATTERN" +Set pattern for log file names. \fIPATTERN\fR must be a valid +.BR strftime (3) +format string. +.B \-d +Print timestamps relative to first record in the file. +.TP +.B \-n +Precede each record by its number in the file (0-based). +.TP +.B \-v +Print information about each file before dumpng it. +.TP +.BI \-t " FORMAT" +Format timestamps according to \fIFORMAT\fR (see +.BR strftime (3)). +Default is \fB%c\fR. +.TP +.BR \-h +Print a short help summary. +.TP +.BR \-V +Show program version and exit. +.SH "SEE ALSO" +.BR vmod-binlog (3), +.BR binlogcat (1), +.BR vcl (7), +.BR varnishd (1). +.SH AUTHORS +Sergey Poznyakoff +.SH "BUG REPORTS" +Report bugs to <gray@gnu.org>. +.SH COPYRIGHT +Copyright \(co 2013 Sergey Poznyakoff +.br +.na +License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html> +.br +.ad +This is free software: you are free to change and redistribute it. +There is NO WARRANTY, to the extent permitted by law. +.\" Local variables: +.\" eval: (add-hook 'write-file-hooks 'time-stamp) +.\" time-stamp-start: ".TH [A-Z_][A-Z0-9_.-]* [0-9] \"" +.\" time-stamp-format: "%:B %:d, %:y" +.\" time-stamp-end: "\"" +.\" time-stamp-line-limit: 20 +.\" end: + + diff --git a/doc/magic b/doc/magic new file mode 100644 index 0000000..a6c1642 --- /dev/null +++ b/doc/magic @@ -0,0 +1,7 @@ +# A "magic" file for file(1). +# +0 string NXCBINLOG NXC binary log file +>18 short x \b, version %d +>16 short x \b.%d +>48 string x \b, format %s +>40 long x \b, %d records diff --git a/doc/vmod-binlog.3 b/doc/vmod-binlog.3 new file mode 100644 index 0000000..f6eb94f --- /dev/null +++ b/doc/vmod-binlog.3 @@ -0,0 +1,287 @@ +.\" This file is part of Vmod-binlog -*- nroff -*- +.\" Copyright (C) 2013 Sergey Poznyakoff +.\" +.\" Vmod-binlog is free software; you can redistribute it and/or modify +.\" it under the terms of the GNU General Public License as published by +.\" the Free Software Foundation; either version 3, or (at your option) +.\" any later version. +.\" +.\" Vmod-binlog is distributed in the hope that it will be useful, +.\" but WITHOUT ANY WARRANTY; without even the implied warranty of +.\" MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +.\" GNU General Public License for more details. +.\" +.\" You should have received a copy of the GNU General Public License +.\" along with vmod-binlog. If not, see <http://www.gnu.org/licenses/>. +.TH VMOD-BINLOG 1 "October 16, 2013" "VMOD-BINLOG" "User Reference" +.SH NAME +vmod-binlog \- binary log file support for Varnish Cache. +.SH SYNOPSIS +.B import binlog; + +.BI "VOID binlog.init(STRING " dir ", STRING " format ", STRING " param ");" + +.B VOID binlog.start(); + +.BI "VOID binlog.pack(STRING " arg ");" + +.B VOID binlog.commit(); +.SH DESCRIPTION +This module provides adds binary log files support to +.BR "Varnish Cache" . +.PP +Binary log file consists of an arbitrary number of equally sized +records. Each record contains a UNIX timestamp in binary form and +user-defined data. The file begins with a header containing auxiliary +information. +.PP +Binary log files are rotated at preconfigured time intervals. In +order to speed up searches, their names follow the +.BR strftime (3) +pattern: +.PP +.EX +%Y%m%dT%H%M%S.log +.EE +.PP +where conversion specifiers refer to the time when the log was +created. The logs are stored in an indexed database structure. Three +levels of indexing are supported: +.TP +.B 0 +Directories indexed by the year of creation. E.g. the log for +September 22, 2013 is stored in + +.EX +$dir/2013/20130922T000000.log +.EE + +(where \fB$dir\fR refers to the top-level storage directory). This is +the default indexing scheme. +.TP +.B 1 +Directories indexed by the year and month of creation. E.g.: + +.EX +$dir/2013/09/20130922T000000.log +.EE +.TP +.B 2 +Directories indexed by the year, month and day. E.g.: + +.EX +$dir/2013/09/22/20130922T000000.log +.EE +.PP +Binary logs can be viewed using the +.BR binlogcat (3) +and +.BR binlogsel (3) +utilities. +.PP +Binary log support is initialized by a call to +.B binlog.init +function (normally it is done in +.BR vcl_init ). +.PP +The +.B dir +argument specifies the top-level storage directory. The +.B format argument defines the format of logfile entries. It is +a simplified form of Perl +.B pack() +template. See the +.B FORMAT +section below for a detailed description. Finally, +.B param +is a semicolon-separated list of parameters. Each parameter is +defined as a key/value pair, with a single equals sign in between. +No whitespace is allowed on either side of the equals sign. +The following parameters are defined: +.TP +\fBdebug\fR=\fINUMBER\fR +Set debugging level. +.TP +\fBpattern\fR=\fISTRING\fR +Sets pattern to use for log file names, instead of the default. +Argument can contain +.BR strftime (3) +conversion specifiers. The default is + +.EX +%Y%m%dT%H%M%S.log +.EE +.TP +\fBindex\fR=\fINUMBER\fR +Sets directory indexing scheme to use. Allowed values are \fB0\fR, +\fB1\fR, and \fB2\fR. See the description of indexing schemes above. +.TP +\fBsize\fR=\fISIZE\fR +Sets maximum size of a single binary log. Any records not fitting +into \fISIZE\fR will be discarded. \fISIZE\fR can be followed by +one of the usual size suffixes: \fBK\R, \fBM\fR, \fBG\fR (or their +lower-case equivalents), standing for kilobytes, megabytes and +gigabytes, correspondingly. + +Logs are mapped into memory, so +the \fISIZE\fR is limited by the amount of memory that can be mapped. +See +.BR mmap (2), +for the details. + +The default value is 1G. +.TP +\fBinterval\fR=\fINUMBER\fR +Sets log rotation interval, in seconds. The default is 86400. +.TP +\fBumask\fR=\fIONUM\fR +Sets umask for creating new log files. Argument is an octal value. +The default is \fB077\fR. +.TP +\fBroundts\fR=\fIBOOL\fR +If set to \fB1\fR, timestamps used for generating log file names will +be rounded to the nearest rotation interval. E.g., if the log file +was created at 01:05:18 on September 22, 2013 and \fBroundts=1\fR is +in effect, the log will be named \fB20130922T000000.log\fR, instead of +\fB20130922T010518.log\fR. +.TP +\fBreuselog\fR=\fIBOOL\fR +If the file to be created already exists upon \fBvarnishd\fR startup, the +module will try to append new data to it, provided that its format +matches the one actually configured. This avoids loss of data upon +restarting \fBvarnishd\fR. Setting \fBreuselog=0\fR disables this +behavior. +.PP +A new log record is initialized by +.BR binlog.start() . +This call should be followed by as many calls to +.B binlog.pack() +as there are conversions in the format string. Each call to +.B binlog.pack() +converts its argument according to the next unused format specifier +and stores it into the record being constructed. Finally, calling +.B binlog.commit() +writes the record to the log file. +.PP +For example, consider the format specification +.PP +.EX +L Z256 +.EE +.PP +(i.e. a 32-bit unsigned long value, followed by a string of up to 256 +characters). Given this, the following sequence of \fBVCL\fR commands +can be used to construct the log entry from the HTTP headers: +.PP +.EX +binlog.start(); +binlog.pack(http.X-Id); +binlog.pack(http.req.url); +binlog.commit(); +.EE +.SH FORMAT +Data format specification consists of a series of template letters +optionally followed by numeric repeat count (which may be enclosed in +square brackets). +.PP +The valid template letters are: +.TP +.BI Z [N] +A null-terminated (ASCIZ) string of at most N-1 characters, will be +null padded. This letter must be followed by repeat count. +.TP +.B c +A signed char (8-bit) value. +.TP +.B s +A signed short (16-bit) value. +.TP +.B S +An unsigned short value. +.TP +.B l +A signed long (32-bit) value. +.TP +.B L +An unsigned long value. +.TP +.B q +A signed quad (64-bit) value. +.TP +.B Q +An unsigned quad value. +.TP +.B i +A signed integer value. +.TP +.B I +A unsigned integer value. +.TP +.B n +An unsigned short (16-bit) in "network" (big-endian) order. +.TP +.B N +An unsigned long (32-bit) in "network" (big-endian) order. +.TP +.B v +An unsigned short (16-bit) in "VAX" (little-endian) order. +.TP +.B V +An unsigned long (32-bit) in "VAX" (little-endian) order. +.TP +.B f +A single-precision float in native format. +.TP +.B d +A double-precision float in native format. +.TP +.B x +A null byte (a.k.a ASCII NUL, "\000", chr(0)) +.TP +.B X +Back up a byte. +.TP +.B @ +Null-fill or truncate to absolute position, counted from the +current position. +.TP +.B . +Null-fill or truncate to absolute position specified by +the repeat count. + + +.\" The MANCGI variable is set by man.cgi script on Ulysses. +.\" The download.inc file contains the default DOWNLOAD section +.\" for man-based doc pages. +.if "\V[MANCGI]"WEBDOC" \{\ +. ds package vmod-binlog +. ds version 1.1 +. so download.inc +\} +.SH "SEE ALSO" +.BR binlogcat (1), +.BR binlogsel (1), +.BR vcl (7), +.BR varnishd (1). +.SH AUTHORS +Sergey Poznyakoff +.SH "BUG REPORTS" +Report bugs to <gray@gnu.org>. +.SH COPYRIGHT +Copyright \(co 2013 Sergey Poznyakoff +.br +.na +License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html> +.br +.ad +This is free software: you are free to change and redistribute it. +There is NO WARRANTY, to the extent permitted by law. +.\" Local variables: +.\" eval: (add-hook 'write-file-hooks 'time-stamp) +.\" time-stamp-start: ".TH [A-Z_][A-Z0-9_.-]* [0-9] \"" +.\" time-stamp-format: "%:B %:d, %:y" +.\" time-stamp-end: "\"" +.\" time-stamp-line-limit: 20 +.\" end: + + diff --git a/doc/vmod_binlog.3 b/doc/vmod_binlog.3 new file mode 100644 index 0000000..d3b8174 --- /dev/null +++ b/doc/vmod_binlog.3 @@ -0,0 +1 @@ +.so man3/vmod-binlog.3 diff --git a/src/binlogsel.c b/src/binlogsel.c index 63a35e7..173476b 100644 --- a/src/binlogsel.c +++ b/src/binlogsel.c @@ -53,10 +53,11 @@ void selglob(const char *dir, const char *pattern); void help() { - printf("usage: %s [-dhnVv] [-t FORMAT] [-F TIME] [-T TIME] [-p PATTERN] [-D DIR] [FILE...]\n", progname); + printf("usage: %s [-dhnVv] [-t FORMAT] [-F TIME] [-T TIME] [-p PATTERN] [-D DIR] [-i 0|1|2] [FILE...]\n", progname); printf("Select records from binary logs\n"); printf("\nOptions are:\n\n"); printf(" -D DIR log file storage directory\n"); + printf(" -i 0|1|2 select directory indexing level\n"); printf(" -F TIME print records starting from TIME\n"); printf(" -T TIME print records starting up to TIME\n"); printf(" -p PATTERN select files matching PATTERN\n"); |