From ef3f928cc712f6774e19f8cb8f880cbeac15e8ff Mon Sep 17 00:00:00 2001 From: Sergey Poznyakoff Date: Tue, 15 Oct 2013 18:38:49 +0300 Subject: Change default log naming. * configure.ac: Call AM_PROG_CC_C_O * src/binlog.c (BLF_TRUNCATE): New flag. (vmod_init): Change pattern initialization. New parameter "reuselog". (createfile): Remove O_TRUNC. (checkheader): New function. (newfile): Reuse existing file, if it is the first file to be opened after varnish startup and its header matches exactly our data. * src/binlogsel.c: Use indexed directory structure to speed up searches. * src/vmod-binlog.h (BINLOG_PATTERN): Change pattern. (BINLOG_GLOB_PATTERN,BINLOG_INDEX): New defines. --- src/binlogsel.c | 245 ++++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 228 insertions(+), 17 deletions(-) (limited to 'src/binlogsel.c') diff --git a/src/binlogsel.c b/src/binlogsel.c index 62299e5..8f1cfce 100644 --- a/src/binlogsel.c +++ b/src/binlogsel.c @@ -38,21 +38,26 @@ char *timefmt = "%c"; int number_option; int verbose_option; int timediff_option; +char *directory; char *pattern; +enum binlog_index_type index_type = index_year; #define FROM_TIME 0x01 #define TO_TIME 0x02 int timemask; time_t from_time, to_time; +static int matchnames(const char *dir, const char *pat, glob_t *gl); +void selglob(const char *dir, const char *pattern); + void help() { printf("usage: %s [-dhnv] [-t FORMAT] [-F FROMTIME] [-T TOTIME] [-p PATTERN] [-D DIR] [FILE...]\n", progname); } - + /* Convert strftime-like pattern into globbing pattern */ -void +char * convpattern(const char *dir) { char *p, *q; @@ -73,13 +78,14 @@ convpattern(const char *dir) for (q = pattern; *q; ) { if (*q == '%') { - *p++ = '*'; + if (p > newpat && p[-1] != '*') + *p++ = '*'; q += 2; } else *p++ = *q++; } *p = 0; - pattern = newpat; + return newpat; } #define getrec(base, recsize, n) \ @@ -332,14 +338,212 @@ selfilelist(char **argv) for (;*argv;++argv) selfile(*argv); } + +static char * +mkfilename(const char *dir, const char *file) +{ + size_t dirlen, size; + char *ret; + + dirlen = strlen(dir); + while (dirlen > 0 && dir[dirlen-1] == '/') + --dirlen; + size = dirlen + 1 + strlen(file) + 1; + ret = xmalloc(size); + memcpy(ret, dir, dirlen); + ret[dirlen++] = '/'; + strcpy(ret + dirlen, file); + return ret; +} + +int +filename_to_int(char *name) +{ + char *p = strrchr(name, '/'); + if (!p) + abort(); + return atoi(p + 1); +} + +void +selidx_day(const char *dir) +{ + int from_day, to_day; + struct tm *tm; + glob_t gl; + int glinit = 0; + char *dirbuf; + size_t dirlen; + + if (index_type == index_month) { + selglob(dir, BINLOG_GLOB_PATTERN); + return; + } + + if (timemask & FROM_TIME) + from_day = gmtime(&from_time)->tm_mday; + else { + glinit = matchnames(dir, "[0-9][0-9]", &gl); + if (glinit) + from_day = filename_to_int(gl.gl_pathv[0]); + else { + error("no matching files"); + exit(1); + } + } + + if (timemask & TO_TIME) + to_day = gmtime(&to_time)->tm_mday; + else { + if (!glinit) { + glinit = matchnames(dir, "[0-9][0-9]", &gl); + if (!glinit) { + error("no matching files"); + exit(1); + } + } + to_day = filename_to_int(gl.gl_pathv[gl.gl_pathc - 1]); + } + + dirlen = strlen(dir) + 4; + dirbuf = xmalloc(dirlen); + for (;from_day <= to_day; from_day++) { + snprintf(dirbuf, dirlen, "%s/%02d", dir, from_day); + selglob(dirbuf, BINLOG_GLOB_PATTERN); + } + free(dirbuf); + if (glinit) + globfree(&gl); +} + +void +selidx_month(const char *dir) +{ + int from_month, to_month; + struct tm *tm; + glob_t gl; + int glinit = 0; + char *dirbuf; + size_t dirlen; + + if (index_type == index_year) { + selglob(dir, BINLOG_GLOB_PATTERN); + return; + } + + if (timemask & FROM_TIME) + from_month = 1 + gmtime(&from_time)->tm_mon; + else { + glinit = matchnames(dir, "[0-9][0-9]", &gl); + if (glinit) + from_month = filename_to_int(gl.gl_pathv[0]); + else { + error("no matching files"); + exit(1); + } + } + + if (timemask & TO_TIME) + to_month = 1 + gmtime(&to_time)->tm_mon; + else { + if (!glinit) { + glinit = matchnames(dir, "[0-9][0-9]", &gl); + if (!glinit) { + error("no matching files"); + exit(1); + } + } + to_month = filename_to_int(gl.gl_pathv[gl.gl_pathc - 1]); + } + + dirlen = strlen(dir) + 4; + dirbuf = xmalloc(dirlen); + for (;from_month <= to_month; from_month++) { + snprintf(dirbuf, dirlen, "%s/%02d", dir, from_month); + selidx_day(dirbuf); + } + free(dirbuf); + if (glinit) + globfree(&gl); +} + +void +selidx_year(const char *dir) +{ + int from_year, to_year; + struct tm *tm; + glob_t gl; + int glinit = 0; + char *dirbuf; + size_t dirlen; + + if (timemask & FROM_TIME) + from_year = 1900 + gmtime(&from_time)->tm_year; + else { + glinit = matchnames(dir, "[0-9][0-9][0-9][0-9]", &gl); + if (glinit) + from_year = filename_to_int(gl.gl_pathv[0]); + else { + error("no matching files"); + exit(1); + } + } + if (timemask & TO_TIME) + to_year = 1900 + gmtime(&to_time)->tm_year; + else { + if (!glinit) { + glinit = matchnames(dir, "[0-9][0-9][0-9][0-9]", &gl); + if (!glinit) { + error("no matching files"); + exit(1); + } + } + to_year = filename_to_int(gl.gl_pathv[gl.gl_pathc - 1]); + } + + dirlen = strlen(dir) + 6; + dirbuf = xmalloc(dirlen); + for (;from_year <= to_year; from_year++) { + snprintf(dirbuf, dirlen, "%s/%04d", dir, from_year); + selidx_month(dirbuf); + } + free(dirbuf); + if (glinit) + globfree(&gl); +} + int globerrfunc (const char *epath, int eerrno) { - error("%s: %s", strerror(eerrno)); + error("%s: %s", epath, strerror(eerrno)); return 0; } +static int +matchnames(const char *dir, const char *pat, glob_t *gl) +{ + char *p = mkfilename(dir, pat); + int rc = glob(p, GLOB_ERR, globerrfunc, gl); + free(p); + switch (rc) { + case 0: + break; + case GLOB_NOSPACE: + error("out of memory"); + exit(1); + + case GLOB_ABORTED: + error("read error"); + exit(1); + + case GLOB_NOMATCH: + return 0; + } + return 1; +} + + struct logfile { char *name; time_t start; @@ -358,13 +562,14 @@ tsort(const void *a, const void *b) } void -selpattern(void) +selglob(const char *dir, const char *pattern) { size_t i, j; glob_t gl; struct logfile *logfiles; - - switch (glob(pattern, GLOB_ERR|GLOB_NOSORT, globerrfunc, &gl)) { + char *p = mkfilename(dir, pattern); + + switch (glob(p, GLOB_ERR|GLOB_NOSORT, globerrfunc, &gl)) { case 0: break; case GLOB_NOSPACE: @@ -379,7 +584,8 @@ selpattern(void) error("no files matched pattern"); exit(1); } - + free(p); + logfiles = xcalloc(gl.gl_pathc, sizeof(*logfiles)); for (i = j = 0; i < gl.gl_pathc; i++) { @@ -400,16 +606,15 @@ selpattern(void) free(logfiles); globfree(&gl); } - + int main(int argc, char **argv) { int c; struct timespec ts; - char *directory; setprogname(argv[0]); - while ((c = getopt(argc, argv, "D:dF:hp:T:t:nv")) != EOF) + while ((c = getopt(argc, argv, "D:dF:hi:p:T:t:nv")) != EOF) switch (c) { case 'D': directory = optarg; @@ -429,6 +634,13 @@ main(int argc, char **argv) case 'h': help(); return 0; + case 'i': + index_type = atoi(optarg); + if (index_type < 0 || index_type > index_last) { + error("invalid index type: %s", optarg); + exit(1); + } + break; case 'p': pattern = optarg; break; @@ -457,17 +669,16 @@ main(int argc, char **argv) argv += optind; if (argc) { - if (pattern) { + if (pattern || directory) { error("either files or pattern (-p) must be given, " "but not both"); exit(1); } selfilelist(argv); + } else if (pattern) { + selglob(directory, convpattern(pattern)); } else { - if (!pattern) - pattern = BINLOG_PATTERN; - convpattern(directory); - selpattern(); + selidx_year(directory); } exit(0); } -- cgit v1.2.1