From b5f2337ad45d26941bf533741d8bff1238d59b5d Mon Sep 17 00:00:00 2001 From: Sergey Poznyakoff Date: Thu, 21 Apr 2016 16:23:35 +0300 Subject: Rewrite SQL configuration support. Multiple configurations can be provided, by calling dbrw.config from branches of a conditional VCL statement. The only requirement is that exactly one dbrw.config is executed within a single HTTP session. This allows the user to select rewrite data from various databases depending on some conditions. Open SQL connections are stored in a pool and reused as needed. So far no limit on the number of connections is imposed. This is a TODO. Connections remain open during the lifetime of the module (i.e. for as long as varnish is not restarted). * src/dbrw.h (dbrw_config): New members: param_str, magic and list. (dbrw_connection): New members: magic, busy, list. * src/vmod_dbrw.c (dbrw_connection_get) (dbrw_connection_release): New functions. (dbrw_event): Install atexit function. (vmod_config): Keep a pool of configurations. (dbrw_sethdr) [VARNISHVERSION]: Remove leftover code. (vmod_rewrite): Rewrite. * src/vmod_dbrw.vcc (config, rewrite): Mark as PRIV_TASK. * doc/vmod-dbrw.3: Update docs. * doc/vmod-dbrw.texi: Update docs. * configure.ac: Raise version number --- configure.ac | 2 +- doc/vmod-dbrw.3 | 12 +-- doc/vmod-dbrw.texi | 12 +-- src/dbrw.h | 14 +++ src/sql.c | 1 + src/vmod_dbrw.c | 278 +++++++++++++++++++++++++++++------------------------ src/vmod_dbrw.vcc | 8 +- 7 files changed, 180 insertions(+), 147 deletions(-) diff --git a/configure.ac b/configure.ac index f0eb02e..becaeef 100644 --- a/configure.ac +++ b/configure.ac @@ -14,7 +14,7 @@ # You should have received a copy of the GNU General Public License # along with vmod-dbrw. If not, see . AC_PREREQ(2.69) -AC_INIT([vmod-dbrw], 2.0.90-4.1.0, [gray@gnu.org]) +AC_INIT([vmod-dbrw], 2.0.91-4.1.0, [gray@gnu.org]) AC_CONFIG_AUX_DIR([build-aux]) AC_CONFIG_MACRO_DIR([m4]) AC_CONFIG_SRCDIR(src/vmod_dbrw.vcc) diff --git a/doc/vmod-dbrw.3 b/doc/vmod-dbrw.3 index f2ee5fe..8c89cc4 100644 --- a/doc/vmod-dbrw.3 +++ b/doc/vmod-dbrw.3 @@ -13,7 +13,7 @@ .\" .\" You should have received a copy of the GNU General Public License .\" along with vmod-dbrw. If not, see . -.TH VMOD-DBRW 1 "November 12, 2014" "VMOD-DBRW" "User Reference" +.TH VMOD-DBRW 1 "April 21, 2016" "VMOD-DBRW" "User Reference" .SH NAME vmod-dbrw \- Database-driven rewrite rules for Varnish Cache .SH SYNOPSIS @@ -28,8 +28,9 @@ is a Varnish Cache module implementing database-driven rewrite procedures. Its intended use is for web sites that need an exceedingly big number of redirect and/or rewrite rules. .PP -The \fBconfig\fR function should be called upon startup, in \fBsub -vcl_init\fR. It configures the module and supplies it the information +The \fBdbrw.config\fR function should be called exactly once per HTTP session, +before calling \fBdbrw.rewrite\fR (normally it is done from +\fBvcl_recv\fR). It configures the module and supplies it the information necessary for accessing the database and interpreting the data obtained from it. .PP @@ -190,15 +191,12 @@ CREATE TABLE redirects ( VCL 3.0 code: .PP .EX -sub vcl_init { +sub vcl_recv { dbrw.config("mysql", "database=dbname;user=varnish", {"SELECT dest FROM redirects WHERE host='$host' AND url='$url'"}); -} - -sub vcl_recv { set req.http.X-Redirect-To = dbrw.rewrite("host=" + req.http.Host + ";" + "url=" + req.url); diff --git a/doc/vmod-dbrw.texi b/doc/vmod-dbrw.texi index 344b940..c0dbbea 100644 --- a/doc/vmod-dbrw.texi +++ b/doc/vmod-dbrw.texi @@ -30,7 +30,7 @@ Published by the Free Software Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -Copyright @copyright{} 2013-2014 Sergey Poznyakoff +Copyright @copyright{} 2013-2016 Sergey Poznyakoff Permission is granted to copy, distribute and/or modify this document under the terms of the GNU Free Documentation License, Version 1.3 or @@ -103,8 +103,6 @@ manageable. Some web sites implement thousands of rewrite rules. The purpose of this module is to facilitate deploying and handling them. -The module can be compiled for VCL 3.0 or 4.0. - @node Overview @chapter Overview @cindex rewrite rules @@ -115,7 +113,7 @@ Rewrite rules are stored in a MySQL or PostgreSQL database. The schema. It only needs to know the SQL query which is to be used to retrieve data. This query is supplied to the module, along with the credentials for accessing the database, by calling the @code{config} -function in the @code{vcl_init} subroutine of the Varnish +function in the @code{vcl_recv} subroutine of the Varnish configuration file. Once the module is configured, the @code{rewrite} function can be called @@ -220,12 +218,12 @@ If @var{flags} is NULL, it is ignored. @deftypefn {function} void config (string @var{dbtype}, string @var{params}, @ string @var{query}) -@cindex vcl_init +@cindex vcl_recv @cindex configuration @cindex initialization This function configures the module and provides it with the data necessary to connect and use the database. It is normally called from the -@code{vcl_init} subroutine. +@code{vcl_recv} subroutine. Arguments: @@ -345,7 +343,7 @@ The example below configures @command{vmod-dbrw} to use MySQL database @group import dbrw; -sub vcl_init @{ +sub vcl_recv @{ dbrw.config("mysql", "database=rewrite;user=varnish;password=guessme", @{"SELECT dest diff --git a/src/dbrw.h b/src/dbrw.h index 855720a..2156e0b 100644 --- a/src/dbrw.h +++ b/src/dbrw.h @@ -21,6 +21,12 @@ #include #include +#include "vcl.h" +#include "vrt.h" +#include "vcc_if.h" +#include "bin/varnishd/cache/cache.h" +#define WSPTR(s) ((s)->ws) + struct dbrw_connection; struct dbrw_backend { @@ -50,21 +56,29 @@ enum { #define HTTP_STATUS_LEN 3 struct dbrw_config { + unsigned magic; +#define DBRW_CONFIG_MAGIC 0x67636667 int debug_level; struct dbrw_backend *backend; char **param; + char *param_str; char *query; int qdisp; int regflags; char status[HTTP_STATUS_LEN+1]; + VTAILQ_ENTRY(dbrw_config) list; }; struct dbrw_connection { + unsigned magic; +#define DBRW_CONNECTION_MAGIC 0x6773716c int state; /* Connection state */ + int busy:1; struct dbrw_config *conf; /* Pointer to the configuration data */ regmatch_t *matches; /* Match map */ size_t matchsize; /* Total number of entries in match map */ void *data; /* Backend-specific data */ + VTAILQ_ENTRY(dbrw_connection) list; }; void dbrw_debug(const char *fmt, ...); diff --git a/src/sql.c b/src/sql.c index b9ba3a4..c42c5de 100644 --- a/src/sql.c +++ b/src/sql.c @@ -184,3 +184,4 @@ sql_get_column(struct dbrw_connection *conn, unsigned row, unsigned col) return NULL; return conn->conf->backend->sql_get_column(conn, row, col); } + diff --git a/src/vmod_dbrw.c b/src/vmod_dbrw.c index 5c88199..d212eb7 100644 --- a/src/vmod_dbrw.c +++ b/src/vmod_dbrw.c @@ -17,17 +17,8 @@ #include "dbrw.h" #include #include "wordsplit.h" -#include "vcl.h" -#include "vrt.h" -#include "vcc_if.h" #include "pthread.h" -#include "bin/varnishd/cache/cache.h" -#define WSPTR(s) ((s)->ws) - -static pthread_once_t thread_once = PTHREAD_ONCE_INIT; -static pthread_key_t thread_key; - enum { QDISP_NONE, QDISP_APPEND, @@ -51,32 +42,80 @@ dbrw_error(const char *fmt, ...) vsyslog(LOG_DAEMON|LOG_ERR, fmt, ap); va_end(ap); } + +static pthread_mutex_t config_pool_mtx = PTHREAD_MUTEX_INITIALIZER; +static VTAILQ_HEAD(, dbrw_config) config_pool = + VTAILQ_HEAD_INITIALIZER(config_pool); + +static pthread_mutex_t connect_pool_mtx = PTHREAD_MUTEX_INITIALIZER; +static VTAILQ_HEAD(, dbrw_connection) connect_pool = + VTAILQ_HEAD_INITIALIZER(connect_pool); + +static struct dbrw_connection * +dbrw_connection_get(struct dbrw_config *cfg) +{ + struct dbrw_connection *cp; + + AZ(pthread_mutex_lock(&connect_pool_mtx)); + VTAILQ_FOREACH(cp, &connect_pool, list) { + CHECK_OBJ_NOTNULL(cp, DBRW_CONNECTION_MAGIC); + if (cp->conf == cfg && !cp->busy) + break; + } + if (!cp) { + ALLOC_OBJ(cp, DBRW_CONNECTION_MAGIC); + AN(cp); + cp->state = state_init; + cp->conf = cfg; + cp->data = NULL; + cp->matches = NULL; + cp->matchsize = 0; + + if (sql_init(cp)) { + free(cp); + cp = NULL; + } else + VTAILQ_INSERT_HEAD(&connect_pool, cp, list); + } + cp->busy = 1; + pthread_mutex_unlock(&connect_pool_mtx); + return cp; +} static void -conn_free(void *f) +dbrw_connection_release(struct dbrw_connection *cp) { - struct dbrw_connection *p = f; - sql_disconnect(p); - sql_destroy(p); + AZ(pthread_mutex_lock(&connect_pool_mtx)); + cp->busy = 0; + pthread_mutex_unlock(&connect_pool_mtx); } static void -make_key() +disconnect(void) { - pthread_key_create(&thread_key, conn_free); + AZ(pthread_mutex_lock(&connect_pool_mtx)); + while (!VTAILQ_EMPTY(&connect_pool)) { + struct dbrw_connection *cp = VTAILQ_FIRST(&connect_pool); + VTAILQ_REMOVE(&connect_pool, cp, list); + sql_disconnect(cp); + sql_destroy(cp); + } + pthread_mutex_unlock(&connect_pool_mtx); } int dbrw_event(VRT_CTX, struct vmod_priv *priv, enum vcl_event_e e) { - if (e == VCL_EVENT_LOAD) { - struct dbrw_config *conf = calloc(1, sizeof(*conf)); - if (!conf) { - dbrw_error("not enough memory"); - abort(); - } - priv->priv = conf; - pthread_once(&thread_once, make_key); + switch (e) { + case VCL_EVENT_LOAD: + atexit(disconnect); + break; + case VCL_EVENT_DISCARD: + break; + + default: + /* ignore */ + break; } return 0; } @@ -176,57 +215,67 @@ VCL_VOID vmod_config(VRT_CTX, struct vmod_priv *priv, VCL_STRING bkname, VCL_STRING param, VCL_STRING query) { - char *s; - struct wordsplit ws; - struct dbrw_config *conf = priv->priv; + struct dbrw_config *conf; + struct dbrw_backend *backend; - ws.ws_delim = ";"; - if (wordsplit(param, &ws, - WRDSF_NOVAR | WRDSF_NOCMD | WRDSF_QUOTE | - WRDSF_CESCAPES | - WRDSF_DELIM)) { - dbrw_error("cannot split string `%s': %s", - param, wordsplit_strerror (&ws)); - return; - } - conf->param = ws.ws_wordv; - ws.ws_wordv = NULL; - ws.ws_wordc = 0; - wordsplit_free(&ws); - - conf->query = strdup(query); - if (!conf->query) { - dbrw_error("not enough memory"); - argv_free(conf->param); - conf->param = NULL; - return; - } - - /* Select backend */ - conf->backend = dbrw_backend_select(bkname); - if (!conf->backend) { + AZ(priv->priv); + backend = dbrw_backend_select(bkname); + if (!backend) { dbrw_error("unsupported backend: %s", bkname); - argv_free(conf->param); - free(conf->query); - conf->query = NULL; - conf->param = NULL; - return; + abort(); } - s = findparam(conf->param, "debug"); - conf->debug_level = s ? atoi(s) : 0; - - conf->qdisp = QDISP_NONE; - conf->regflags = REG_EXTENDED; - conf->status[0] = 0; - - s = findparam(conf->param, "flags"); - if (s && parse_flags(s, &conf->qdisp, &conf->regflags, conf->status)) { - argv_free(conf->param); - free(conf->query); - conf->query = NULL; - conf->param = NULL; + AZ(pthread_mutex_lock(&config_pool_mtx)); + VTAILQ_FOREACH(conf, &config_pool, list) { + CHECK_OBJ_NOTNULL(conf, DBRW_CONFIG_MAGIC); + if (conf->backend == backend + && strcmp(conf->query, query) == 0 + && strcmp(conf->param_str, param) == 0) + break; + } + pthread_mutex_unlock(&config_pool_mtx); + + if (!conf) { + char *s; + struct wordsplit ws; + ws.ws_delim = ";"; + if (wordsplit(param, &ws, + WRDSF_NOVAR | WRDSF_NOCMD | WRDSF_QUOTE | + WRDSF_CESCAPES | + WRDSF_DELIM)) { + dbrw_error("cannot split string `%s': %s", + param, wordsplit_strerror (&ws)); + abort(); + } + ALLOC_OBJ(conf, DBRW_CONFIG_MAGIC); + AN(conf); + conf->param = ws.ws_wordv; + ws.ws_wordv = NULL; + ws.ws_wordc = 0; + wordsplit_free(&ws); + conf->param_str = strdup(param); + AN(conf->param_str); + conf->query = strdup(query); + AN(conf->query); + conf->backend = backend; + + s = findparam(conf->param, "debug"); + conf->debug_level = s ? atoi(s) : 0; + + conf->qdisp = QDISP_NONE; + conf->regflags = REG_EXTENDED; + conf->status[0] = 0; + + s = findparam(conf->param, "flags"); + if (s) + parse_flags(s, &conf->qdisp, &conf->regflags, + conf->status); + + AZ(pthread_mutex_lock(&config_pool_mtx)); + VTAILQ_INSERT_HEAD(&config_pool, conf, list); + pthread_mutex_unlock(&config_pool_mtx); } + priv->priv = conf; } static char * @@ -308,12 +357,8 @@ expand_backref(VRT_CTX, const char *str, const char *val, static void dbrw_sethdr(VRT_CTX, int where, const char *what, const char *value) { -#if VARNISHVERSION == 3 - VRT_SetHdr(ctx, where, what, value, vrt_magic_string_end); -#else struct gethdr_s s = { where, what }; VRT_SetHdr(ctx, &s, value, vrt_magic_string_end); -#endif } static char * @@ -447,59 +492,17 @@ findmatch(VRT_CTX, struct dbrw_connection *conn, char **param) return res; } -static struct dbrw_connection * -get_connection(struct dbrw_config *conf) -{ - struct dbrw_connection *cp = pthread_getspecific(thread_key); - - if (!cp) { - cp = malloc(sizeof(*cp)); - if (!cp) - return NULL; - cp->state = state_init; - cp->conf = conf; - cp->data = NULL; - cp->matches = NULL; - cp->matchsize = 0; - - if (sql_init(cp)) { - free(cp); - return NULL; - } - if (pthread_setspecific(thread_key, cp)) { - dbrw_error("cannot set thread-specific data"); - sql_destroy(cp); - free(cp); - return NULL; - } - } - return cp; -} - -VCL_STRING -vmod_rewrite(VRT_CTX, struct vmod_priv *priv, VCL_STRING arg) +static char * +do_rewrite(VRT_CTX, struct dbrw_connection *cp, VCL_STRING arg) { - struct dbrw_config *conf = priv->priv; - struct dbrw_connection *cp; struct wordsplit ws, wsenv; int i, rc; - char *res; + char *res; - debug(conf, 2, ("vmod_rewrite(%s) begin", arg)); - if (!conf || !conf->query) { - debug(conf, 2, ("vmod_rewrite: not configured; exiting")); - return NULL; - } - cp = get_connection(conf); - if (!cp) { - debug(conf, 2, ("vmod_rewrite: no private data; exiting")); - return NULL; - } - if (sql_connect(cp) || cp->state != state_connected) return NULL; - debug(conf, 2, ("vmod_rewrite: splitting arg")); + debug(cp->conf, 2, ("vmod_rewrite: splitting arg")); wsenv.ws_delim = ";"; if (wordsplit(arg, &wsenv, WRDSF_NOVAR|WRDSF_NOCMD|WRDSF_DELIM)) { dbrw_error("cannot split string `%s': %s", @@ -507,8 +510,8 @@ vmod_rewrite(VRT_CTX, struct vmod_priv *priv, VCL_STRING arg) return NULL; } - if (conf->backend->sql_escape) { - debug(conf, 2, ("escaping variables")); + if (cp->conf->backend->sql_escape) { + debug(cp->conf, 2, ("escaping variables")); for (i = 0; i < wsenv.ws_wordc; i++) { char *p = sql_escape(cp, wsenv.ws_wordv[i]); if (!p) { @@ -518,19 +521,19 @@ vmod_rewrite(VRT_CTX, struct vmod_priv *priv, VCL_STRING arg) } free(wsenv.ws_wordv[i]); wsenv.ws_wordv[i] = p; - debug(conf, 3, ("%d: %s",i,p)); + debug(cp->conf, 3, ("%d: %s",i,p)); } } - debug(conf, 2, ("expanding query")); + debug(cp->conf, 2, ("expanding query")); ws.ws_env = (const char **)wsenv.ws_wordv; - rc = wordsplit(conf->query, &ws, + rc = wordsplit(cp->conf->query, &ws, WRDSF_NOCMD | WRDSF_QUOTE | WRDSF_NOSPLIT | WRDSF_ENV | WRDSF_UNDEF); if (rc) { dbrw_error("cannot expand query `%s': %s", - conf->query, wordsplit_strerror(&ws)); + cp->conf->query, wordsplit_strerror(&ws)); wordsplit_free(&wsenv); return NULL; } @@ -539,7 +542,7 @@ vmod_rewrite(VRT_CTX, struct vmod_priv *priv, VCL_STRING arg) wordsplit_free(&ws); if (rc) { - debug(conf, 1, ("vmod_rewrite: query failed")); + debug(cp->conf, 1, ("vmod_rewrite: query failed")); wordsplit_free(&wsenv); return NULL; } @@ -549,7 +552,26 @@ vmod_rewrite(VRT_CTX, struct vmod_priv *priv, VCL_STRING arg) sql_free_result(cp); - debug(conf, 1, ("vmod_rewrite: res=%s", res ? res : "(NULL)")); - + return res; +} + +VCL_STRING +vmod_rewrite(VRT_CTX, struct vmod_priv *priv, VCL_STRING arg) +{ + struct dbrw_config *conf = priv->priv; + struct dbrw_connection *cp; + char *res = NULL; + + AN(priv->priv); + debug(conf, 2, ("vmod_rewrite(%s) begin", arg)); + if (!conf || !conf->query) { + debug(conf, 2, ("vmod_rewrite: not configured; exiting")); + return NULL; + } + cp = dbrw_connection_get(conf); + AN(cp); + res = do_rewrite(ctx, cp, arg); + dbrw_connection_release(cp); + debug(conf, 1, ("vmod_rewrite: res=%s", res ? res : "(NULL)")); return res; } diff --git a/src/vmod_dbrw.vcc b/src/vmod_dbrw.vcc index 827ff4f..9a15af1 100644 --- a/src/vmod_dbrw.vcc +++ b/src/vmod_dbrw.vcc @@ -1,5 +1,5 @@ # This file is part of vmod-dbrw -# Copyright (C) 2013-2014 Sergey Poznyakoff +# Copyright (C) 2013-2016 Sergey Poznyakoff # # Vmod-dbrw is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -39,12 +39,12 @@ httpd server, and that they are stored in an SQL database, which makes them easily manageable. $Event dbrw_event -$Function VOID config(PRIV_VCL, STRING, STRING, STRING) +$Function VOID config(PRIV_TASK, STRING, STRING, STRING) Description Configures the module and provides it with the data necessary to connect and use the database. It is normally called - from the **vcl_init** subroutine. + from the **vcl_recv** subroutine. Example :: @@ -57,7 +57,7 @@ Example AND url='$url'"}); -$Function STRING rewrite(PRIV_VCL, STRING) +$Function STRING rewrite(PRIV_TASK, STRING) Description Rewrites its argument using the database configured in the previous -- cgit v1.2.1