diff options
author | Sergey Poznyakoff <gray@gnu.org.ua> | 2016-04-21 16:23:35 +0300 |
---|---|---|
committer | Sergey Poznyakoff <gray@gnu.org.ua> | 2016-04-21 16:47:22 +0300 |
commit | b5f2337ad45d26941bf533741d8bff1238d59b5d (patch) | |
tree | 878a65c333272ceb198ec0d8e871e431f8015f7a /src | |
parent | 5ee1409fc878cd65dd224ec5f709507e37e9ed5d (diff) | |
download | vmod-dbrw-b5f2337ad45d26941bf533741d8bff1238d59b5d.tar.gz vmod-dbrw-b5f2337ad45d26941bf533741d8bff1238d59b5d.tar.bz2 |
Rewrite SQL configuration support.
Multiple configurations can be provided, by calling dbrw.config from
branches of a conditional VCL statement. The only requirement is that
exactly one dbrw.config is executed within a single HTTP session. This
allows the user to select rewrite data from various databases depending
on some conditions.
Open SQL connections are stored in a pool and reused as needed. So far
no limit on the number of connections is imposed. This is a TODO.
Connections remain open during the lifetime of the module (i.e. for as
long as varnish is not restarted).
* src/dbrw.h (dbrw_config): New members: param_str, magic and list.
(dbrw_connection): New members: magic, busy, list.
* src/vmod_dbrw.c (dbrw_connection_get)
(dbrw_connection_release): New functions.
(dbrw_event): Install atexit function.
(vmod_config): Keep a pool of configurations.
(dbrw_sethdr) [VARNISHVERSION]: Remove leftover code.
(vmod_rewrite): Rewrite.
* src/vmod_dbrw.vcc (config, rewrite): Mark as PRIV_TASK.
* doc/vmod-dbrw.3: Update docs.
* doc/vmod-dbrw.texi: Update docs.
* configure.ac: Raise version number
Diffstat (limited to 'src')
-rw-r--r-- | src/dbrw.h | 14 | ||||
-rw-r--r-- | src/sql.c | 1 | ||||
-rw-r--r-- | src/vmod_dbrw.c | 278 | ||||
-rw-r--r-- | src/vmod_dbrw.vcc | 8 |
4 files changed, 169 insertions, 132 deletions
@@ -21,6 +21,12 @@ #include <syslog.h> #include <regex.h> +#include "vcl.h" +#include "vrt.h" +#include "vcc_if.h" +#include "bin/varnishd/cache/cache.h" +#define WSPTR(s) ((s)->ws) + struct dbrw_connection; struct dbrw_backend { @@ -50,21 +56,29 @@ enum { #define HTTP_STATUS_LEN 3 struct dbrw_config { + unsigned magic; +#define DBRW_CONFIG_MAGIC 0x67636667 int debug_level; struct dbrw_backend *backend; char **param; + char *param_str; char *query; int qdisp; int regflags; char status[HTTP_STATUS_LEN+1]; + VTAILQ_ENTRY(dbrw_config) list; }; struct dbrw_connection { + unsigned magic; +#define DBRW_CONNECTION_MAGIC 0x6773716c int state; /* Connection state */ + int busy:1; struct dbrw_config *conf; /* Pointer to the configuration data */ regmatch_t *matches; /* Match map */ size_t matchsize; /* Total number of entries in match map */ void *data; /* Backend-specific data */ + VTAILQ_ENTRY(dbrw_connection) list; }; void dbrw_debug(const char *fmt, ...); @@ -184,3 +184,4 @@ sql_get_column(struct dbrw_connection *conn, unsigned row, unsigned col) return NULL; return conn->conf->backend->sql_get_column(conn, row, col); } + diff --git a/src/vmod_dbrw.c b/src/vmod_dbrw.c index 5c88199..d212eb7 100644 --- a/src/vmod_dbrw.c +++ b/src/vmod_dbrw.c @@ -17,17 +17,8 @@ #include "dbrw.h" #include <stdarg.h> #include "wordsplit.h" -#include "vcl.h" -#include "vrt.h" -#include "vcc_if.h" #include "pthread.h" -#include "bin/varnishd/cache/cache.h" -#define WSPTR(s) ((s)->ws) - -static pthread_once_t thread_once = PTHREAD_ONCE_INIT; -static pthread_key_t thread_key; - enum { QDISP_NONE, QDISP_APPEND, @@ -51,32 +42,80 @@ dbrw_error(const char *fmt, ...) vsyslog(LOG_DAEMON|LOG_ERR, fmt, ap); va_end(ap); } + +static pthread_mutex_t config_pool_mtx = PTHREAD_MUTEX_INITIALIZER; +static VTAILQ_HEAD(, dbrw_config) config_pool = + VTAILQ_HEAD_INITIALIZER(config_pool); + +static pthread_mutex_t connect_pool_mtx = PTHREAD_MUTEX_INITIALIZER; +static VTAILQ_HEAD(, dbrw_connection) connect_pool = + VTAILQ_HEAD_INITIALIZER(connect_pool); + +static struct dbrw_connection * +dbrw_connection_get(struct dbrw_config *cfg) +{ + struct dbrw_connection *cp; + + AZ(pthread_mutex_lock(&connect_pool_mtx)); + VTAILQ_FOREACH(cp, &connect_pool, list) { + CHECK_OBJ_NOTNULL(cp, DBRW_CONNECTION_MAGIC); + if (cp->conf == cfg && !cp->busy) + break; + } + if (!cp) { + ALLOC_OBJ(cp, DBRW_CONNECTION_MAGIC); + AN(cp); + cp->state = state_init; + cp->conf = cfg; + cp->data = NULL; + cp->matches = NULL; + cp->matchsize = 0; + + if (sql_init(cp)) { + free(cp); + cp = NULL; + } else + VTAILQ_INSERT_HEAD(&connect_pool, cp, list); + } + cp->busy = 1; + pthread_mutex_unlock(&connect_pool_mtx); + return cp; +} static void -conn_free(void *f) +dbrw_connection_release(struct dbrw_connection *cp) { - struct dbrw_connection *p = f; - sql_disconnect(p); - sql_destroy(p); + AZ(pthread_mutex_lock(&connect_pool_mtx)); + cp->busy = 0; + pthread_mutex_unlock(&connect_pool_mtx); } static void -make_key() +disconnect(void) { - pthread_key_create(&thread_key, conn_free); + AZ(pthread_mutex_lock(&connect_pool_mtx)); + while (!VTAILQ_EMPTY(&connect_pool)) { + struct dbrw_connection *cp = VTAILQ_FIRST(&connect_pool); + VTAILQ_REMOVE(&connect_pool, cp, list); + sql_disconnect(cp); + sql_destroy(cp); + } + pthread_mutex_unlock(&connect_pool_mtx); } int dbrw_event(VRT_CTX, struct vmod_priv *priv, enum vcl_event_e e) { - if (e == VCL_EVENT_LOAD) { - struct dbrw_config *conf = calloc(1, sizeof(*conf)); - if (!conf) { - dbrw_error("not enough memory"); - abort(); - } - priv->priv = conf; - pthread_once(&thread_once, make_key); + switch (e) { + case VCL_EVENT_LOAD: + atexit(disconnect); + break; + case VCL_EVENT_DISCARD: + break; + + default: + /* ignore */ + break; } return 0; } @@ -176,57 +215,67 @@ VCL_VOID vmod_config(VRT_CTX, struct vmod_priv *priv, VCL_STRING bkname, VCL_STRING param, VCL_STRING query) { - char *s; - struct wordsplit ws; - struct dbrw_config *conf = priv->priv; + struct dbrw_config *conf; + struct dbrw_backend *backend; - ws.ws_delim = ";"; - if (wordsplit(param, &ws, - WRDSF_NOVAR | WRDSF_NOCMD | WRDSF_QUOTE | - WRDSF_CESCAPES | - WRDSF_DELIM)) { - dbrw_error("cannot split string `%s': %s", - param, wordsplit_strerror (&ws)); - return; - } - conf->param = ws.ws_wordv; - ws.ws_wordv = NULL; - ws.ws_wordc = 0; - wordsplit_free(&ws); - - conf->query = strdup(query); - if (!conf->query) { - dbrw_error("not enough memory"); - argv_free(conf->param); - conf->param = NULL; - return; - } - - /* Select backend */ - conf->backend = dbrw_backend_select(bkname); - if (!conf->backend) { + AZ(priv->priv); + backend = dbrw_backend_select(bkname); + if (!backend) { dbrw_error("unsupported backend: %s", bkname); - argv_free(conf->param); - free(conf->query); - conf->query = NULL; - conf->param = NULL; - return; + abort(); } - s = findparam(conf->param, "debug"); - conf->debug_level = s ? atoi(s) : 0; - - conf->qdisp = QDISP_NONE; - conf->regflags = REG_EXTENDED; - conf->status[0] = 0; - - s = findparam(conf->param, "flags"); - if (s && parse_flags(s, &conf->qdisp, &conf->regflags, conf->status)) { - argv_free(conf->param); - free(conf->query); - conf->query = NULL; - conf->param = NULL; + AZ(pthread_mutex_lock(&config_pool_mtx)); + VTAILQ_FOREACH(conf, &config_pool, list) { + CHECK_OBJ_NOTNULL(conf, DBRW_CONFIG_MAGIC); + if (conf->backend == backend + && strcmp(conf->query, query) == 0 + && strcmp(conf->param_str, param) == 0) + break; + } + pthread_mutex_unlock(&config_pool_mtx); + + if (!conf) { + char *s; + struct wordsplit ws; + ws.ws_delim = ";"; + if (wordsplit(param, &ws, + WRDSF_NOVAR | WRDSF_NOCMD | WRDSF_QUOTE | + WRDSF_CESCAPES | + WRDSF_DELIM)) { + dbrw_error("cannot split string `%s': %s", + param, wordsplit_strerror (&ws)); + abort(); + } + ALLOC_OBJ(conf, DBRW_CONFIG_MAGIC); + AN(conf); + conf->param = ws.ws_wordv; + ws.ws_wordv = NULL; + ws.ws_wordc = 0; + wordsplit_free(&ws); + conf->param_str = strdup(param); + AN(conf->param_str); + conf->query = strdup(query); + AN(conf->query); + conf->backend = backend; + + s = findparam(conf->param, "debug"); + conf->debug_level = s ? atoi(s) : 0; + + conf->qdisp = QDISP_NONE; + conf->regflags = REG_EXTENDED; + conf->status[0] = 0; + + s = findparam(conf->param, "flags"); + if (s) + parse_flags(s, &conf->qdisp, &conf->regflags, + conf->status); + + AZ(pthread_mutex_lock(&config_pool_mtx)); + VTAILQ_INSERT_HEAD(&config_pool, conf, list); + pthread_mutex_unlock(&config_pool_mtx); } + priv->priv = conf; } static char * @@ -308,12 +357,8 @@ expand_backref(VRT_CTX, const char *str, const char *val, static void dbrw_sethdr(VRT_CTX, int where, const char *what, const char *value) { -#if VARNISHVERSION == 3 - VRT_SetHdr(ctx, where, what, value, vrt_magic_string_end); -#else struct gethdr_s s = { where, what }; VRT_SetHdr(ctx, &s, value, vrt_magic_string_end); -#endif } static char * @@ -447,59 +492,17 @@ findmatch(VRT_CTX, struct dbrw_connection *conn, char **param) return res; } -static struct dbrw_connection * -get_connection(struct dbrw_config *conf) -{ - struct dbrw_connection *cp = pthread_getspecific(thread_key); - - if (!cp) { - cp = malloc(sizeof(*cp)); - if (!cp) - return NULL; - cp->state = state_init; - cp->conf = conf; - cp->data = NULL; - cp->matches = NULL; - cp->matchsize = 0; - - if (sql_init(cp)) { - free(cp); - return NULL; - } - if (pthread_setspecific(thread_key, cp)) { - dbrw_error("cannot set thread-specific data"); - sql_destroy(cp); - free(cp); - return NULL; - } - } - return cp; -} - -VCL_STRING -vmod_rewrite(VRT_CTX, struct vmod_priv *priv, VCL_STRING arg) +static char * +do_rewrite(VRT_CTX, struct dbrw_connection *cp, VCL_STRING arg) { - struct dbrw_config *conf = priv->priv; - struct dbrw_connection *cp; struct wordsplit ws, wsenv; int i, rc; - char *res; + char *res; - debug(conf, 2, ("vmod_rewrite(%s) begin", arg)); - if (!conf || !conf->query) { - debug(conf, 2, ("vmod_rewrite: not configured; exiting")); - return NULL; - } - cp = get_connection(conf); - if (!cp) { - debug(conf, 2, ("vmod_rewrite: no private data; exiting")); - return NULL; - } - if (sql_connect(cp) || cp->state != state_connected) return NULL; - debug(conf, 2, ("vmod_rewrite: splitting arg")); + debug(cp->conf, 2, ("vmod_rewrite: splitting arg")); wsenv.ws_delim = ";"; if (wordsplit(arg, &wsenv, WRDSF_NOVAR|WRDSF_NOCMD|WRDSF_DELIM)) { dbrw_error("cannot split string `%s': %s", @@ -507,8 +510,8 @@ vmod_rewrite(VRT_CTX, struct vmod_priv *priv, VCL_STRING arg) return NULL; } - if (conf->backend->sql_escape) { - debug(conf, 2, ("escaping variables")); + if (cp->conf->backend->sql_escape) { + debug(cp->conf, 2, ("escaping variables")); for (i = 0; i < wsenv.ws_wordc; i++) { char *p = sql_escape(cp, wsenv.ws_wordv[i]); if (!p) { @@ -518,19 +521,19 @@ vmod_rewrite(VRT_CTX, struct vmod_priv *priv, VCL_STRING arg) } free(wsenv.ws_wordv[i]); wsenv.ws_wordv[i] = p; - debug(conf, 3, ("%d: %s",i,p)); + debug(cp->conf, 3, ("%d: %s",i,p)); } } - debug(conf, 2, ("expanding query")); + debug(cp->conf, 2, ("expanding query")); ws.ws_env = (const char **)wsenv.ws_wordv; - rc = wordsplit(conf->query, &ws, + rc = wordsplit(cp->conf->query, &ws, WRDSF_NOCMD | WRDSF_QUOTE | WRDSF_NOSPLIT | WRDSF_ENV | WRDSF_UNDEF); if (rc) { dbrw_error("cannot expand query `%s': %s", - conf->query, wordsplit_strerror(&ws)); + cp->conf->query, wordsplit_strerror(&ws)); wordsplit_free(&wsenv); return NULL; } @@ -539,7 +542,7 @@ vmod_rewrite(VRT_CTX, struct vmod_priv *priv, VCL_STRING arg) wordsplit_free(&ws); if (rc) { - debug(conf, 1, ("vmod_rewrite: query failed")); + debug(cp->conf, 1, ("vmod_rewrite: query failed")); wordsplit_free(&wsenv); return NULL; } @@ -549,7 +552,26 @@ vmod_rewrite(VRT_CTX, struct vmod_priv *priv, VCL_STRING arg) sql_free_result(cp); - debug(conf, 1, ("vmod_rewrite: res=%s", res ? res : "(NULL)")); - + return res; +} + +VCL_STRING +vmod_rewrite(VRT_CTX, struct vmod_priv *priv, VCL_STRING arg) +{ + struct dbrw_config *conf = priv->priv; + struct dbrw_connection *cp; + char *res = NULL; + + AN(priv->priv); + debug(conf, 2, ("vmod_rewrite(%s) begin", arg)); + if (!conf || !conf->query) { + debug(conf, 2, ("vmod_rewrite: not configured; exiting")); + return NULL; + } + cp = dbrw_connection_get(conf); + AN(cp); + res = do_rewrite(ctx, cp, arg); + dbrw_connection_release(cp); + debug(conf, 1, ("vmod_rewrite: res=%s", res ? res : "(NULL)")); return res; } diff --git a/src/vmod_dbrw.vcc b/src/vmod_dbrw.vcc index 827ff4f..9a15af1 100644 --- a/src/vmod_dbrw.vcc +++ b/src/vmod_dbrw.vcc @@ -1,5 +1,5 @@ # This file is part of vmod-dbrw -# Copyright (C) 2013-2014 Sergey Poznyakoff +# Copyright (C) 2013-2016 Sergey Poznyakoff # # Vmod-dbrw is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -39,12 +39,12 @@ httpd server, and that they are stored in an SQL database, which makes them easily manageable. $Event dbrw_event -$Function VOID config(PRIV_VCL, STRING, STRING, STRING) +$Function VOID config(PRIV_TASK, STRING, STRING, STRING) Description Configures the module and provides it with the data necessary to connect and use the database. It is normally called - from the **vcl_init** subroutine. + from the **vcl_recv** subroutine. Example :: @@ -57,7 +57,7 @@ Example AND url='$url'"}); -$Function STRING rewrite(PRIV_VCL, STRING) +$Function STRING rewrite(PRIV_TASK, STRING) Description Rewrites its argument using the database configured in the previous |