aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorSergey Poznyakoff <gray@gnu.org.ua>2016-04-21 16:23:35 +0300
committerSergey Poznyakoff <gray@gnu.org.ua>2016-04-21 16:47:22 +0300
commitb5f2337ad45d26941bf533741d8bff1238d59b5d (patch)
tree878a65c333272ceb198ec0d8e871e431f8015f7a /src
parent5ee1409fc878cd65dd224ec5f709507e37e9ed5d (diff)
downloadvmod-dbrw-b5f2337ad45d26941bf533741d8bff1238d59b5d.tar.gz
vmod-dbrw-b5f2337ad45d26941bf533741d8bff1238d59b5d.tar.bz2
Rewrite SQL configuration support.
Multiple configurations can be provided, by calling dbrw.config from branches of a conditional VCL statement. The only requirement is that exactly one dbrw.config is executed within a single HTTP session. This allows the user to select rewrite data from various databases depending on some conditions. Open SQL connections are stored in a pool and reused as needed. So far no limit on the number of connections is imposed. This is a TODO. Connections remain open during the lifetime of the module (i.e. for as long as varnish is not restarted). * src/dbrw.h (dbrw_config): New members: param_str, magic and list. (dbrw_connection): New members: magic, busy, list. * src/vmod_dbrw.c (dbrw_connection_get) (dbrw_connection_release): New functions. (dbrw_event): Install atexit function. (vmod_config): Keep a pool of configurations. (dbrw_sethdr) [VARNISHVERSION]: Remove leftover code. (vmod_rewrite): Rewrite. * src/vmod_dbrw.vcc (config, rewrite): Mark as PRIV_TASK. * doc/vmod-dbrw.3: Update docs. * doc/vmod-dbrw.texi: Update docs. * configure.ac: Raise version number
Diffstat (limited to 'src')
-rw-r--r--src/dbrw.h14
-rw-r--r--src/sql.c1
-rw-r--r--src/vmod_dbrw.c278
-rw-r--r--src/vmod_dbrw.vcc8
4 files changed, 169 insertions, 132 deletions
diff --git a/src/dbrw.h b/src/dbrw.h
index 855720a..2156e0b 100644
--- a/src/dbrw.h
+++ b/src/dbrw.h
@@ -21,6 +21,12 @@
#include <syslog.h>
#include <regex.h>
+#include "vcl.h"
+#include "vrt.h"
+#include "vcc_if.h"
+#include "bin/varnishd/cache/cache.h"
+#define WSPTR(s) ((s)->ws)
+
struct dbrw_connection;
struct dbrw_backend {
@@ -50,21 +56,29 @@ enum {
#define HTTP_STATUS_LEN 3
struct dbrw_config {
+ unsigned magic;
+#define DBRW_CONFIG_MAGIC 0x67636667
int debug_level;
struct dbrw_backend *backend;
char **param;
+ char *param_str;
char *query;
int qdisp;
int regflags;
char status[HTTP_STATUS_LEN+1];
+ VTAILQ_ENTRY(dbrw_config) list;
};
struct dbrw_connection {
+ unsigned magic;
+#define DBRW_CONNECTION_MAGIC 0x6773716c
int state; /* Connection state */
+ int busy:1;
struct dbrw_config *conf; /* Pointer to the configuration data */
regmatch_t *matches; /* Match map */
size_t matchsize; /* Total number of entries in match map */
void *data; /* Backend-specific data */
+ VTAILQ_ENTRY(dbrw_connection) list;
};
void dbrw_debug(const char *fmt, ...);
diff --git a/src/sql.c b/src/sql.c
index b9ba3a4..c42c5de 100644
--- a/src/sql.c
+++ b/src/sql.c
@@ -184,3 +184,4 @@ sql_get_column(struct dbrw_connection *conn, unsigned row, unsigned col)
return NULL;
return conn->conf->backend->sql_get_column(conn, row, col);
}
+
diff --git a/src/vmod_dbrw.c b/src/vmod_dbrw.c
index 5c88199..d212eb7 100644
--- a/src/vmod_dbrw.c
+++ b/src/vmod_dbrw.c
@@ -17,17 +17,8 @@
#include "dbrw.h"
#include <stdarg.h>
#include "wordsplit.h"
-#include "vcl.h"
-#include "vrt.h"
-#include "vcc_if.h"
#include "pthread.h"
-#include "bin/varnishd/cache/cache.h"
-#define WSPTR(s) ((s)->ws)
-
-static pthread_once_t thread_once = PTHREAD_ONCE_INIT;
-static pthread_key_t thread_key;
-
enum {
QDISP_NONE,
QDISP_APPEND,
@@ -51,32 +42,80 @@ dbrw_error(const char *fmt, ...)
vsyslog(LOG_DAEMON|LOG_ERR, fmt, ap);
va_end(ap);
}
+
+static pthread_mutex_t config_pool_mtx = PTHREAD_MUTEX_INITIALIZER;
+static VTAILQ_HEAD(, dbrw_config) config_pool =
+ VTAILQ_HEAD_INITIALIZER(config_pool);
+
+static pthread_mutex_t connect_pool_mtx = PTHREAD_MUTEX_INITIALIZER;
+static VTAILQ_HEAD(, dbrw_connection) connect_pool =
+ VTAILQ_HEAD_INITIALIZER(connect_pool);
+
+static struct dbrw_connection *
+dbrw_connection_get(struct dbrw_config *cfg)
+{
+ struct dbrw_connection *cp;
+
+ AZ(pthread_mutex_lock(&connect_pool_mtx));
+ VTAILQ_FOREACH(cp, &connect_pool, list) {
+ CHECK_OBJ_NOTNULL(cp, DBRW_CONNECTION_MAGIC);
+ if (cp->conf == cfg && !cp->busy)
+ break;
+ }
+ if (!cp) {
+ ALLOC_OBJ(cp, DBRW_CONNECTION_MAGIC);
+ AN(cp);
+ cp->state = state_init;
+ cp->conf = cfg;
+ cp->data = NULL;
+ cp->matches = NULL;
+ cp->matchsize = 0;
+
+ if (sql_init(cp)) {
+ free(cp);
+ cp = NULL;
+ } else
+ VTAILQ_INSERT_HEAD(&connect_pool, cp, list);
+ }
+ cp->busy = 1;
+ pthread_mutex_unlock(&connect_pool_mtx);
+ return cp;
+}
static void
-conn_free(void *f)
+dbrw_connection_release(struct dbrw_connection *cp)
{
- struct dbrw_connection *p = f;
- sql_disconnect(p);
- sql_destroy(p);
+ AZ(pthread_mutex_lock(&connect_pool_mtx));
+ cp->busy = 0;
+ pthread_mutex_unlock(&connect_pool_mtx);
}
static void
-make_key()
+disconnect(void)
{
- pthread_key_create(&thread_key, conn_free);
+ AZ(pthread_mutex_lock(&connect_pool_mtx));
+ while (!VTAILQ_EMPTY(&connect_pool)) {
+ struct dbrw_connection *cp = VTAILQ_FIRST(&connect_pool);
+ VTAILQ_REMOVE(&connect_pool, cp, list);
+ sql_disconnect(cp);
+ sql_destroy(cp);
+ }
+ pthread_mutex_unlock(&connect_pool_mtx);
}
int
dbrw_event(VRT_CTX, struct vmod_priv *priv, enum vcl_event_e e)
{
- if (e == VCL_EVENT_LOAD) {
- struct dbrw_config *conf = calloc(1, sizeof(*conf));
- if (!conf) {
- dbrw_error("not enough memory");
- abort();
- }
- priv->priv = conf;
- pthread_once(&thread_once, make_key);
+ switch (e) {
+ case VCL_EVENT_LOAD:
+ atexit(disconnect);
+ break;
+ case VCL_EVENT_DISCARD:
+ break;
+
+ default:
+ /* ignore */
+ break;
}
return 0;
}
@@ -176,57 +215,67 @@ VCL_VOID
vmod_config(VRT_CTX, struct vmod_priv *priv,
VCL_STRING bkname, VCL_STRING param, VCL_STRING query)
{
- char *s;
- struct wordsplit ws;
- struct dbrw_config *conf = priv->priv;
+ struct dbrw_config *conf;
+ struct dbrw_backend *backend;
- ws.ws_delim = ";";
- if (wordsplit(param, &ws,
- WRDSF_NOVAR | WRDSF_NOCMD | WRDSF_QUOTE |
- WRDSF_CESCAPES |
- WRDSF_DELIM)) {
- dbrw_error("cannot split string `%s': %s",
- param, wordsplit_strerror (&ws));
- return;
- }
- conf->param = ws.ws_wordv;
- ws.ws_wordv = NULL;
- ws.ws_wordc = 0;
- wordsplit_free(&ws);
-
- conf->query = strdup(query);
- if (!conf->query) {
- dbrw_error("not enough memory");
- argv_free(conf->param);
- conf->param = NULL;
- return;
- }
-
- /* Select backend */
- conf->backend = dbrw_backend_select(bkname);
- if (!conf->backend) {
+ AZ(priv->priv);
+ backend = dbrw_backend_select(bkname);
+ if (!backend) {
dbrw_error("unsupported backend: %s", bkname);
- argv_free(conf->param);
- free(conf->query);
- conf->query = NULL;
- conf->param = NULL;
- return;
+ abort();
}
- s = findparam(conf->param, "debug");
- conf->debug_level = s ? atoi(s) : 0;
-
- conf->qdisp = QDISP_NONE;
- conf->regflags = REG_EXTENDED;
- conf->status[0] = 0;
-
- s = findparam(conf->param, "flags");
- if (s && parse_flags(s, &conf->qdisp, &conf->regflags, conf->status)) {
- argv_free(conf->param);
- free(conf->query);
- conf->query = NULL;
- conf->param = NULL;
+ AZ(pthread_mutex_lock(&config_pool_mtx));
+ VTAILQ_FOREACH(conf, &config_pool, list) {
+ CHECK_OBJ_NOTNULL(conf, DBRW_CONFIG_MAGIC);
+ if (conf->backend == backend
+ && strcmp(conf->query, query) == 0
+ && strcmp(conf->param_str, param) == 0)
+ break;
+ }
+ pthread_mutex_unlock(&config_pool_mtx);
+
+ if (!conf) {
+ char *s;
+ struct wordsplit ws;
+ ws.ws_delim = ";";
+ if (wordsplit(param, &ws,
+ WRDSF_NOVAR | WRDSF_NOCMD | WRDSF_QUOTE |
+ WRDSF_CESCAPES |
+ WRDSF_DELIM)) {
+ dbrw_error("cannot split string `%s': %s",
+ param, wordsplit_strerror (&ws));
+ abort();
+ }
+ ALLOC_OBJ(conf, DBRW_CONFIG_MAGIC);
+ AN(conf);
+ conf->param = ws.ws_wordv;
+ ws.ws_wordv = NULL;
+ ws.ws_wordc = 0;
+ wordsplit_free(&ws);
+ conf->param_str = strdup(param);
+ AN(conf->param_str);
+ conf->query = strdup(query);
+ AN(conf->query);
+ conf->backend = backend;
+
+ s = findparam(conf->param, "debug");
+ conf->debug_level = s ? atoi(s) : 0;
+
+ conf->qdisp = QDISP_NONE;
+ conf->regflags = REG_EXTENDED;
+ conf->status[0] = 0;
+
+ s = findparam(conf->param, "flags");
+ if (s)
+ parse_flags(s, &conf->qdisp, &conf->regflags,
+ conf->status);
+
+ AZ(pthread_mutex_lock(&config_pool_mtx));
+ VTAILQ_INSERT_HEAD(&config_pool, conf, list);
+ pthread_mutex_unlock(&config_pool_mtx);
}
+ priv->priv = conf;
}
static char *
@@ -308,12 +357,8 @@ expand_backref(VRT_CTX, const char *str, const char *val,
static void
dbrw_sethdr(VRT_CTX, int where, const char *what, const char *value)
{
-#if VARNISHVERSION == 3
- VRT_SetHdr(ctx, where, what, value, vrt_magic_string_end);
-#else
struct gethdr_s s = { where, what };
VRT_SetHdr(ctx, &s, value, vrt_magic_string_end);
-#endif
}
static char *
@@ -447,59 +492,17 @@ findmatch(VRT_CTX, struct dbrw_connection *conn, char **param)
return res;
}
-static struct dbrw_connection *
-get_connection(struct dbrw_config *conf)
-{
- struct dbrw_connection *cp = pthread_getspecific(thread_key);
-
- if (!cp) {
- cp = malloc(sizeof(*cp));
- if (!cp)
- return NULL;
- cp->state = state_init;
- cp->conf = conf;
- cp->data = NULL;
- cp->matches = NULL;
- cp->matchsize = 0;
-
- if (sql_init(cp)) {
- free(cp);
- return NULL;
- }
- if (pthread_setspecific(thread_key, cp)) {
- dbrw_error("cannot set thread-specific data");
- sql_destroy(cp);
- free(cp);
- return NULL;
- }
- }
- return cp;
-}
-
-VCL_STRING
-vmod_rewrite(VRT_CTX, struct vmod_priv *priv, VCL_STRING arg)
+static char *
+do_rewrite(VRT_CTX, struct dbrw_connection *cp, VCL_STRING arg)
{
- struct dbrw_config *conf = priv->priv;
- struct dbrw_connection *cp;
struct wordsplit ws, wsenv;
int i, rc;
- char *res;
+ char *res;
- debug(conf, 2, ("vmod_rewrite(%s) begin", arg));
- if (!conf || !conf->query) {
- debug(conf, 2, ("vmod_rewrite: not configured; exiting"));
- return NULL;
- }
- cp = get_connection(conf);
- if (!cp) {
- debug(conf, 2, ("vmod_rewrite: no private data; exiting"));
- return NULL;
- }
-
if (sql_connect(cp) || cp->state != state_connected)
return NULL;
- debug(conf, 2, ("vmod_rewrite: splitting arg"));
+ debug(cp->conf, 2, ("vmod_rewrite: splitting arg"));
wsenv.ws_delim = ";";
if (wordsplit(arg, &wsenv, WRDSF_NOVAR|WRDSF_NOCMD|WRDSF_DELIM)) {
dbrw_error("cannot split string `%s': %s",
@@ -507,8 +510,8 @@ vmod_rewrite(VRT_CTX, struct vmod_priv *priv, VCL_STRING arg)
return NULL;
}
- if (conf->backend->sql_escape) {
- debug(conf, 2, ("escaping variables"));
+ if (cp->conf->backend->sql_escape) {
+ debug(cp->conf, 2, ("escaping variables"));
for (i = 0; i < wsenv.ws_wordc; i++) {
char *p = sql_escape(cp, wsenv.ws_wordv[i]);
if (!p) {
@@ -518,19 +521,19 @@ vmod_rewrite(VRT_CTX, struct vmod_priv *priv, VCL_STRING arg)
}
free(wsenv.ws_wordv[i]);
wsenv.ws_wordv[i] = p;
- debug(conf, 3, ("%d: %s",i,p));
+ debug(cp->conf, 3, ("%d: %s",i,p));
}
}
- debug(conf, 2, ("expanding query"));
+ debug(cp->conf, 2, ("expanding query"));
ws.ws_env = (const char **)wsenv.ws_wordv;
- rc = wordsplit(conf->query, &ws,
+ rc = wordsplit(cp->conf->query, &ws,
WRDSF_NOCMD | WRDSF_QUOTE |
WRDSF_NOSPLIT |
WRDSF_ENV | WRDSF_UNDEF);
if (rc) {
dbrw_error("cannot expand query `%s': %s",
- conf->query, wordsplit_strerror(&ws));
+ cp->conf->query, wordsplit_strerror(&ws));
wordsplit_free(&wsenv);
return NULL;
}
@@ -539,7 +542,7 @@ vmod_rewrite(VRT_CTX, struct vmod_priv *priv, VCL_STRING arg)
wordsplit_free(&ws);
if (rc) {
- debug(conf, 1, ("vmod_rewrite: query failed"));
+ debug(cp->conf, 1, ("vmod_rewrite: query failed"));
wordsplit_free(&wsenv);
return NULL;
}
@@ -549,7 +552,26 @@ vmod_rewrite(VRT_CTX, struct vmod_priv *priv, VCL_STRING arg)
sql_free_result(cp);
- debug(conf, 1, ("vmod_rewrite: res=%s", res ? res : "(NULL)"));
-
+ return res;
+}
+
+VCL_STRING
+vmod_rewrite(VRT_CTX, struct vmod_priv *priv, VCL_STRING arg)
+{
+ struct dbrw_config *conf = priv->priv;
+ struct dbrw_connection *cp;
+ char *res = NULL;
+
+ AN(priv->priv);
+ debug(conf, 2, ("vmod_rewrite(%s) begin", arg));
+ if (!conf || !conf->query) {
+ debug(conf, 2, ("vmod_rewrite: not configured; exiting"));
+ return NULL;
+ }
+ cp = dbrw_connection_get(conf);
+ AN(cp);
+ res = do_rewrite(ctx, cp, arg);
+ dbrw_connection_release(cp);
+ debug(conf, 1, ("vmod_rewrite: res=%s", res ? res : "(NULL)"));
return res;
}
diff --git a/src/vmod_dbrw.vcc b/src/vmod_dbrw.vcc
index 827ff4f..9a15af1 100644
--- a/src/vmod_dbrw.vcc
+++ b/src/vmod_dbrw.vcc
@@ -1,5 +1,5 @@
# This file is part of vmod-dbrw
-# Copyright (C) 2013-2014 Sergey Poznyakoff
+# Copyright (C) 2013-2016 Sergey Poznyakoff
#
# Vmod-dbrw is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -39,12 +39,12 @@ httpd server, and that they are stored in an SQL database, which makes
them easily manageable.
$Event dbrw_event
-$Function VOID config(PRIV_VCL, STRING, STRING, STRING)
+$Function VOID config(PRIV_TASK, STRING, STRING, STRING)
Description
Configures the module and provides it with the data
necessary to connect and use the database. It is normally called
- from the **vcl_init** subroutine.
+ from the **vcl_recv** subroutine.
Example
::
@@ -57,7 +57,7 @@ Example
AND url='$url'"});
-$Function STRING rewrite(PRIV_VCL, STRING)
+$Function STRING rewrite(PRIV_TASK, STRING)
Description
Rewrites its argument using the database configured in the previous

Return to:

Send suggestions and report system problems to the System administrator.