Implement built-in and external preprocessors

git-svn-id: file:///svnroot/mailfromd/trunk@1456 7a8a7f39-df28-0410-adc6-e0d955640f24

9 files changed, 859 insertions, 400 deletions

diff --git a/ChangeLog b/ChangeLog
index 4f12af3e..ea29d1ef 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,16 @@
 2007-05-20  Sergey Poznyakoff  <gray@gnu.org.ua>
 
+	Implement built-in and external preprocessors.
+	
+	* src/pp.c: New file.  Built-in preprocessor
+	* src/lex.l, src/gram.y, src/mailfromd.h, src/main.c: Use
+	preprocessor feature.
+	* src/Makefile.am (mailfromd_SOURCES): Add pp.c
+	(AM_CPPFLAGS): Define DEFAULT_PREPROCESSOR
+	* tests/atlocal.in (MFOPTS): Disable external preprocessor
+	* configure.ac: New option --with-preprocessor and variable
+	DEFAULT_PREPROCESSOR
+	
 	* src/engine.c (listens_on): port argument was ignored
 
 2007-05-17  Sergey Poznyakoff  <gray@gnu.org.ua>
diff --git a/configure.ac b/configure.ac
index cdfa2bc5..47165acb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -310,6 +310,36 @@ if test "$status_dbm" = "no"; then
    AC_MSG_ERROR([Cannot find DBM library to link with.])
 fi
 
+AC_ARG_WITH([preprocessor],
+            AC_HELP_STRING([--without-preprocessor],
+                           [do not use external preprocessor]),
+            [
+case "${withval}" in
+yes) use_ext_pp=yes ;;
+no)  use_ext_pp=no ;;
+*)   AC_MSG_ERROR(bad value ${withval} for --with-preprocessor) ;;
+esac],[use_ext_pp=yes])
+
+AC_ARG_VAR([DEFAULT_PREPROCESSOR],
+           [Set default preprocessor name])
+if test -z "$DEFAULT_PREPROCESSOR" ; then
+  DEFAULT_PREPROCESSOR="m4 -s"
+fi
+if test $use_ext_pp != no; then
+  save_PATH=$PATH
+  PREPROC_OPTIONS=`echo $DEFAULT_PREPROCESSOR | sed -n 's/[[^ ]][[^ ]]* //p'`
+  case "$DEFAULT_PREPROCESSOR" in
+  /*) PATH=`expr $DEFAULT_PREPROCESSOR : '\(.*\)/.*'`:$PATH
+      DEFAULT_PREPROCESSOR=`expr $DEFAULT_PREPROCESSOR : '.*/\(.*\)'`;;
+  esac
+  AC_PATH_PROG(PPBIN, $DEFAULT_PREPROCESSOR)
+  DEFAULT_PREPROCESSOR=$PPBIN
+  if test -n "$DEFAULT_PREPROCESSOR"; then
+    DEFAULT_PREPROCESSOR="$DEFAULT_PREPROCESSOR $PREPROC_OPTIONS"
+  fi
+  PATH=$save_PATH
+fi
+
 AC_ARG_WITH([readline],
             AC_HELP_STRING([--without-readline],
                            [do not use readline]),
@@ -510,6 +540,7 @@ cat <<EOF
 *******************************************************************
 Mailfromd configured with the following settings:
 
+External preprocessor..................... $ext_pp
 DBM version............................... $status_dbm
 Default user.............................. $user
 State directory........................... $stat_dir
@@ -524,7 +555,12 @@ Documentation rendition type.............. $rendition
 
 EOF
 ],
-[status_dbm="$status_dbm"
+[if test -z "$DEFAULT_PREPROCESSOR"; then
+  ext_pp=no
+else
+  ext_pp="$DEFAULT_PREPROCESSOR"
+fi
+status_dbm="$status_dbm"
 stat_dir='$DEFAULT_STATE_DIR'
 socket='$DEFAULT_SOCKET'
 user=$DEFAULT_USER
diff --git a/src/Makefile.am b/src/Makefile.am
index ccc2f426..46d68577 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -51,6 +51,7 @@ mailfromd_SOURCES = \
  mu_dbm.c\
  optab.c\
  optab.h\
+ pp.c\
  prog.c\
  prog.h\
  spf.c\
@@ -115,6 +116,7 @@ AM_CPPFLAGS=-DSYSCONFDIR=\"$(sysconfdir)\"\
  -DDEFAULT_SOCKET=\"$(DEFAULT_SOCKET)\"\
  -DDEFAULT_VERSION_INCLUDE_DIR=\"$(incdir)\"\
  -DDEFAULT_INCLUDE_DIR=\"$(pkgdatadir)/include\"\
+ -DDEFAULT_PREPROCESSOR="\"$(DEFAULT_PREPROCESSOR)\""\
  -DLOCALEDIR=\"$(localedir)\"
 
 INCLUDES = $(MAILUTILS_INCLUDES) $(MU_COMMON_INCLUDES) -I$(top_srcdir)/lib -I../lib $(MILTER_INCLUDES)
diff --git a/src/gram.y b/src/gram.y
index 11a38be8..8a02e038 100644
--- a/src/gram.y
+++ b/src/gram.y
@@ -1902,11 +1902,11 @@ yyerror(char *s)
 }
 
 int
-parse_program(char *name, int ydebug, int ldebug)
+parse_program(char *name, int ydebug)
 {
 	int rc;
 	yydebug = ydebug;
-	if (source(name, ldebug))
+	if (source(name))
 		return -1;
 	outer_context = inner_context = context_none;
 	catch_nesting = 0;
diff --git a/src/lex.l b/src/lex.l
index 98691617..c6a9ea1d 100644
--- a/src/lex.l
+++ b/src/lex.l
@@ -31,14 +31,7 @@
 #include "gram.h"
 #include "prog.h"
 	
-struct input_file_ident {
-	ino_t i_node;
-	dev_t device;
-};
- 
 static struct locus locus; /* Current input location */
-static struct input_file_ident source_id; /* Identification of the input
-					     file */
  
 static struct obstack string_stk; /* Obstack for constructing string values */
 static char *multiline_delimiter; /* End of here-document delimiter */ 
@@ -85,349 +78,14 @@ keyword(int kw)
 
 /* Input stack support */
 
-#define xinput() (yyin ? getc(yyin) : EOF)
-
 #undef YY_INPUT
-#define YY_INPUT(buf,result,max_size)  do {                             \
-        int i;                                                          \
-        for (i = 0; i < max_size; i++) {                                \
-                int ch = xinput();                                      \
-                if (ch == EOF)                                          \
-                        break;                                          \
-                buf[i] = ch;                                            \
-        }                                                               \
-        result = i;                                                     \
-} while (0)
-
-#define LEX_BUFFER_STATE YY_BUFFER_STATE
-
-#define SET_BUFFER_STATE(s) do {                                        \
-        (s) = YY_CURRENT_BUFFER;                                        \
-        yy_switch_to_buffer(yy_create_buffer(yyin, YY_BUF_SIZE));       \
+#define YY_INPUT(buf,result,max_size) do {                  \
+	if (ext_pp)                                         \
+		result = fread(buf, 1, max_size, yyin);     \
+	else                                                \
+		result = pp_fill_buffer(buf, max_size);     \
 } while (0)
 
-#define RESTORE_BUFFER_STATE(s) do {                                    \
-        yy_delete_buffer(YY_CURRENT_BUFFER);                            \
-        yy_switch_to_buffer(s);                                         \
-} while (0)
-
-struct buffer_ctx {
-	struct buffer_ctx *prev;
-	struct locus locus;
-	struct input_file_ident id;
-	FILE *yyin;
-	LEX_BUFFER_STATE state;
-};
-	
-static struct buffer_ctx *context_stack;
-
-#define STAT_ID_EQ(st,id) ((id).i_node == (st).st_ino       \
-			   && (id).device == (st).st_dev)
-
-struct buffer_ctx *
-ctx_lookup(struct stat *st)
-{
-	struct buffer_ctx *ctx;
-
-	for (ctx = context_stack; ctx; ctx = ctx->prev)
-		if (STAT_ID_EQ(*st, ctx->id))
-			break;
-	return ctx;
-}
-
-static mu_list_t include_path;
-static mu_list_t std_include_path;
-
-struct file_data {
-	const char *name;
-	size_t namelen;
-	char *buf;
-	size_t buflen;
-	int found;
-};
-
-static int
-find_include_file(void *item, void *data)
-{
-	char *dir = item;
-	struct file_data *dptr = data;
-	size_t size = strlen(dir) + 1 + dptr->namelen + 1;
-	if (size > dptr->buflen) {
-		dptr->buflen = size;
-		dptr->buf = xrealloc(dptr->buf, dptr->buflen);
-	}
-	strcpy(dptr->buf, dir);
-	strcat(dptr->buf, "/");
-	strcat(dptr->buf, dptr->name);
-	return dptr->found = access(dptr->buf, F_OK) == 0;
-}
-
-void
-lex_setup()
-{
-	mu_list_create(&include_path);
-	mu_list_create(&std_include_path);
-	mu_list_append(std_include_path, DEFAULT_VERSION_INCLUDE_DIR);
-	mu_list_append(std_include_path, DEFAULT_INCLUDE_DIR);
-	mu_list_append(std_include_path, "/usr/share/mailfromd/include");
-	mu_list_append(std_include_path, "/usr/local/share/mailfromd/include");
-}
-
-static mu_list_t /* of struct input_file_ident */ incl_sources;
-
-static int
-input_file_ident_cmp(const void *item, const void *data)
-{
-	const struct input_file_ident *id = item;
-	const struct stat *st = data;
-	return !STAT_ID_EQ(*st, *id);
-}
-
-static void
-input_file_ident_free(void *item)
-{
-	free(item);
-}
-
-static int
-source_lookup(struct stat *st)
-{
-	int rc;
-	if (!incl_sources) {
-		mu_list_create(&incl_sources);
-		mu_list_set_comparator(incl_sources, input_file_ident_cmp);
-		mu_list_set_destroy_item(incl_sources, input_file_ident_free);
-	}
-	rc = mu_list_locate (incl_sources, st, NULL);
-	if (rc == MU_ERR_NOENT) {
-		struct input_file_ident *id = xmalloc(sizeof *id);
-		id->i_node = st->st_ino;
-		id->device = st->st_dev;
-		mu_list_append(incl_sources, id);
-	}
-	return rc == 0;
-}
-
-int
-push_source(const char *name, int once)
-{
-	FILE *fp;
-	struct buffer_ctx *ctx;
-	struct stat st;
-	struct literal *lit;
-	
-	if (stat(name, &st)) {
-		parse_error(_("Cannot stat `%s': %s"),
-			    name, mu_strerror(errno));
-		return 1;
-	}
-
-	if (locus.file && STAT_ID_EQ(st, source_id)) {
-		parse_error(_("Recursive inclusion"));
-		return 1;
-	}
-
-	if ((ctx = ctx_lookup(&st))) {
-		parse_error(_("Recursive inclusion"));
-		if (ctx->prev)
-			parse_error_locus(&ctx->prev->locus,
-					  _("`%s' already included here"),
-					  name);
-		else
-			parse_error(_("`%s' already included at top level"),
-				    name);
-		return 1;
-	}
-
-	if (once && source_lookup(&st))
-		return 0;
-	
-	fp = fopen(name, "r");
-	if (!fp) {
-		parse_error(_("Cannot open `%s': %s"),
-			    name, mu_strerror(errno));
-		return 1;
-	}
-
-	/* Push current context */
-	if (locus.file) {
-		ctx = xmalloc (sizeof (*ctx));
-		ctx->locus = locus;
-		ctx->id = source_id;
-		ctx->yyin = yyin;
-		ctx->prev = context_stack;
-		context_stack = ctx;
-      
-		/* Switch to the new context */
-		yyin = fp;
-		SET_BUFFER_STATE (ctx->state);
-	} else {
-		yyrestart (fp);
-	}
-	lit = string_alloc(name, strlen(name));
-	lit->flags |= VAR_REFERENCED;
-	locus.file = lit->text;
-	locus.line = 1;
-	source_id.i_node = st.st_ino;
-	source_id.device = st.st_dev;
-
-	if (yy_flex_debug)
-		fprintf(stderr, "Processing file `%s'\n", name);
-
-	return 0;
-}
-
-int
-pop_source()
-{
-	struct buffer_ctx *ctx;
-
-	if (yyin)
-		fclose(yyin);
-	if (!context_stack) {
-		yyin = NULL;
-		locus.file = NULL;
-		return 1;
-	}
-	/* Restore previous context */
-	locus = context_stack->locus;
-	if (yy_flex_debug) 
-		fprintf(stderr, "Resuming file `%s' at line %lu\n",
-			locus.file, (unsigned long) locus.line);
-	source_id = context_stack->id;
-	RESTORE_BUFFER_STATE(context_stack->state);
-	ctx = context_stack->prev;
-	free(context_stack);
-	context_stack = ctx;
-
-	return 0;
-}
-
-int
-try_file(const char *name, int allow_cwd, char **newp)
-{
-	static char *cwd = ".";
-	struct file_data fd;
-			
-	fd.name = name;
-	fd.namelen = strlen(name);
-	fd.buf = NULL;
-	fd.buflen = 0;
-	fd.found = 0;
-	
-	if (allow_cwd) {
-		mu_list_prepend(include_path, cwd);
-		mu_list_do(include_path, find_include_file, &fd);
-		mu_list_remove(std_include_path, cwd);
-	} else
-		mu_list_do(include_path, find_include_file, &fd);
-			
-	if (!fd.found) {
-		mu_list_do(std_include_path, find_include_file, &fd);
-					
-		if (!fd.found) {
-			parse_error(_("%s: No such file or directory"), name);
-			*newp = NULL;
-		} 
-	}
-	if (fd.found)
-		*newp = fd.buf;
-	return fd.found;
-}
-
-void
-parse_include(int once)
-{
-	int argc;
-	char **argv;
-	char *tmp = NULL;
-	char *p = NULL;
-	
-	if (mu_argcv_get(yytext, "", NULL, &argc, &argv)) 
-		parse_error(_("Cannot parse include line"));
-	else if (argc != 2) 
-		parse_error(_("Invalid include statement"));
-	else {
-		size_t len;
-		int allow_cwd;
-
-		p = argv[1];
-		len = strlen(p);
-		
-		if (p[0] == '<' && p[len - 1] == '>') {
-			allow_cwd = 0;
-			p[len - 1] = 0;
-			p++;
-		} else 
-			allow_cwd = 1;
-
-		if (p[0] != '/' && try_file(p, allow_cwd, &tmp))
-			p = tmp;
-	}
-
-	locus.line++;
-	if (p)
-		push_source(p, once);
-	free(tmp);
-	mu_argcv_free(argc, argv);
-}
-
-#define DEFAULT_SUFFIX ".mf"
-
-void
-parse_require()
-{
-	int argc;
-	char **argv;
-	char *tmp = NULL;
-	char *p = NULL;
-	
-	if (mu_argcv_get(yytext, "", NULL, &argc, &argv)) 
-		parse_error(_("Cannot parse require line"));
-	else if (argc != 2) 
-		parse_error(_("Invalid require statement"));
-	else {
-		size_t len;
-		char *fname;
-		
-		p = argv[1];
-		len = strlen(p);
-
-		/* For compatibility with #include */
-		if (p[0] == '<' && p[len - 1] == '>') {
-			p[--len] = 0;
-			p++;
-		} 
-
-		if (len < sizeof DEFAULT_SUFFIX
-		    || memcmp(p + len - sizeof DEFAULT_SUFFIX - 1,
-			      DEFAULT_SUFFIX,
-			      sizeof DEFAULT_SUFFIX - 1)) {
-			fname = xmalloc(len + sizeof DEFAULT_SUFFIX);
-			strcpy(fname, p);
-			strcat(fname, DEFAULT_SUFFIX);
-			p = fname;
-		} else
-			fname = NULL;
-			
-		if (p[0] != '/') {
-			if (try_file(p, 0, &tmp))
-				p = tmp;
-			else
-				p = NULL;
-			free(fname);
-		} else if (fname) {
-			tmp = fname;
-			fname = NULL;
-		}
-	}
-	locus.line++;
-	if (p)
-		push_source(p, 1);
-	free(tmp);
-	mu_argcv_free(argc, argv);
-}
-
 static int
 get_const(const char *name)
 {
@@ -448,7 +106,25 @@ get_const(const char *name)
 	}
 	return 0;
 }
-		
+
+size_t input_line;
+
+static void
+advance_line()
+{
+	++locus.line;
+	++input_line;
+}
+
+static int
+is_stdin(const char *name)
+{
+	return name == NULL
+		|| strcmp(name, "stdin") == 0
+		|| strcmp(name, "<stdin>") == 0
+		|| strcmp(name, "-") == 0;
+}
+
 %}
 
 /* Exclusive states:
@@ -487,16 +163,24 @@ MACRO {LOCUS}|{VMACRO}|__statedir__
 "/*"         BEGIN(COMMENT);
 <COMMENT>[^*\n]*        /* eat anything that's not a '*' */
 <COMMENT>"*"+[^*/\n]*   /* eat up '*'s not followed by '/'s */
-<COMMENT>\n             ++locus.line;
+<COMMENT>\n             advance_line();
 <COMMENT>"*"+"/"        BEGIN(INITIAL);
          /* Configuration directives */
+^[ \t]*#[ \t]*{P}[ \t]+".*".*\n parse_line_cpp(yytext, &locus);
+^[ \t]*#[ \t]*line[ \t].*\n {
+	struct locus newloc;
+	memset(&newloc, 0, sizeof newloc);
+	parse_line(yytext, &newloc);
+	if (is_stdin(newloc.file))
+		locus.line += newloc.line - input_line;
+	else
+		locus = newloc;
+	input_line++;
+}
 ^[ \t]*#[ \t]*pragma[ \t].*/\n parse_pragma(yytext);
-^[ \t]*#[ \t]*include_once[ \t].*\n parse_include(1);
-^[ \t]*#[ \t]*include[ \t].*\n parse_include(0);
-^[ \t]*#[ \t]*require[ \t].*\n parse_require(yytext);
-^[ \t]*#[ \t]*error[ \t].*\n  { parse_error("%s", yytext); locus.line++; }
+^[ \t]*#[ \t]*error[ \t].*\n  { parse_error("%s", yytext); advance_line(); }
          /* End-of-line comments */
-#.*\n   { locus.line++; }
+#.*\n   { advance_line(); }
 #.*     /* end-of-file comment */;
          /* Reserved words */
 accept    return keyword(ACT_ACCEPT);
@@ -642,7 +326,7 @@ end       return keyword(KW_END);
 		
 		while ((c = input()) && isascii(c) && isspace(c))
 			if (c == '\n')
-				++locus.line;
+				advance_line();
 	
 		paren_follows = (c == '(');
 		unput(c);
@@ -663,7 +347,7 @@ end       return keyword(KW_END);
   }
 '[^\n']*'             { string(yytext+1, yyleng-2); return STRING; }
 \"[^\\\"$%\n]*\"      { string(yytext+1, yyleng-2); return STRING; }
-\"[^\\\"$%\n]*\\\n    { ++locus.line;
+\"[^\\\"$%\n]*\\\n    { advance_line();
                         BEGIN(STR);
                         line_begin();
                         line_add(yytext + 1, yyleng - 3); }     
@@ -703,7 +387,7 @@ end       return keyword(KW_END);
          BEGIN(STR);
          line_add_char(mu_argcv_unquote_char(yytext[2]));
 }
-<STR>[^\\\"$%\n]*\\\n { ++locus.line; line_add(yytext, yyleng - 2); }
+<STR>[^\\\"$%\n]*\\\n { advance_line(); line_add(yytext, yyleng - 2); }
 <STR>[^\\\"$%\n]*\"   { BEGIN(INITIAL);
                         if (yyleng > 1) 
                           line_add(yytext, yyleng - 1); 
@@ -748,7 +432,7 @@ end       return keyword(KW_END);
 "<<"(-" "?)?'{IDENT}'[ \t]*.*\n {
         char *p;
 
-        ++locus.line;
+        advance_line();
         char_to_strip = NULL;
         multiline_unescape = 1;
 
@@ -789,7 +473,7 @@ end       return keyword(KW_END);
 <QML>[^\\\n]*\n {
         char *p;
 
-        ++locus.line;
+        advance_line();
         p = yytext;
         if (char_to_strip)
                for (; char_to_strip (*p); p++)
@@ -836,7 +520,7 @@ end       return keyword(KW_END);
 <ML>[^\\$%\n]*\n {
         char *p;
 
-        ++locus.line;
+        advance_line();
         p = yytext;
         if (char_to_strip)
                for (; char_to_strip (*p); p++)
@@ -871,7 +555,7 @@ end       return keyword(KW_END);
 }
          /* Other tokens */
 {WS}     ;
-\n { locus.line++; }
+\n { advance_line(); }
 "="|"==" return keyword(EQ);
 "!=" return keyword(NE);
 "<"  return keyword(LT);
@@ -1020,35 +704,54 @@ parse_error_locus(const struct locus *loc, char *fmt, ...)
 }
 
 int
-source(char *name, int ldebug)
+source(char *name)
 {
-	yy_flex_debug = ldebug;
-	return push_source(name, 0);
+	if (ext_pp) {
+		int fd;
+		char *ppcmd;
+		int argc;
+		char **argv;
+		
+		fd = open(name, O_RDONLY);
+		if (fd == -1) {
+			mu_error(_("Cannot open `%s': %s"),
+				 name, mu_strerror(errno));
+			return EX_NOINPUT;
+		}
+		close(fd);
+
+		pp_make_argcv(&argc, &argv);
+		if (mu_argcv_string(argc, argv, &ppcmd)) {
+			mu_error(_("Cannot build preprocessor command line: %s"),
+				 mu_strerror(errno));
+			return EX_OSERR;
+		}
+
+		if (yy_flex_debug)
+			__debug1("Running preprocessor: `%s'", ppcmd);
+		
+		yyin = popen(ppcmd, "r");
+		if (!yyin) {
+			mu_error(_("Unable to start external preprocessor `%s': %s"),
+				 ppcmd,
+				 mu_strerror(errno));
+			return EX_OSFILE;
+		}
+	} else 
+		return pp_init(name) ? EX_NOINPUT : EX_OK;
+
+	return EX_OK;
 }
 
 int
 yywrap()
 {
-	return pop_source();
-}
-
-void
-add_include_dir(char *dir)
-{
-	int rc;
-	
-	if (!include_path) {
-		rc = mu_list_create(&include_path);
-		if (rc) {
-			mu_error(_("Cannot create include path list: %s"),
-				 mu_strerror(rc));
-			return;
-		}
-	}
-	rc = mu_list_append(include_path, dir);
-	if (rc) 
-		mu_error(_("Cannot append to the include path list: %s"),
-			 mu_strerror(rc));
+	if (yyin)
+		pclose(yyin);
+	else
+		pp_done();
+	locus.file = NULL;
+	return 1;
 }
 
 static int
diff --git a/src/mailfromd.h b/src/mailfromd.h
index 39bef0d2..b041e33d 100644
--- a/src/mailfromd.h
+++ b/src/mailfromd.h
@@ -208,6 +208,7 @@ void disable_prog_trace(const char *modlist);
 #define MAILFROMD_SHOW_DEFAULTS 6
 
 extern char *script_file;
+extern char *ext_pp;
 extern int mode;
 extern int mtasim_option;
 extern unsigned optimization_level;
@@ -238,6 +239,7 @@ extern size_t lock_retry_count_option;
 extern time_t lock_retry_timeout_option;
 extern char *mailfromd_state_dir;
 extern int stack_trace_option;
+extern int log_to_stderr;
 
 
 /* Configuration file parser */
@@ -650,14 +652,22 @@ struct rt_regex {
 void register_regex(struct sym_regex *rp);
 void finalize_regex(void);
 
+/* Preprocessor functions */
+void include_path_setup(void);
+void parse_line_cpp(char *text, struct locus *ploc);
+int parse_line(char *text, struct locus *ploc);
+size_t pp_fill_buffer(char *buf, size_t size);
+int preprocess_input(char *);
+int source(char *);
+void pp_make_argcv(int *pargc, char ***pargv);
+
 /* Parser functions */
 int yyparse();
 int yylex();
 int yyerror(char *s);
-int source(char *, int);
 void add_include_dir(char *dir);
 void onblock(int enable);
-int parse_program(char *name, int ydebug, int ldebug);
+int parse_program(char *name, int ydebug);
 void parse_pragma(char *text);
 const struct locus *get_locus(void);
 const char *header_command_str(enum header_opcode opcode);
@@ -864,4 +874,3 @@ void mailfromd_version(const char *progname, FILE *stream,
 		       struct argp_state *state);
 
 
-
diff --git a/src/main.c b/src/main.c
index ce8cf993..aa0c69a8 100644
--- a/src/main.c
+++ b/src/main.c
@@ -51,12 +51,21 @@ int need_config = 1;/* Set if the current mode requires reading the
 		       configuration file */
 char *script_file = DEFAULT_SCRIPT_FILE;
 int script_check;      /* Check config file syntax and exit */
-int script_ldebug;     /* Enable tracing the lexical analyzer */
+extern int yy_flex_debug;   /* Enable tracing the lexical analyzer */
 int script_ydebug;     /* Enable tracing the parser */
 int script_dump_tree;  /* Dump created config tree to stdout */
 int script_dump_code;  /* Dump disassembled code to stdout */
 int script_dump_macros; /* Dump used Sendmail macros */
 int script_dump_xref;  /* Dump cross-reference */
+int preprocess_option; /* Only preprocess the sources */
+
+#ifdef DEFAULT_PREPROCESSOR
+# define DEF_EXT_PP DEFAULT_PREPROCESSOR
+#else
+# define DEF_EXT_PP NULL
+#endif
+char *ext_pp = DEF_EXT_PP; /* External preprocessor to use */
+
 int do_transcript;     /* Enable session transript */
 int do_trace;          /* Enable tracing configuration */
 int mtasim_option;     /* mtasim compatibility mode */
@@ -817,9 +826,11 @@ enum mailfromd_option {
 	OPTION_LOCK_RETRY_TIMEOUT,
 	OPTION_MILTER_TIMEOUT,
 	OPTION_MTASIM,
+	OPTION_NO_PREPROCESSOR,
 	OPTION_PIDFILE,
 	OPTION_POSTMASTER_EMAIL,
 	OPTION_PREDICT_NEXT,
+	OPTION_PREPROCESSOR,
 	OPTION_SHOW_DEFAULTS,
 	OPTION_SINGLE_PROCESS,
 	OPTION_STACK_TRACE,
@@ -854,6 +865,8 @@ static struct argp_option options[] = {
 	{ "daemon", OPTION_DAEMON, NULL, 0,
 	  N_("Run in daemon mode (default)"), GRP+1 },
 
+	{ NULL, 'E', NULL, 0,
+	  N_("Preprocess source files and exit"), GRP+1 },
 	/* Reserved for future use: */
 	{ "compile", 'c', NULL, OPTION_HIDDEN,
 	  N_("Compile files"), GRP+1 },
@@ -926,6 +939,10 @@ static struct argp_option options[] = {
 	  N_("Assign VALUE to VAR"), GRP+1 },
 	{ "relayed-domain-file", OPTION_DOMAIN_FILE, N_("FILE"), 0,
 	  N_("Read relayed domains from FILE"), GRP+1 },
+	{ "preprocessor", OPTION_PREPROCESSOR, N_("COMMAND"), 0,
+	  N_("Use command as external preprocessor"), GRP+1 },
+	{ "no-preprocessor", OPTION_NO_PREPROCESSOR, NULL, 0,
+	  N_("Disable the use of external preprocessor"), GRP+1 },
 #undef GRP
 
 #undef GRP
@@ -1020,6 +1037,10 @@ parse_opt (int key, char *arg, struct argp_state *state)
 		set_option("debug", arg, 1);
 		break;
 
+	case 'E':
+		preprocess_option = 1;
+		break;
+		
 	case 'e':
 		set_option("expire-interval", arg, 1);
 		break;
@@ -1068,6 +1089,14 @@ parse_opt (int key, char *arg, struct argp_state *state)
 		mode = MAILFROMD_LIST;
 		break;
 
+	case OPTION_PREPROCESSOR:
+		ext_pp = arg;
+		break;
+
+	case OPTION_NO_PREPROCESSOR:
+		ext_pp = NULL;
+		break;
+		
 	case OPTION_TIME_FORMAT:
 		time_format_string = arg;
 		break;
@@ -1187,7 +1216,7 @@ parse_opt (int key, char *arg, struct argp_state *state)
 		break;
 		
 	case OPTION_DUMP_LEX_TRACE:
-		script_ldebug = 1;
+		yy_flex_debug = 1;
 		break;
 		
 	case OPTION_DUMP_MACROS:
@@ -1658,6 +1687,7 @@ mailfromd_show_defaults()
 {
 	printf("version:             %s\n", VERSION);
 	printf("script file:         %s\n", script_file);
+	printf("preprocessor:        %s\n", ext_pp ? ext_pp : "none");
 	printf("user:                %s\n", user);
 	printf("statedir:            %s\n", mailfromd_state_dir);
 	printf("socket:              %s\n", portspec);
@@ -1716,7 +1746,7 @@ int
 main(int argc, char **argv)
 {
 	int index;
-	
+
 #ifdef ENABLE_NLS
 	mu_init_nls();
 	setlocale (LC_ALL, "");
@@ -1729,22 +1759,23 @@ main(int argc, char **argv)
 	if (!program_invocation_short_name)
 		program_invocation_short_name = argv[0];
 	argp_program_version_hook = version;
+	yy_flex_debug = 0;
 
 	/* Set default logging */
-	log_facility = DEFAULT_LOG_FACILITY;
+  	log_facility = DEFAULT_LOG_FACILITY;
 	log_setup(!stderr_closed_p());
 
 	init_names();
 	init_string_space();
 	builtin_setup();
 	db_format_setup();
-	lex_setup();
+	include_path_setup();
 	save_cmdline(argc, argv);
 	mu_argp_init(program_version, "<" PACKAGE_BUGREPORT ">");
 	mu_argp_parse(&argp, &argc, &argv, 0, capa, &index, NULL);
 
-	log_setup(log_to_stderr);
-	
+  	log_setup(log_to_stderr);
+
 	argv += index;
 	argc -= index;
 
@@ -1773,7 +1804,9 @@ main(int argc, char **argv)
 		}
 		if (script_file[0] != '/')
 			save_cmdline(0, NULL); /* Clear saved command line */
-		if (parse_program(script_file, script_ydebug, script_ldebug))
+		if (preprocess_option)
+			exit(preprocess_input(ext_pp));
+		if (parse_program(script_file, script_ydebug))
 			exit(EX_CONFIG);
 	}
 	process_options();
@@ -1799,7 +1832,7 @@ main(int argc, char **argv)
 	
 	if (script_check || script_dump_macros
 	    || script_dump_code || script_dump_tree
-	    || script_ldebug || script_ydebug)
+	    || yy_flex_debug || script_ydebug)
 		exit(EX_OK);
 
 	free_symbols();
diff --git a/src/pp.c b/src/pp.c
new file mode 100644
index 00000000..92eea078
--- /dev/null
+++ b/src/pp.c
@@ -0,0 +1,665 @@
+/* This file is part of mailfromd.
+   Copyright (C) 2005, 2006, 2007 Sergey Poznyakoff
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+   MA 02110-1301 USA */
+
+#define MF_SOURCE_NAME  MF_SOURCE_PP
+
+#ifdef HAVE_CONFIG_H
+# include <config.h>
+#endif
+#include <sys/types.h>	
+#include <sys/stat.h>
+#define obstack_chunk_alloc malloc
+#define obstack_chunk_free free
+#include <obstack.h>
+#include "mailfromd.h"
+#include "prog.h"
+
+struct input_file_ident {
+	ino_t i_node;
+	dev_t device;
+};
+ 
+struct buffer_ctx {
+	struct buffer_ctx *prev;
+	struct locus locus;
+	size_t namelen;
+	struct input_file_ident id;
+	FILE *infile;
+};
+
+extern int yy_flex_debug;
+static struct buffer_ctx *context_stack;
+
+#define INFILE context_stack->infile
+#define LOCUS context_stack->locus
+
+static char *linebuf;
+static size_t bufsize;
+static char *putback_buffer;
+static size_t putback_size;
+static size_t putback_max;
+
+static int push_source(const char *name, int once);
+static int pop_source(void);
+static int parse_include(const char *text, int once);
+static int parse_require(const char *text);
+
+static void
+putback(const char *str)
+{
+	size_t len;
+	
+	if (!*str)
+		return;
+	len = strlen(str) + 1;
+	if (len > putback_max) {
+		putback_max = len;
+		putback_buffer = xrealloc(putback_buffer, putback_max);
+	}
+	strcpy(putback_buffer, str);
+	putback_size = len - 1;
+}
+
+/* Compute the size of the line
+
+   #line NNN "FILENAME"
+*/
+static size_t
+pp_line_stmt_size()
+{
+	char nbuf[NUMERIC_BUFSIZE_BOUND];
+	size_t n = snprintf(nbuf, sizeof nbuf, "%lu",
+			    (unsigned long) LOCUS.line);
+	if (context_stack->namelen == 0)
+		context_stack->namelen = strlen(LOCUS.file);
+	/* "#line " is 6 chars, another space, two quotes and a linefeed
+	   make another 4, summa facit 10 */
+	return 10 + n + context_stack->namelen;
+}
+
+static void
+pp_line_stmt()
+{
+	char *p;
+	size_t ls_size = pp_line_stmt_size();
+	size_t pb_size = putback_size + ls_size + 1;
+	
+	if (pb_size > putback_max) {
+		putback_max = pb_size;
+		putback_buffer = xrealloc(putback_buffer, putback_max);
+	}
+
+	p = putback_buffer + putback_size;
+	snprintf(p, putback_max - putback_size,
+		 "#line %lu \"%s\"\n",
+		 (unsigned long) LOCUS.line,
+		 LOCUS.file);
+	putback_size += ls_size;
+}
+
+#define STRMATCH(p, len, s) (len >= sizeof(s)                        \
+		 	     && memcmp (p, s, sizeof(s) - 1) == 0    \
+			     && isspace(p[sizeof(s) - 1]))
+
+static int
+next_line()
+{
+	ssize_t rc;
+
+	do {
+		if (putback_size) {
+			if (putback_size > bufsize) {
+				bufsize = putback_size + 1;
+				linebuf = xrealloc(linebuf, bufsize);
+			}
+			strcpy(linebuf, putback_buffer);
+			rc = putback_size;
+			putback_size = 0;
+		} else if (!context_stack)
+			return 0;
+		else 
+			rc = getline(&linebuf, &bufsize, INFILE);
+	} while (rc == -1 && pop_source() == 0);
+	return rc;
+}
+	
+size_t
+pp_fill_buffer(char *buf, size_t size)
+{
+	size_t bufsize = size;
+	int c;
+	char *linep = NULL;
+	
+	while (next_line() > 0) {
+		char *p;
+		size_t len;
+		int is_line = 0;
+			
+		for (p = linebuf; *p && isspace(*p); p++)
+			;
+		if (*p == '#') {
+			size_t l;
+			for (p++; *p && isspace(*p); p++)
+				;
+			l = strlen(p);
+			if (STRMATCH(p, l, "include_once")) {
+				if (parse_include(linebuf, 1))
+					putback("/*include_once*/\n");
+				continue;
+			} else if (STRMATCH(p, l, "include")) {
+				if (parse_include(linebuf, 0))
+					putback("/*include*/\n");
+				continue;
+			} else if (STRMATCH(p, l, "require")) {
+				if (parse_require(linebuf))
+					putback("/*require*/\n");
+				continue;
+			} else if (STRMATCH(p, l, "line")) 
+				is_line = 1;
+		}
+
+		len = strlen (linebuf);
+
+		if (len > size)
+			len = size;
+
+		memcpy(buf, linebuf, len);
+		buf += len;
+		size -= len;
+
+		if (size == 0) {
+			putback(linebuf + len);
+			break;
+		}
+		
+		if (!is_line && len > 0 && linebuf[len-1] == '\n')
+			LOCUS.line++;
+	}
+	return bufsize - size;
+}
+
+#define STAT_ID_EQ(st,id) ((id).i_node == (st).st_ino       \
+			   && (id).device == (st).st_dev)
+
+static struct buffer_ctx *
+ctx_lookup(struct stat *st)
+{