Rewrite preprocessing of native grecs configuration files.

When external preprocessing is required, each source file is preprocessed
separately.  This means that if you use #include or #include_once
statements, a separate copy of external preprocessor will be started
for each included file.  In other words, when both internal and external
preprocessing is used, the order is: external first, internal next.
Previous versions of grecs implemented opposite order, when one external
preprocessor was started, which obtained input from the internal
preprocessor.

One concequence of this is that any macros defined in a configuration file
are no longer visible for files included from it.

* include/grecs/lex.h (grecs_parse_line_directive)
(grecs_parse_line_directive_cpp): Change signature.
* include/grecs/preproc.h (grecs_preproc_init)
(grecs_preproc_done)
(grecs_preproc_run): Remove.
(grecs_preprocess): New proto.
* src/grecs-lex.l: Rewrite.
* src/Make.am: Remove preproc.c
* src/preproc.c: Remove.
* src/bind-lex.l: Update calls to grecs_parse_line_directive*
functions.
* src/dhcpd-lex.l: Likewise.
* tests/Makefile.am: Add new tests.
* tests/testsuite.at: Inlcude new tests.
* tests/epp.at: New file.
* tests/ipp.at: New file.
* tests/fpp.c: Rename to tests/failpp.c
* tests/gcfpp.c: New file.
* tests/incl01.at: Change expected strings.
* tests/ppexit.at: Change expected strings.

16 files changed, 1261 insertions, 1041 deletions

diff --git a/include/grecs/lex.h b/include/grecs/lex.h
index 6ef7114..545427a 100644
--- a/include/grecs/lex.h
+++ b/include/grecs/lex.h
@@ -25,11 +25,9 @@ void grecs_destroy_text(void);
 struct grecs_symtab *grecs_text_table(void);
 
 void grecs_parse_line_directive(char *text, grecs_locus_t *ploc,
-				struct grecs_locus_point *ppoint, 
-				size_t *pxlines);
+				struct grecs_locus_point *ppoint);
 void grecs_parse_line_directive_cpp(char *text, grecs_locus_t *ploc,
-				    struct grecs_locus_point *ppoint, 
-				    size_t *pxlines);
+				    struct grecs_locus_point *ppoint);
 
 void grecs_line_acc_create(void);
 void grecs_line_acc_free(void);
diff --git a/include/grecs/preproc.h b/include/grecs/preproc.h
index 24bc665..5ce26f1 100644
--- a/include/grecs/preproc.h
+++ b/include/grecs/preproc.h
@@ -21,11 +21,8 @@
 
 extern const char *grecs_preprocessor;
 
-size_t grecs_preproc_fill_buffer(char *buf, size_t size);
 void grecs_preproc_add_include_dir(char *dir);
-int grecs_preproc_init(const char *name);
-void grecs_preproc_done(void);
-int grecs_preproc_run(const char *config_file, const char *extpp);
+int grecs_preprocess(char const *name, int trace);
 
 #define GRECS_STD_INCLUDE 0x01
 #define GRECS_USR_INCLUDE 0x02
diff --git a/src/Make.am b/src/Make.am
index 38edba6..8cd8ebd 100644
--- a/src/Make.am
+++ b/src/Make.am
@@ -39,7 +39,6 @@ GRECS_SRC = \
  parser.c\
  parsertab.c\
  path-parser.c\
- preproc.c\
  sort.c\
  symtab.c\
  text.c\
diff --git a/src/bind-lex.l b/src/bind-lex.l
index c1f37d7..7201e53 100644
--- a/src/bind-lex.l
+++ b/src/bind-lex.l
@@ -60,12 +60,10 @@ P [1-9][0-9]*
          /* Line directive */
 ^[ \t]*#[ \t]*{P}[ \t]+\".*\".*\n { grecs_parse_line_directive_cpp(yytext,
 						    &yylloc,
-						    &grecs_current_locus_point,
-						    NULL); }
+						    &grecs_current_locus_point); }
 ^[ \t]*#[ \t]*line[ \t].*\n       { grecs_parse_line_directive(yytext,
 						    &yylloc,
-						    &grecs_current_locus_point,
-						    NULL); }
+						    &grecs_current_locus_point); }
          /* End-of-line comments */
 #.*\n     { grecs_locus_point_advance_line(grecs_current_locus_point); }
 #.*     /* end-of-file comment */;
diff --git a/src/dhcpd-lex.l b/src/dhcpd-lex.l
index 6d05fc9..2e0b667 100644
--- a/src/dhcpd-lex.l
+++ b/src/dhcpd-lex.l
@@ -56,12 +56,10 @@ P [1-9][0-9]*
          /* Line directive */
 ^[ \t]*#[ \t]*{P}[ \t]+\".*\".*\n { grecs_parse_line_directive_cpp(yytext,
 						    &yylloc,
-						    &grecs_current_locus_point,
-						    NULL); }
+						    &grecs_current_locus_point); }
 ^[ \t]*#[ \t]*line[ \t].*\n       { grecs_parse_line_directive(yytext,
 						    &yylloc,
-						    &grecs_current_locus_point,
-						    NULL); }
+						    &grecs_current_locus_point); }
          /* End-of-line comments */
 #.*\n     { grecs_locus_point_advance_line(grecs_current_locus_point); }
 #.*     /* end-of-file comment */;
diff --git a/src/grecs-lex.l b/src/grecs-lex.l
index 2f416ec..c09020e 100644
--- a/src/grecs-lex.l
+++ b/src/grecs-lex.l
@@ -1,6 +1,5 @@
 /* grecs - Gray's Extensible Configuration System       -*- c -*- */
 %option nounput
-%option noinput
 %top {
 #ifdef HAVE_CONFIG_H
 # include <config.h>
@@ -30,34 +29,112 @@
 #include <ctype.h>
 #include <stdlib.h>
 #include <errno.h>
-  
+#include <sys/stat.h>
+#include <sys/wait.h>	
+#include <glob.h>
+#include <signal.h>
 #include <wordsplit.h>
 
-static char *multiline_delimiter;
-static size_t multiline_delimiter_len;
-static int multiline_unescape;         /* Unescape here-document contents */
-static int (*char_to_strip)(char);    /* Strip matching characters of each
-					  here-document line */
-
+/*
+ * Locus tracking
+ */
 struct grecs_locus_point grecs_current_locus_point;   /* Input file location */
-/* Line correction.  Equals to the number of #line directives inserted into
-   the input by the preprocessor instance.  The external preprocessor, if
-   any, counts these as input lines and therefore the line numbers in *its*
-   #line directives are offset by the value of XLINES.
-   
-   Uff, running two preprocessors is confusing...
-*/
-static size_t xlines; 
 
-extern int grecs_yyerror_suppress;;
+#define YY_USER_ACTION do {						\
+		if (YYSTATE == INITIAL || YYSTATE == PP) {		\
+			yylloc.beg = grecs_current_locus_point;		\
+			yylloc.beg.col++;				\
+		}							\
+  		grecs_current_locus_point.col += yyleng;		\
+ 		yylloc.end = grecs_current_locus_point;			\
+   	} while (0);
+
+/*
+ * Redefine default error reporting function.
+ */
+#define YY_FATAL_ERROR(t) grecs_error(NULL, errno, "%s", t)
+
+/*
+ * Preprocessing and synclines.
+ */
+static int initial_state; /* Lexer initial state.
+                           * Set to INITIAL when parsing the file, and to PP
+                           * when generating preprocessed output.
+			   */
+static int emit_syncline; /* If not null, emit syncline before next output
+			   * line.
+			   */
+static int beginning_of_line; /* True if output is at the beginning of line */
+
+/*
+ * Echo LEN bytes from TEXT to yyout.
+ */
+static void
+echo(char const *text, size_t len)
+{
+	if (len == 0) return;
+	/* Check if we should print a syncline */
+	if (beginning_of_line && emit_syncline) {
+		fprintf(yyout, "#line %lu \"%s\"\n",
+			(unsigned long)yylloc.end.line,
+			yylloc.end.file);
+		emit_syncline = 0;
+	}
+	/* Actually echo the text. */
+	fwrite(text, len, 1, yyout);
+	/* Determine the beginning of line status. */
+	beginning_of_line = text[len-1] == '\n';
+}
+/* Inform flex that it should use echo. */
+#define ECHO echo(yytext, yyleng)
+
+/*
+ * String concatenation support.
+ *
+ * String concatenation is a legacy feature of Grecs parser that instructs
+ * it to concatenate adjacent quoted strings.  It is enabled if the
+ * GRECS_OPTION_QUOTED_STRING_CONCAT bit is set in parser options.  This
+ * bit causes the lexer to qualify each double-quoted string as QSTRING.
+ */
+#define qstring()							\
+	((grecs_parser_options & GRECS_OPTION_QUOTED_STRING_CONCAT)	\
+	 ? QSTRING : STRING)
+
+/*
+ * Lexical tie-in variables.
+ */
+extern int grecs_yyerror_suppress;
+
+/*
+ * Here-document support.
+ */
+static char *multiline_delimiter;       /* Here-document delimiter text. */
+static size_t multiline_delimiter_len;  /* Its length. */
+static int multiline_unescape;          /* Unescape here-document contents */
+static int (*char_to_strip)(char);      /* Strips matching characters of each
+					   here-document line */
 
 static void multiline_begin(char *);
 static void multiline_add(char *);
 static char *multiline_strip_tabs(char *text);
-static int ident(void);
 static int isemptystr(int off);
+
+/*
+ * Forward declarations.
+ */
+static int ident(void);
 static void qstring_locus_fixup(void);
+static int parse_include(char *text);
+static int push_source(const char *name, int once);
+static int pop_source(void);
+static void ctx_abort(void);
 
+/*
+ * The report_diag should be called only when yytext begins with the following
+ * prefix:
+ *       ^[ \t]*#[ \t]*.{LEN}
+ * It returns a pointer to yytext past the prefix.
+ */
 static char *
 report_diag(size_t len)
 {
@@ -81,32 +158,9 @@ report_diag(size_t len)
 	}
 	return p;
 }
- 
-#define qstring() \
-	((grecs_parser_options & GRECS_OPTION_QUOTED_STRING_CONCAT)	\
-	 ? QSTRING : STRING)
-
-#undef YY_INPUT
-#define YY_INPUT(buf,result,max_size)					\
-	do {								\
-		if (grecs_preprocessor)					\
-			result = fread(buf, 1, max_size, yyin);		\
-		else							\
-			result = grecs_preproc_fill_buffer(buf, max_size); \
-	} while(0)
-
-#define YY_USER_ACTION do {						\
-		if (YYSTATE == 0) {					\
-			yylloc.beg = grecs_current_locus_point;		\
-			yylloc.beg.col++;				\
-		}							\
-  		grecs_current_locus_point.col += yyleng;		\
- 		yylloc.end = grecs_current_locus_point;			\
-   	} while (0);
-
 %}
-    
-%x COMMENT ML STR
+
+%x COMMENT ML STR PP
 
 WS [ \t\f][ \t\f]*
 ID [a-zA-Z_][a-zA-Z_0-9-]*
@@ -114,20 +168,46 @@ P [1-9][0-9]*
 
 %%
          /* C-style comments */
-"/*"         BEGIN(COMMENT);
-<COMMENT>[^*\n]*        /* eat anything that's not a '*' */
-<COMMENT>"*"+[^*/\n]*   /* eat up '*'s not followed by '/'s */
-<COMMENT>\n          grecs_locus_point_advance_line(grecs_current_locus_point);
-<COMMENT>"*"+"/"        BEGIN(INITIAL);
+<INITIAL,PP>"/*"   { BEGIN(COMMENT); beginning_of_line = 0; }
+<COMMENT>{
+[^*\n]*        /* eat anything that's not a '*' */
+"*"+[^*/\n]*   /* eat up '*'s not followed by '/'s */
+\n          { grecs_locus_point_advance_line(grecs_current_locus_point);
+	      emit_syncline = 1; }
+"*"+"/"     { BEGIN(initial_state); }
+}
+
+         /* Line directive */
+^[ \t]*#[ \t]*{P}[ \t]+\".*\".*\n {
+		 grecs_parse_line_directive_cpp(yytext,
+						&yylloc,
+						&grecs_current_locus_point);
+	 }
+^[ \t]*#[ \t]*line[ \t].*\n       {
+		 grecs_parse_line_directive(yytext,
+					    &yylloc,
+					    &grecs_current_locus_point);
+	 }
+
+<PP>{
          /* Line directive */
-^[ \t]*#[ \t]*{P}[ \t]+\".*\".*\n { grecs_parse_line_directive_cpp(yytext,
-						    &yylloc,
-						    &grecs_current_locus_point,
-						    &xlines); }
-^[ \t]*#[ \t]*line[ \t].*\n       { grecs_parse_line_directive(yytext,
-							       &yylloc,
-							       &grecs_current_locus_point,
-							       &xlines); }
+^[ \t]*#[ \t]*{P}[ \t]+\".*\".*\n {
+		 grecs_parse_line_directive_cpp(yytext,
+						&yylloc,
+						&grecs_current_locus_point);
+		 ECHO;
+	 }
+^[ \t]*#[ \t]*line[ \t].*\n       {
+		 grecs_parse_line_directive(yytext,
+					    &yylloc,
+					    &grecs_current_locus_point);
+		 ECHO;
+	 }
+}
+
+<INITIAL,PP>{
+         /* Includes */
+^[ \t]*#[ \t]*include(_once)?.*/\n { parse_include(yytext); }
          /* Error directives */
 ^[ \t]*#[ \t]*error[ \t].*\n {
   grecs_error(&yylloc, 0, "%s", report_diag(sizeof("error")));
@@ -137,16 +217,20 @@ P [1-9][0-9]*
  }
 ^[ \t]*#[ \t]*abend[ \t].*\n {
   grecs_error(&yylloc, 0, "%s", report_diag(sizeof("abend")));
+  ctx_abort();
   grecs_yyerror_suppress = 1;
-  YY_BREAK;
+  return 0;
  }
          /* End-of-line comments */
-#.*\n     { grecs_locus_point_advance_line(grecs_current_locus_point); }
-#.*     /* end-of-file comment */;
-"//".*\n  { grecs_locus_point_advance_line(grecs_current_locus_point); }
-"//".*    /* end-of-file comment */;
+#.*\n     { grecs_locus_point_advance_line(grecs_current_locus_point);
+            emit_syncline = 1; }
+#.*      /* end-of-file comment */;
+"//".*\n  { grecs_locus_point_advance_line(grecs_current_locus_point);
+            emit_syncline = 1; }
+"//".*   /* end-of-file comment */;
+}
         /* Identifiers */
-<INITIAL>{ID}           return ident();
+{ID}           return ident();
          /* Strings */
 [a-zA-Z0-9_\.\*/:@\[\]-]([a-zA-Z0-9_\./:@\[\]-][a-zA-Z0-9_\.\*/:@\[\]-]*)? {
 	                   grecs_line_begin();
@@ -170,17 +254,17 @@ P [1-9][0-9]*
 		         grecs_line_acc_grow_unescape_last(yytext + 1,
                                                            yyleng - 1, 
                                                            &yylloc); }
-<STR>\"[^\\"\n]*\\\n  { grecs_line_acc_grow_unescape_last(yytext, yyleng, 
-                                                          &yylloc);
-                        grecs_locus_point_advance_line(grecs_current_locus_point); }
-<STR>[^\\"\n]*\\.     { grecs_line_acc_grow_unescape_last(yytext, yyleng, 
-                                                          &yylloc); }
-<STR>[^\\"\n]*\"      { BEGIN(INITIAL);
-                        if (yyleng > 1) 
-				grecs_line_add(yytext, yyleng - 1); 
-                        yylval.string = grecs_line_finish();
-			qstring_locus_fixup();    
-		        return qstring(); }
+<STR>{
+\"[^\\"\n]*\\\n  { grecs_line_acc_grow_unescape_last(yytext, yyleng, &yylloc);
+                   grecs_locus_point_advance_line(grecs_current_locus_point); }
+[^\\"\n]*\\.     { grecs_line_acc_grow_unescape_last(yytext, yyleng, &yylloc); }
+[^\\"\n]*\"      { BEGIN(initial_state);
+                   if (yyleng > 1) 
+			grecs_line_add(yytext, yyleng - 1); 
+                   yylval.string = grecs_line_finish();
+		   qstring_locus_fixup();    
+		   return qstring(); }
+}
          /* Multiline strings */
 "<<"(-" "?)?\\?{ID}[ \t]*#.*\n |
 "<<"(-" "?)?\\?{ID}[ \t]*"//".*\n |
@@ -190,31 +274,35 @@ P [1-9][0-9]*
 "<<"(-" "?)?\"{ID}\"[ \t]*\n {
                 BEGIN(ML);
 		multiline_begin(yytext+2); }
-         /* Ignore m4 line statements */
-<ML>^"#line ".*\n {
+<ML>{
+        /* Ignore m4 line statements */
+^"#line ".*\n {
 	grecs_locus_point_advance_line(grecs_current_locus_point);
 }
-<ML>.*\n { char *p = multiline_strip_tabs(yytext);
+.*\n { char *p = multiline_strip_tabs(yytext);
 	   
-           if (!strncmp(p, multiline_delimiter, multiline_delimiter_len)
-	       && isemptystr(p + multiline_delimiter_len - yytext)) {
-		   grecs_free(multiline_delimiter);
-		   multiline_delimiter = NULL;
-		   BEGIN(INITIAL);
-		   yylval.string = grecs_line_finish();
-
-		   /* Update end pos */
-		   yylloc.end.line--;
-		   for (yylloc.end.col = 0,
-				p = yylval.string + strlen(yylval.string) - 1;
-			p > yylval.string && p[-1] != '\n';
-			yylloc.end.col++, p--);
-		   if (yylloc.end.col == 0)
-			   yylloc.end.col = 1;
-		   return MSTRING;
-	   }
-           grecs_locus_point_advance_line(grecs_current_locus_point);
-	   multiline_add(p); } 
+       if (!strncmp(p, multiline_delimiter, multiline_delimiter_len)
+           && isemptystr(p + multiline_delimiter_len - yytext)) {
+	       grecs_free(multiline_delimiter);
+	       multiline_delimiter = NULL;
+	       BEGIN(initial_state);
+	       yylval.string = grecs_line_finish();
+
+	       /* Update end pos */
+	       yylloc.end.line--;
+	       for (yylloc.end.col = 0,
+			    p = yylval.string + strlen(yylval.string) - 1;
+		    p > yylval.string && p[-1] != '\n';
+		    yylloc.end.col++, p--);
+	       if (yylloc.end.col == 0)
+		       yylloc.end.col = 1;
+	       return MSTRING;
+       }
+       grecs_locus_point_advance_line(grecs_current_locus_point);
+       multiline_add(p);
+    }
+}
+
 {WS}     ;
          /* Other tokens */
 \n       { grecs_locus_point_advance_line(grecs_current_locus_point); } 
@@ -226,60 +314,559 @@ P [1-9][0-9]*
 		grecs_error(&yylloc, 0,
 			     _("stray character \\%03o"),
 			       (unsigned char) yytext[0]); }
+
+<PP>{
+\n       {  grecs_locus_point_advance_line(grecs_current_locus_point);
+	    ECHO; }
+}
+	
 %%
+/*
+ * Source inclusion support.
+ */
+static glob_t include_glob;
+static size_t include_pos;
+static int include_once;
 
-pid_t grecs_preproc_pid;
+static int
+isglob(const char *s)
+{
+	return s[strcspn(s, "*?[")] != 0;
+}
 
-int
-yywrap()
+static int
+parse_include(char *text)
 {
-	if (grecs_preprocessor) {
-	        if (grecs_preproc_extrn_shutdown(grecs_preproc_pid)) {
-			grecs_yyerror_suppress = 1;
-		}
-		fclose(yyin);
+	char *p;
+	int rc = 1;
+	int once;
+
+	p = strchr(text, '#');
+	p += strspn(p, " \t") + sizeof("include");
+	if (strncmp(p, "_once", 5) == 0) {
+		once = 1;
+		p += 5;
 	} else
-		grecs_preproc_done();
-	grecs_current_locus_point.file = NULL;
-	return 1;
+		once = 0;
+	p += strspn(p, " \t");
+	
+	if (!*p) {
+		grecs_error(&yylloc, 0, _("invalid include statement"));
+		return 1;
+	} else {
+		size_t len;
+		int allow_cwd;
+		
+		len = strlen (p);
+		
+		if (p[0] == '<' && p[len - 1] == '>') {
+			allow_cwd = 0;
+			p[len - 1] = 0;
+			p++;
+		} else if (p[0] == '"' && p[len - 1] == '"') {
+			allow_cwd = 1;
+			p[len - 1] = 0;
+			p++;
+		} else {
+			grecs_error(&yylloc, 0, _("invalid include statement"));
+			return 1;
+		}
+		
+		if (isglob(p)) {
+			switch (glob(p, 0, NULL, &include_glob)) {
+			case 0:
+				include_pos = 0;
+				include_once = once;
+				break;
+			case GLOB_NOSPACE:
+				grecs_alloc_die();
+			case GLOB_NOMATCH:
+				break;
+			default:
+				grecs_error(&yylloc, 0,  _("read error"));
+			}
+			p = NULL;
+		} else if (p[0] != '/') {
+			char *q = p;
+			p = grecs_find_include_file(q, allow_cwd);
+			if (!p)
+				grecs_error(&yylloc, 0,
+					    _("%s: No such file or directory"),
+					    q);
+		}
+ 	}
+  
+	if (p)
+		rc = push_source(p, once);
+	else if (include_pos < include_glob.gl_pathc)
+		rc = push_source(include_glob.gl_pathv[include_pos++], once);
+	return rc;
+}
+
+/*
+ * Include search path.
+ */
+static struct grecs_list *grecs_usr_include_path;
+static struct grecs_list *grecs_std_include_path;
+
+size_t
+grecs_include_path_count(int flag)
+{
+    size_t count = 0;
+    if (flag & GRECS_STD_INCLUDE)
+	count += grecs_list_size(grecs_std_include_path);
+    if (flag & GRECS_USR_INCLUDE)
+	count += grecs_list_size(grecs_usr_include_path);
+    return count;
+}
+
+static int
+foreach_dir(struct grecs_list *list, int flag,
+	    int (*fun)(int, const char *, void *), void *data)
+{
+    int rc = 0;
+    struct grecs_list_entry *ep;
+
+    for (ep = list->head; rc == 0 && ep; ep = ep->next)
+	rc = fun(flag, ep->data, data);
+    return rc;
 }
 
 int
-grecs_lex_begin(const char *name, int trace)
+grecs_foreach_include_dir(int flag, int (*fun)(int, const char *, void *),
+			  void *data)
 {
-	yy_flex_debug = trace;
+    int rc = 0;
 
-	grecs_line_acc_create();
+    if (flag & GRECS_STD_INCLUDE)
+	rc = foreach_dir(grecs_std_include_path, GRECS_STD_INCLUDE, fun, data);
+    if (rc == 0 && (flag & GRECS_USR_INCLUDE))
+	rc = foreach_dir(grecs_usr_include_path, GRECS_USR_INCLUDE, fun, data);
+    return rc;
+}
+
+void
+grecs_include_path_clear()
+{
+	if (grecs_usr_include_path)
+		grecs_list_clear(grecs_usr_include_path);
+	if (grecs_std_include_path)
+		grecs_list_clear(grecs_std_include_path);
+}
+
+static void
+incl_free(void *data)
+{
+	grecs_free(data);
+}
+
+void
+grecs_include_path_setup_v(char **dirs)
+{
+	if (!grecs_usr_include_path) {
+		grecs_usr_include_path = grecs_list_create();
+		grecs_usr_include_path->free_entry = incl_free;
+	}
+	grecs_std_include_path = grecs_list_create();
+	grecs_std_include_path->free_entry = incl_free;
+	if (dirs) {
+		int i;
+		for (i = 0; dirs[i]; i++)
+			/* FIXME: Element never freed */
+			grecs_list_append(grecs_std_include_path,
+					  grecs_strdup(dirs[i]));
+	}
+}
+
+void
+grecs_include_path_setup(const char *dir, ...)
+{
+	const char *p;
+	char **argv = NULL;
+	size_t argc = 0;
+	size_t argi = 0;
+	va_list ap;
 	
-	if (grecs_preprocessor) {
-		int fd;
+	va_start(ap, dir);
+	p = dir;
+	while (1) {
+		if (argi == argc) {
+			if (argc == 0)
+				argc = 16;
+			else
+				argc += 16;
+			argv = grecs_realloc(argv, argc * sizeof(argv[0]));
+		}
+		argv[argi++] = (char*) p;
+		if (!p)
+			break;
+		p = va_arg(ap, const char*);
+	}
+	grecs_include_path_setup_v(argv);
+	grecs_free(argv);
+	va_end(ap);
+}
+
+void
+grecs_preproc_add_include_dir(char *dir)
+{
+	if (!grecs_usr_include_path) {
+		grecs_usr_include_path = grecs_list_create();
+		grecs_usr_include_path->free_entry = incl_free;
+	}
+	grecs_list_append(grecs_usr_include_path, grecs_strdup(dir));
+}
+
+struct file_data {
+	const char *name;
+	size_t namelen;
+	char *buf;
+	size_t buflen;
+	int found;
+};
+
+static int
+pp_list_find(struct grecs_list *list, struct file_data *dptr)
+{
+	struct grecs_list_entry *ep;
+
+	if (!list)
+		return 0;
+	for (ep = list->head; !dptr->found && ep; ep = ep->next) {
+		const char *dir = ep->data;
+		size_t size = strlen (dir) + 1 + dptr->namelen + 1;
+		if (size > dptr->buflen) {
+			dptr->buflen = size;
+			dptr->buf = grecs_realloc(dptr->buf, dptr->buflen);
+		}
+		strcpy(dptr->buf, dir);
+		strcat(dptr->buf, "/");
+		strcat(dptr->buf, dptr->name);
+		dptr->found = access(dptr->buf, F_OK) == 0;
+	}
+	return dptr->found;
+}
+
+char *
+grecs_find_include_file(const char *name, int allow_cwd)
+{
+	static char *cwd = ".";
+	struct file_data fd;
+
+	fd.name = name;
+	fd.namelen = strlen(name);
+	fd.buf = NULL;
+	fd.buflen = 0;
+	fd.found = 0;
+
+	if (!grecs_usr_include_path)
+		grecs_include_path_setup(NULL);
+	if (allow_cwd) {
+		grecs_list_append(grecs_usr_include_path, cwd);
+		pp_list_find(grecs_usr_include_path, &fd);
+		grecs_list_remove_tail(grecs_usr_include_path);
+	} else
+		pp_list_find(grecs_usr_include_path, &fd);
+
+	if (!fd.found) {
+		pp_list_find(grecs_std_include_path, &fd);
+		if (!fd.found)
+			return NULL;
+	}
+	return fd.buf;
+}
+
+/*
+ * Input context support.
+ */
+
+struct input_file_ident {
+	ino_t i_node;
+	dev_t device;
+	pid_t pid;
+};
+
+struct buffer_ctx {
+	struct buffer_ctx *prev;	/* Pointer to previous context */
+	struct grecs_locus locus;
+	struct grecs_locus_point point;	/* Current input location */
+	size_t namelen;		/* Length of the file name */
+	struct input_file_ident id;
+	FILE *infile;
+	YY_BUFFER_STATE state;
+};
+
+static struct buffer_ctx *context_stack;
+static struct input_file_ident input_file_ident;
+
+#define STAT_ID_EQ(st,id) ((id).i_node == (st).st_ino       \
+			    && (id).device == (st).st_dev)
+
+static struct buffer_ctx *
+ctx_lookup(struct stat *st)
+{
+	struct buffer_ctx *ctx;
+
+	if (!context_stack)
+		return NULL;
+
+	for (ctx = context_stack; ctx; ctx = ctx->prev)
+		if (STAT_ID_EQ(*st, ctx->id))
+			break;
+	return ctx;
+}
+
+static void
+ctx_abort(void)
+{
+	/* Delete current input buffer */
+	yy_delete_buffer(YY_CURRENT_BUFFER);
+			
+	/* Clean-up context stack */
+	while (context_stack) {
+		struct buffer_ctx *prev = context_stack->prev;
+		if (context_stack->id.pid > 0)
+			kill(context_stack->id.pid, SIGKILL);
+		fclose(context_stack->infile);
+		grecs_free(context_stack);
+		context_stack = prev;
+	}
+}
+
+static struct grecs_symtab *incl_sources;
+
+/* Calculate the hash of a struct input_file_ident.  */
+static unsigned
+incl_hasher(void *data, unsigned long n_buckets)
+{
+	const struct input_file_ident *id = data;
+	return (id->i_node + id->device) % n_buckets;
+}
+
+/* Compare two input_file_idents for equality.  */
+static int
+incl_compare(void const *data1, void const *data2)
+{
+	const struct input_file_ident *id1 = data1;
+	const struct input_file_ident *id2 = data2;
+	return !(id1->device == id2->device && id1->i_node == id2->i_node);
+}
+
+static int
+incl_copy(void *dst, void *src)
+{
+	memcpy(dst, src, sizeof(struct input_file_ident));
+	return 0;
+}
+
+static int
+source_lookup(struct stat *st)
+{
+	struct input_file_ident key;
+	int install = 1;
+  
+	if (!incl_sources) {
+		incl_sources = grecs_symtab_create(
+			sizeof(struct input_file_ident), 
+			incl_hasher,
+			incl_compare,
+			incl_copy,
+			NULL,/*FIXME: alloc*/
+			NULL);
+		if (!incl_sources)
+			grecs_alloc_die();
+	}
 	
-		fd = open(name, O_RDONLY);
-		if (fd == -1) {
-			grecs_error(NULL, errno, _("Cannot open `%s'"), name);
+	key.i_node = st->st_ino;
+	key.device = st->st_dev;
+	if (!grecs_symtab_lookup_or_install(incl_sources, &key, &install))
+		grecs_alloc_die();
+	return !install;
+}
+
+static int
+push_source(const char *name, int once)
+{
+	FILE *fp;
+	pid_t pid;
+	struct buffer_ctx *ctx;
+	struct stat st;
+	int rc = stat(name, &st);
+
+	if (context_stack) {
+		if (rc) {
+			grecs_error(&yylloc, errno,
+				    _("Cannot stat `%s'"), name);
 			return 1;
 		}
-		close(fd);
 
-		yyin = grecs_preproc_extrn_start(name, &grecs_preproc_pid);
-		if (!yyin) {
-			grecs_error(NULL, errno,
-				     _("Unable to start external preprocessor `%s'"),
-				     grecs_preprocessor);
+		if (grecs_current_locus_point.col &&
+		    STAT_ID_EQ(st, input_file_ident)) {
+			grecs_error(&yylloc, 0, _("Recursive inclusion"));
 			return 1;
 		}
-	} else
-		return grecs_preproc_init(name);
+		
+		if ((ctx = ctx_lookup(&st))) {
+			grecs_error(&yylloc, 0, _("Recursive inclusion"));
+			if (ctx->prev)
+				grecs_error(&ctx->prev->locus, 0,
+					    _("`%s' already included here"),
+					    name);
+			else
+				grecs_error(&yylloc, 0,
+					    _("`%s' already included at top level"),
+					    name);
+			return 1;
+		}
+	} else if (rc) {
+		grecs_error(NULL, errno, _("Cannot stat `%s'"), name);
+		return 1;
+	}
+
+	if (once && source_lookup(&st))
+		return -1;
+
+	if (grecs_preprocessor) {
+		if (access(name,R_OK)) {
+			grecs_error(context_stack ? &yylloc : NULL, errno,
+				    _("Cannot open `%s'"), name);
+			return -1;
+		}
+		fp = grecs_preproc_extrn_start(name, &pid);
+	} else {
+		fp = fopen(name, "r");
+		pid = -1;
+	}
+	if (!fp) {
+		grecs_error(context_stack ? &yylloc : NULL, errno,
+			    _("Cannot open `%s'"), name);
+		return 1;
+	}
+
+	/* Push current context */
+	if (yyin) {
+		ctx = grecs_malloc(sizeof(*ctx));
+		ctx->locus = yylloc;
+		ctx->point = grecs_current_locus_point;
+		ctx->id = input_file_ident;
+		ctx->infile = yyin;
+		ctx->state = YY_CURRENT_BUFFER;
+		ctx->prev = context_stack;
+		context_stack = ctx;
+	}
+	
+	yyin = fp;
+	yy_switch_to_buffer(yy_create_buffer(yyin, YY_BUF_SIZE));
+
+	grecs_current_locus_point.file = grecs_install_text(name);
+	grecs_current_locus_point.line = 1;
+	grecs_current_locus_point.col = 0;
+	
+	input_file_ident.i_node = st.st_ino;
+	input_file_ident.device = st.st_dev;
+	input_file_ident.pid = pid;
+
+	beginning_of_line = 1;
+	
+	if (yy_flex_debug)
+	  fprintf (stderr, "Processing file `%s'\n", name);
+
+	if (initial_state == PP)
+		emit_syncline = 1;
+	
+	return 0;
+}
+
+static int
+pop_source(void)
+{
+	struct buffer_ctx *ctx;
+
+	fclose(yyin);
+	if (input_file_ident.pid > 0) {
+		if (grecs_preproc_extrn_shutdown(input_file_ident.pid)) {
+			/* Abort further input */
+			ctx_abort();
+
+			/* Suppress further errors */
+			grecs_yyerror_suppress = 1;
+
+			/* Indicate there is no more input */
+			return 1;
+		}
+	}
+	
+	if (!context_stack) {
+		if (yy_flex_debug)
+			fprintf(stderr, "End of input\n");
+		yyin = NULL;
+		return 1;
+	}
+	
+	/* Restore previous context */
+	yyin = context_stack->infile;
+	grecs_current_locus_point = context_stack->point;
+	input_file_ident = context_stack->id;
+	
+	yy_delete_buffer(YY_CURRENT_BUFFER);
+	yy_switch_to_buffer(context_stack->state);
+	
+	ctx = context_stack->prev;
+	grecs_free(context_stack);
+	context_stack = ctx;
+
+	while (include_pos < include_glob.gl_pathc) {
+		if (push_source(include_glob.gl_pathv[include_pos++],
+				include_once) == 0)
+			return 0;
+	}
+
+	if (include_glob.gl_pathc) {
+		globfree(&include_glob);
+		include_pos = include_glob.gl_pathc = 0;
+	}
+	
+	if (yy_flex_debug)
+		fprintf(stderr, "Resuming file `%s' at line %lu\n",
+			grecs_current_locus_point.file,
+			(unsigned long) grecs_current_locus_point.line);
 
+	if (initial_state == PP) {
+		input();
+		grecs_locus_point_advance_line(grecs_current_locus_point);
+		emit_syncline = 1;
+	}
+	
 	return 0;
 }
+
+/*
+ * Internal lexer functions.
+ */
+int
+yywrap()
+{
+	return pop_source();
+}
+
+int
+grecs_lex_begin(const char *name, int trace)
+{
+	yy_flex_debug = trace;
+	initial_state = INITIAL;
+	grecs_line_acc_create();
+	return push_source(name, 1);
+}
 
 void
 grecs_lex_end(int err)
 {
 	grecs_line_acc_free();
 }
-
+
+/*
+ * Here-document functions.
+ */
 static int
 isemptystr(int off)
 {
@@ -371,9 +958,12 @@ multiline_begin(char *p)
 	yylloc.beg = grecs_current_locus_point;
 	yylloc.beg.col++;
 }
-
+
+/*
+ * Auxiliary lexer functions.
+ */
 static int
-ident()
+ident(void)
 {
 	char *p;
 	char *str;
@@ -390,7 +980,7 @@ ident()
 }
 
 static void
-qstring_locus_fixup()
+qstring_locus_fixup(void)
 {
 	if (grecs_parser_options & GRECS_OPTION_ADJUST_STRING_LOCATIONS) {
 		yylloc.beg.col++;
@@ -405,27 +995,27 @@ grecs_value_ptr_from_static(grecs_value_t *input)
 	*ptr = *input;
 	return ptr;