Pull fixes to wordsplit from mailutils

This includes the following commits pushed between
2015-09-19 and 2017-10-10:

 090c7b9a       Allow ws_getvar to set value to NULL and
                return MU_WRDSE_OK.
                The value is processed as if it were "", i.e.
                MU_WRDSE_UNDEF is returned.
 64313fdf       Fix MU_WRDSF_INCREMENTAL | MU_WRDSF_NOSPLIT
 46d7640f       Add wordsplit_append function
 151eb4b9       Fix nested expansions and command expansions
                occurring after variable expansions.
 ad3cc340       Replace void wordsplit_getwords with
                int wordsplit_get_words.

* include/wordsplit.h (wordsplit_get_words): New function.
(wordsplit_getwords): Mark as deprecated.
(wordsplit_append): New function.
* src/wordsplit.c (wordsplit_append): New function. MU 46d7640f.
(expvar): Treat NULL value as "". MU 090c7b9a.

(expcmd): Allow command and variable expansions in subsplit.
(exptab): Change ordering of expansions so that command expansion
occurs first. This fixes nested expansions and command expansions
occurring after variable expansions. MU 151eb4b9.

(wordsplit_process_list): Update wsp->ws_endp in nosplit mode. This
fixes wordsplit MU_WRDSF_INCREMENTAL | MU_WRDSF_NOSPLIT. MU 64313fdf.

(wordsplit_get_words): New function. MU ad3cc340.

* tests/wordsplit.at: Test the above changes.
* tests/wsp.c: Accept extra arguments to append using wordsplit_append.

4 files changed, 155 insertions, 23 deletions

diff --git a/include/wordsplit.h b/include/wordsplit.h
index a7f6dd5..eed88bc 100644
--- a/include/wordsplit.h
+++ b/include/wordsplit.h
@@ -235,7 +235,18 @@ int wordsplit_len (const char *s, size_t len, wordsplit_t *ws, int flags);
 void wordsplit_free (wordsplit_t *ws);
 void wordsplit_free_words (wordsplit_t *ws);
 void wordsplit_free_envbuf (wordsplit_t *ws);
-void wordsplit_getwords (wordsplit_t *ws, size_t *wordc, char ***wordv);
+int wordsplit_get_words (wordsplit_t *ws, size_t *wordc, char ***wordv);
+
+static inline void wordsplit_getwords (wordsplit_t *ws, size_t *wordc, char ***wordv)
+  __attribute__ ((deprecated));
+
+static inline void
+wordsplit_getwords (wordsplit_t *ws, size_t *wordc, char ***wordv)
+{
+  wordsplit_get_words (ws, wordc, wordv);
+}
+
+int wordsplit_append (wordsplit_t *wsp, int argc, char **argv);
 
 int wordsplit_c_unquote_char (int c);
 int wordsplit_c_quote_char (int c);
diff --git a/src/wordsplit.c b/src/wordsplit.c
index 9381ae7..4884a22 100644
--- a/src/wordsplit.c
+++ b/src/wordsplit.c
@@ -674,7 +674,35 @@ wordsplit_finish (struct wordsplit *wsp)
   return 0;
 }
 
+int
+wordsplit_append (wordsplit_t *wsp, int argc, char **argv)
+{
+  int rc;
+  size_t i;
 
+  rc = alloc_space (wsp, wsp->ws_wordc + argc + 1);
+  if (rc)
+    return rc;
+  for (i = 0; i < argc; i++)
+    {
+      char *newstr = strdup (argv[i]);
+      if (!newstr)
+	{
+	  while (i > 0)
+	    {
+	      free (wsp->ws_wordv[wsp->ws_offs + wsp->ws_wordc + i - 1]);
+	      wsp->ws_wordv[wsp->ws_offs + wsp->ws_wordc + i - 1] = NULL;
+	      i--;
+	    }
+	  return _wsplt_nomem (wsp);
+	}
+      wsp->ws_wordv[wsp->ws_offs + wsp->ws_wordc + i] = newstr;
+    }
+  wsp->ws_wordc += i;
+  wsp->ws_wordv[wsp->ws_offs + wsp->ws_wordc] = NULL;
+  return 0;
+}
+
 /* Variable expansion */
 static int
 node_split_prefix (struct wordsplit *wsp,
@@ -1015,7 +1043,9 @@ expvar (struct wordsplit *wsp, const char *str, size_t len,
       else
 	rc = WRDSE_UNDEF;
 
-      if (rc == WRDSE_OK && value[0] == 0 && defstr && defstr[-1] == ':')
+      if (rc == WRDSE_OK
+	  && (!value || value[0] == 0)
+	  && defstr && defstr[-1] == ':')
 	{
 	  free (value);
 	  rc = WRDSE_UNDEF;
@@ -1332,7 +1362,6 @@ expcmd (struct wordsplit *wsp, const char *str, size_t len,
       struct wordsplit ws;
 
       rc = _wsplt_subsplit (wsp, &ws, str, j,
-			    WRDSF_NOVAR | WRDSF_NOCMD |
 			    WRDSF_WS | WRDSF_QUOTE);
       if (rc)
 	{
@@ -2065,29 +2094,41 @@ wordsplit_c_quote_copy (char *dst, const char *src, int quote_hex)
     }
 }
 
+
+/* This structure describes a single expansion phase */
 struct exptab
 {
-  char *descr;
-  int flag;
-  int opt;
-  int (*expansion) (struct wordsplit *wsp);
+  char *descr;     /* Textual description (for debugging) */
+  int flag;        /* WRDSF_ bit that controls this phase */
+  int opt;         /* Entry-specific options (see EXPOPT_ flags below */
+  int (*expansion) (struct wordsplit *wsp); /* expansion function */
 };
 
+/* The following options control expansions: */
+/* Normally the exptab entry is run if its flag bit is set in struct
+   wordsplit.  The EXPOPT_NEG option negates this test so that expansion
+   is performed if its associated flag bit is not set in struct wordsplit. */
 #define EXPOPT_NEG      0x01
+/* Coalesce the input list before running the expansion. */
 #define EXPOPT_COALESCE 0x02
 
 static struct exptab exptab[] = {
-  { N_("WS trimming"),          WRDSF_WS,         0, wordsplit_trimws },
-  { N_("tilde expansion"),      WRDSF_PATHEXPAND, 0, wordsplit_tildexpand },
+  { N_("WS trimming"),          WRDSF_WS,         0,
+    wordsplit_trimws },
+  { N_("command substitution"), WRDSF_NOCMD,      EXPOPT_NEG|EXPOPT_COALESCE,
+    wordsplit_cmdexp },
+  { N_("coalesce list"),        0,                   EXPOPT_NEG|EXPOPT_COALESCE,
+    NULL },
+  { N_("tilde expansion"),      WRDSF_PATHEXPAND, 0,
+    wordsplit_tildexpand },
   { N_("variable expansion"),   WRDSF_NOVAR,      EXPOPT_NEG,
     wordsplit_varexp },
   { N_("quote removal"),        0,                EXPOPT_NEG,
     wsnode_quoteremoval },
-  { N_("command substitution"), WRDSF_NOCMD,      EXPOPT_NEG|EXPOPT_COALESCE,
-    wordsplit_cmdexp },
   { N_("coalesce list"),        0,                EXPOPT_NEG|EXPOPT_COALESCE,
     NULL },
-  { N_("path expansion"),       WRDSF_PATHEXPAND, 0, wordsplit_pathexpand },
+  { N_("path expansion"),       WRDSF_PATHEXPAND, 0,
+    wordsplit_pathexpand },
   { NULL }
 };
     
@@ -2101,6 +2142,7 @@ wordsplit_process_list (struct wordsplit *wsp, size_t start)
       /* Treat entire input as a quoted argument */
       if (wordsplit_add_segm (wsp, start, wsp->ws_len, _WSNF_QUOTE))
 	return wsp->ws_errno;
+      wsp->ws_endp = wsp->ws_len;
     }
   else
     {
@@ -2282,16 +2324,21 @@ wordsplit_free (struct wordsplit *ws)
   wordsplit_free_envbuf (ws);
 }
 
-void
-wordsplit_getwords (struct wordsplit *ws, size_t *wordc, char ***wordv)
+int
+wordsplit_get_words (struct wordsplit *ws, size_t *wordc, char ***wordv)
 {
   char **p = realloc (ws->ws_wordv,
 		      (ws->ws_wordc + 1) * sizeof (ws->ws_wordv[0]));
-  *wordv = p ? p : ws->ws_wordv;
+  if (!p)
+    return -1;
+  *wordv = p;
   *wordc = ws->ws_wordc;
+
   ws->ws_wordv = NULL;
   ws->ws_wordc = 0;
   ws->ws_wordn = 0;
+
+  return 0;
 }
 
 const char *_wordsplit_errstr[] = {
diff --git a/tests/wordsplit.at b/tests/wordsplit.at
index 895a392..49d47e9 100644
--- a/tests/wordsplit.at
+++ b/tests/wordsplit.at
@@ -637,6 +637,14 @@ NF: 1
 [input exhausted
 ])
 
+TESTWSP([incremental nosplit],[],[incremental nosplit],
+[incremental "input test" line
+],
+[NF: 1
+0: "incremental \"input test\" line"
+],
+[input exhausted
+])
 
 dnl Something that doesn't fit into TESTWSP
 
@@ -717,15 +725,16 @@ mkdir dir
 > dir/file
 
 DIR=dir wsp -nocmd -novar<<'EOT'
-begin $(find $DIR) end
+begin $DIR $(find $DIR) end
 EOT
 ],
 [0],
-[NF: 4
+[NF: 5
 0: begin
 1: dir
-2: dir/file
-3: end
+2: dir
+3: dir/file
+4: end
 ])
 AT_CLEANUP
 
@@ -735,8 +744,8 @@ AT_CHECK([
 mkdir dir
 > dir/file
 
-DIR=dir wsp -nocmd -novar<<'EOT'
-"begin($(find $DIR))end"
+DIR=dir BEGIN=begin wsp -nocmd -novar<<'EOT'
+"${BEGIN}($(find $DIR))end"
 EOT
 ],
 [0],
@@ -744,7 +753,27 @@ EOT
 0: "begin(dir dir/file)end"
 ])
 AT_CLEANUP
-
+
+AT_SETUP([nested commands])
+AT_KEYWORDS([wordsplit wsp wsp-cmd])
+AT_CHECK([
+AT_DATA([input],[foo
+bar
+baz
+])
+SUFFIX=put wsp -nocmd -novar <<'EOT'
+$(echo output $(cat in$SUFFIX))
+EOT
+],
+[0],
+[NF: 4
+0: output
+1: foo
+2: bar
+3: baz
+])
+AT_CLEANUP
+
 AT_SETUP([pathname expansion])
 AT_KEYWORDS([wordsplit wsp wsp-path wsp-path-1])
 AT_CHECK([
@@ -820,6 +849,32 @@ EOT
 ])
 AT_CLEANUP
 
+TESTWSP([append],[],[-- extra arguments follow],
+[some words and],
+[NF: 6
+0: some
+1: words
+2: and
+3: extra
+4: arguments
+5: follow
+])
+
+TESTWSP([append + dooffs + env],[],
+[dooffs 2 preface words V=2 -- extra arguments follow],
+[some words and var=$V],
+[NF: 7 (2)
+(0): preface
+(1): words
+2: some
+3: words
+4: and
+5: var=2
+6: extra
+7: arguments
+8: follow
+])
+
 m4_popdef([TESTWSP])
 m4_popdef([wspnum])
 m4_popdef([wspid])
diff --git a/tests/wsp.c b/tests/wsp.c
index e8eb9c6..84efc13 100644
--- a/tests/wsp.c
+++ b/tests/wsp.c
@@ -91,7 +91,7 @@ help ()
 {
   size_t i;
   
-  printf ("usage: %s [options] [VAR=VALUE...]\n", progname);
+  printf ("usage: %s [options] [VAR=VALUE...] [-- EXTRA...]\n", progname);
   printf ("options are:\n");
   printf (" [-]trimnl\n");
   printf (" [-]plaintext\n");
@@ -334,6 +334,8 @@ main (int argc, char **argv)
   size_t fenvidx = 0;
   size_t fenvmax = sizeof (fenvbase) / sizeof (fenvbase[0]);
   int use_env = env_sys;
+  int appendc = 0;
+  char **appendv = NULL;
   
   progname = argv[0];
 
@@ -346,6 +348,12 @@ main (int argc, char **argv)
 
       if (opt[0] == '-')
 	{
+	  if (opt[1] == '-' && opt[2] == 0)
+	    {
+	      appendc = argc - i - 1;
+	      appendv = argv + i + 1;
+	      break;
+	    }
 	  negate = 1;
 	  opt++;
 	}
@@ -604,6 +612,17 @@ main (int argc, char **argv)
 	  offarg = 0;
 	}
 
+      if (appendc)
+	{
+	  rc = wordsplit_append (&ws, appendc, appendv);
+	  if (rc)
+	    {
+	      if (!(wsflags & WRDSF_SHOWERR))
+		wordsplit_perror (&ws);
+	      continue;
+	    }
+	}
+      
       wsflags |= WRDSF_REUSE | (ws.ws_flags & WRDSF_ENV);
       printf ("NF: %lu", (unsigned long) ws.ws_wordc);
       if (wsflags & WRDSF_DOOFFS)