imap4d: implement SEARCH CHARSET; fix BODY and TEXT searches.

* imap4d/search.c: Implement proper BODY searches. Implement CHARSET.
* imap4d/util.c (util_strcasestr): Remove. Use mu_c_strcasestr instead.
* imap4d/imap4d.h: Likewise.
* NEWS: Document changes.

5 files changed, 241 insertions, 41 deletions

diff --git a/NEWS b/NEWS
index 84d36010c..4f5c0dec9 100644
--- a/NEWS
+++ b/NEWS
@@ -1,4 +1,4 @@
-GNU mailutils NEWS -- history of user-visible changes. 2019-01-23
+GNU mailutils NEWS -- history of user-visible changes. 2019-01-26
 Copyright (C) 2002-2019 Free Software Foundation, Inc.
 See the end of file for copying conditions.
 
@@ -6,6 +6,15 @@ Please send mailutils bug reports to <bug-mailutils@gnu.org>.
 
 Version 3.5.90 (git)
 
+* imap4d: SEARCH command
+
+** Implemented SEARCH CHARSET
+
+** Improved SEARCH BODY and SEARCH TEXT commands
+
+Both commands now properly descend into multipart message parts and
+decode messages, if necessary
+
 * Fixes in the 'mail' utility
 
 ** New mailbox notation @
diff --git a/imap4d/imap4d.h b/imap4d/imap4d.h
index 2ad339a30..b90d653a0 100644
--- a/imap4d/imap4d.h
+++ b/imap4d/imap4d.h
@@ -454,7 +454,6 @@ extern int util_parse_822_date (const char *date, time_t *timep,
                                 enum datetime_parse_mode flag);
 extern int util_parse_ctime_date (const char *date, time_t *timep,
                                   enum datetime_parse_mode flag);
-extern char *util_strcasestr (const char *haystack, const char *needle);
 extern char *util_localname (void);
 
 void util_print_flags (mu_attribute_t attr);
diff --git a/imap4d/search.c b/imap4d/search.c
index 1d28b570f..3c243eddf 100644
--- a/imap4d/search.c
+++ b/imap4d/search.c
@@ -15,6 +15,7 @@
    along with GNU Mailutils.  If not, see <http://www.gnu.org/licenses/>. */
 
 #include "imap4d.h"
+#include <mailutils/assoc.h>
 
 /*
  * This will be a royal pain in the arse to implement
@@ -224,6 +225,7 @@ struct parsebuf
   int isuid;                    /* UIDs instead of msgnos are required */
   char *err_mesg;               /* Error message if a parse error occured */
   struct mem_chain *alloc;      /* Chain of objects allocated during parsing */
+  char *charset;                /* Charset, other than US-ASCII requested */
 
   struct search_node *tree;     /* Parse tree */
 
@@ -241,6 +243,7 @@ static struct search_node *parse_search_key (struct parsebuf *pb);
 static int parse_gettoken (struct parsebuf *pb, int req);
 static int search_run (struct parsebuf *pb);
 static void do_search (struct parsebuf *pb);
+static int available_charset (const char *charset);
 
 /*
 6.4.4.  SEARCH Command
@@ -296,16 +299,21 @@ imap4d_search0 (imap4d_tokbuf_t tok, int isuid, char **err_text)
       /* Currently only ASCII is supported */
       if (mu_c_strcasecmp (parsebuf.token, "US-ASCII"))
         {
-          *err_text = "Charset not supported";
+          parsebuf.charset = parse_strdup (&parsebuf, parsebuf.token);
+          if (!available_charset (parsebuf.charset))
+            {
+              *err_text = "[BADCHARSET] Charset not supported";
               return RESP_NO;
             }
+        }
+      else
+        parsebuf.charset = NULL;
 
       if (!parse_gettoken (&parsebuf, 0))
         {
           *err_text = "Too few args";
           return RESP_BAD;
         }
-
     }
 
   /* Compile the expression */
@@ -796,15 +804,38 @@ search_run (struct parsebuf *pb)
 static int
 _scan_header (struct parsebuf *pb, char *name, char *value)
 {
-  const char *hval;
+  char *hval;
   mu_header_t header = NULL;
+  int i, rc;
+  int result = 0;
 
   mu_message_get_header (pb->msg, &header);
-  if (mu_header_sget_value (header, name, &hval) == 0)
+
+  for (i = 1;
+       result == 0
+         && (rc = mu_header_aget_value_unfold_n (header, name, i, &hval)) == 0;
+       i++)
+    {
+      if (pb->charset)
+        {
+          char *tmp;
+          rc = mu_rfc2047_decode (pb->charset, hval, &tmp);
+          if (rc)
             {
-      return util_strcasestr (hval, value) != NULL;
+              mu_diag_funcall (MU_DIAG_ERR, "mu_rfc2047_decode", hval, rc);
+              free (hval);
+              continue;
             }
-  return 0;
+          free (hval);
+          hval = tmp;
+        }
+      result = mu_c_strcasestr (hval, value) != NULL;
+      free (hval);
+    }
+  if (!(rc == 0 || rc == MU_ERR_NOENT))
+    mu_diag_funcall (MU_DIAG_ERR, "mu_header_aget_value_unfold_n", NULL, rc);
+
+  return result;
 }
 
 /* Get the value of Date: field and convert it to timestamp */
@@ -825,47 +856,192 @@ _header_date (struct parsebuf *pb, time_t *timep)
 static int
 _scan_header_all (struct parsebuf *pb, char *text)
 {
-  const char *hval;
   mu_header_t header = NULL;
   size_t fcount = 0;
   int i, rc;
+  int result;
 
   mu_message_get_header (pb->msg, &header);
   mu_header_get_field_count (header, &fcount);
-  for (i = rc = 0; i < fcount; i++)
+  result = 0;
+  for (i = 1; result == 0 && i < fcount; i++)
     {
-      if (mu_header_sget_field_value (header, i, &hval) == 0)
-        rc = util_strcasestr (hval, text) != NULL;
+      char *hval;
+
+      rc = mu_header_aget_field_value_unfold (header, i, &hval);
+      if (rc)
+        {
+          mu_diag_funcall (MU_DIAG_ERR, "mu_header_aget_field_value_unfold",
+                           NULL, rc);
+          continue;
         }
-  return rc;
+
+      if (pb->charset)
+        {
+          char *tmp;
+          rc = mu_rfc2047_decode (pb->charset, hval, &tmp);
+          if (rc)
+            {
+              mu_diag_funcall (MU_DIAG_ERR, "mu_rfc2047_decode", hval, rc);
+              free (hval);
+              continue;
+            }
+
+          free (hval);
+          hval = tmp;
+        }
+       result = mu_c_strcasestr (hval, text) != NULL;
+       free (hval);
+    }
+  return result;
 }
 
-/* Scan body of the message for the occurrence of a substring */
-/* FIXME: The algorithm below is broken */
 static int
-_scan_body (struct parsebuf *pb, char *text)
+_match_text (struct parsebuf *pb, mu_message_t msg, mu_content_type_t ct,
+             char const *encoding,
+             char *text)
 {
-  mu_body_t body = NULL;
-  mu_stream_t stream = NULL;
-  size_t size = 0, lines = 0;
-  char buffer[128];
-  size_t n = 0;
+  mu_body_t body;
+  mu_stream_t str;
   int rc;
+  int result;
+  char *buffer = NULL;
+  size_t bufsize = 0;
+  size_t n;
+
+  mu_message_get_body (msg, &body);
+  mu_body_get_streamref (body, &str);
+
+  if (encoding)
+    {
+      mu_stream_t flt;