New function to reconstruct message envelope from the headers.

* include/mailutils/sys/message_stream.h (_mu_message_stream): Reorganize
structure.
* libmailutils/base/amd.c: Use mu_message_reconstruct_envelope instead
of the amd-specific envelope functions.
* libmailutils/stream/message_stream.c: Likewise.
* libmailutils/tests/readmesg.at: New test case.
* libmailutils/tests/readmesg.c: New test program.
* libmailutils/tests/Makefile.am: Add new tests.
* libmailutils/tests/testsuite.at: Likewise.
* mh/tests/mhl.at: Update expected output.
* mh/tests/mhn.at: Likewise.
* testsuite/mh/mbox1/: Fix timestamps in X-Envelope-Date
* testsuite/mh/teaparty/: Likewise.

6 files changed, 256 insertions, 240 deletions

diff --git a/libmailutils/base/amd.c b/libmailutils/base/amd.c
index 2bc8d4bb9..a4e9daf49 100644
--- a/libmailutils/base/amd.c
+++ b/libmailutils/base/amd.c
@@ -105,10 +105,6 @@ static int amd_pool_open_count (struct _amd_data *amd);
 static void amd_pool_flush (struct _amd_data *amd);
 static struct _amd_message **amd_pool_lookup (struct _amd_message *mhm);
 
-static int amd_envelope_date (mu_envelope_t envelope, char *buf, size_t len,
-			      size_t *psize);
-static int amd_envelope_sender (mu_envelope_t envelope, char *buf, size_t len,
-			        size_t *psize);
 static int amd_remove_mbox (mu_mailbox_t mailbox);
 
 
@@ -634,14 +630,12 @@ _amd_attach_message (mu_mailbox_t mailbox, struct _amd_message *mhm,
   /* Set the envelope.  */
   {
     mu_envelope_t envelope = NULL;
-    status = mu_envelope_create (&envelope, msg);
+    status = mu_message_reconstruct_envelope (msg, &envelope);
     if (status != 0)
       {
 	mu_message_destroy (&msg, mhm);
 	return status;
       }
-    mu_envelope_set_sender (envelope, amd_envelope_sender, msg);
-    mu_envelope_set_date (envelope, amd_envelope_date, msg);
     mu_message_set_envelope (msg, envelope, mhm);
   }
 
@@ -2126,97 +2120,6 @@ amd_unset_attr_flags (mu_attribute_t attr, int flags)
   return 0;
 }
 
-/* Envelope */
-static int
-amd_envelope_date (mu_envelope_t envelope, char *buf, size_t len,
-		   size_t *psize)
-{
-  mu_message_t msg = mu_envelope_get_owner (envelope);
-  struct _amd_message *mhm = mu_message_get_owner (msg);
-  mu_header_t hdr = NULL;
-  const char *date;
-  char datebuf[25]; /* Buffer for the output of ctime (terminating nl being
-		       replaced by 0) */
-  int status;
-  
-  if (mhm == NULL)
-    return EINVAL;
-
-  if ((status = mu_message_get_header (msg, &hdr)) != 0)
-    return status;
-  if (mu_header_sget_value (hdr, MU_HEADER_ENV_DATE, &date)
-      && mu_header_sget_value (hdr, MU_HEADER_DELIVERY_DATE, &date))
-    return MU_ERR_NOENT;
-  else
-    {
-      time_t t;
-      int rc;
-      
-      /* Convert to ctime format */
-      rc = mu_parse_date (date, &t, NULL); /* FIXME: TZ info is lost */
-      if (rc)
-	return MU_ERR_NOENT;
-      memcpy (datebuf, ctime (&t), sizeof (datebuf) - 1);
-      datebuf[sizeof (datebuf) - 1] = 0; /* Kill the terminating newline */
-      date = datebuf;
-    }
-
-  /* Format:  "sender date" */
-  if (buf && len > 0)
-    {
-      len--; /* Leave space for the null.  */
-      strncpy (buf, date, len);
-      if (strlen (date) < len)
-	{
-	  len = strlen (buf);
-	  if (buf[len-1] != '\n')
-	    buf[len++] = '\n';
-	}
-      buf[len] = '\0';
-    }
-  else
-    len = strlen (date);
-  
-  if (psize)
-    *psize = len;
-  return 0;
-}
-
-static int
-amd_envelope_sender (mu_envelope_t envelope, char *buf, size_t len, size_t *psize)
-{
-  mu_message_t msg = mu_envelope_get_owner (envelope);
-  struct _amd_message *mhm = mu_message_get_owner (msg);
-  mu_header_t hdr = NULL;
-  char *from;
-  int status;
-
-  if (mhm == NULL)
-    return EINVAL;
-
-  if ((status = mu_message_get_header (msg, &hdr)))
-    return status;
-  if ((status = mu_header_aget_value (hdr, MU_HEADER_ENV_SENDER, &from)))
-    return status;
-
-  if (buf && len > 0)
-    {
-      int slen = strlen (from);
-
-      if (len < slen + 1)
-	slen = len - 1;
-      memcpy (buf, from, slen);
-      buf[slen] = 0;
-    }
-  else
-    len = strlen (from);
-
-  if (psize)
-    *psize = len;
-  free (from);
-  return 0;
-}
-
 
 int
 amd_remove_dir (const char *name)
diff --git a/libmailutils/stream/message_stream.c b/libmailutils/stream/message_stream.c
index 3a103678d..95ce08f5c 100644
--- a/libmailutils/stream/message_stream.c
+++ b/libmailutils/stream/message_stream.c
@@ -44,47 +44,50 @@
 
 
 static int
-_env_msg_date (mu_envelope_t envelope, char *buf, size_t len, size_t *pnwrite)
+envelope_substr (struct _mu_message_stream *str,
+		 struct mu_substring_location *loc,
+		 char *buf, size_t len, size_t *pnwrite)
 {
-  struct _mu_message_stream *str = mu_message_get_owner (mu_envelope_get_owner (envelope));
-  
-  if (!str || !str->date)
-    return EINVAL;
   if (buf)
     {
-      strncpy (buf, str->date, len);
-      buf[len-1] = 0;
+      if (len == 0)
+	return EINVAL;
+      --len;
+      if (len > loc->length)
+	len = loc->length;
+      strncpy (buf, str->envelope_string + loc->start, len);
+      buf[len] = 0;
       if (pnwrite)
 	*pnwrite = len;
     }
   else if (!pnwrite)
     return EINVAL;
   else
-    *pnwrite = strlen (str->date);
+    *pnwrite = loc->length;
   return 0;
 }
 
 static int
+_env_msg_date (mu_envelope_t envelope, char *buf, size_t len, size_t *pnwrite)
+{
+  struct _mu_message_stream *str =
+    mu_message_get_owner (mu_envelope_get_owner (envelope));
+  
+  if (!str)
+    return EINVAL;
+  return envelope_substr (str, &str->date, buf, len, pnwrite);
+}
+
+static int
 _env_msg_sender (mu_envelope_t envelope, char *buf, size_t len,
 		 size_t *pnwrite)
 {
-  struct _mu_message_stream *str = mu_message_get_owner (mu_envelope_get_owner (envelope));
+  struct _mu_message_stream *str =
+    mu_message_get_owner (mu_envelope_get_owner (envelope));
   
-  if (!str || !str->from)
-    return EINVAL;
-  if (buf)
-    {
-      strncpy (buf, str->from, len);
-      buf[len-1] = 0;
-      if (pnwrite)
-	*pnwrite = len;
-    }
-  else if (!pnwrite)
+  if (!str)
     return EINVAL;
-  else
-    *pnwrite = strlen (str->from);
-    
-  return 0;
+  return envelope_substr (str, &str->from, buf, len, pnwrite);
 }
 
 
@@ -96,13 +99,13 @@ _message_read (mu_stream_t stream, char *optr, size_t osize, size_t *nbytes)
   mu_off_t offset = s->offset + s->envelope_length;
   size_t rsize;
   
-  if (offset < s->mark_offset)
+  if (offset < s->mark.start)
     {
-      if (offset + osize >= s->mark_offset)
-	osize = s->mark_offset - offset;
+      if (offset + osize >= s->mark.start)
+	osize = s->mark.start - offset;
     }
   else
-    offset += s->mark_length;
+    offset += s->mark.length;
   /* FIXME: Seeking each time before read is awkward. The streamref
      should be modified to take care of it */
   rc = mu_stream_seek (s->transport, offset, MU_SEEK_SET, NULL);
@@ -126,25 +129,10 @@ _message_size (mu_stream_t stream, mu_off_t *psize)
   int rc = mu_stream_size (s->transport, psize);
   
   if (rc == 0)
-    *psize -= s->envelope_length + s->mark_length;
+    *psize -= s->envelope_length + s->mark.length;
   return rc;
 }
 
-static char *
-copy_trimmed_value (const char *str)
-{
-  char *p;
-  size_t len;
-  
-  str = mu_str_skip_class (str, MU_CTYPE_SPACE);
-  p = mu_str_skip_class_comp (str, MU_CTYPE_ENDLN);
-  len = p - str;
-  p = malloc (len + 1);
-  memcpy (p, str, len);
-  p[len] = 0;
-  return p;
-}
-
 static int
 is_header_start (const char *buf)
 {
@@ -168,9 +156,6 @@ static int
 _message_open (mu_stream_t stream)
 {
   struct _mu_message_stream *str = (struct _mu_message_stream*) stream;
-  char *from = NULL;
-  char *env_from = NULL;
-  char *env_date = NULL;
   int rc;
   char *buffer = NULL;
   size_t bufsize = 0;
@@ -191,37 +176,27 @@ _message_open (mu_stream_t stream)
 	  str->envelope_length = len;
 	  if (str->construct_envelope)
 	    {
-	      char *s, *p;
-	  
-	      str->envelope_string = mu_strdup (buffer);
+	      len = mu_rtrim_class (buffer, MU_CTYPE_SPACE);
+	      str->envelope_string = strdup (buffer);
 	      if (!str->envelope_string)
 		return ENOMEM;
-	      str->envelope_string[len - 1] = 0;
-	      
-	      s = str->envelope_string + 5;
-	      p = strchr (s, ' ');
-	      
-	      if (p)
-		{
-		  size_t n = p - s;
-		  env_from = mu_alloc (n + 1);
-		  if (!env_from)
-		    return ENOMEM;
-		  memcpy (env_from, s, n);
-		  env_from[n] = 0;
-		  env_date = mu_strdup (p + 1);
-		  if (!env_date)
-		    {
-		      free (env_from);
-		      return ENOMEM;
-		    }
-		}
+
+	      str->from.start = 5;
+	      str->from.length =
+		strcspn (str->envelope_string + str->from.start, " \t");
+
+	      str->date.start = str->from.start + str->from.length +
+		strspn (str->envelope_string + str->from.start +
+			str->from.length, " \t");
+	      str->date.length = len - str->date.start;
+
+	      str->construct_envelope = 0;
 	    }
 	}
       else if (mu_mh_delim (buffer))
 	{
-	  str->mark_offset = offset;
-	  str->mark_length = len - 1; /* do not count the terminating
+	  str->mark.start = offset;
+	  str->mark.length = len - 1; /* do not count the terminating
 					 newline */
 	  break;
 	}
@@ -237,23 +212,6 @@ _message_open (mu_stream_t stream)
 	      return MU_ERR_INVALID_EMAIL;
 	    }
 	  has_headers = 1;
-	  if (str->construct_envelope && (!env_from || !env_date))
-	    {
-	      if (!from && mu_c_strncasecmp (buffer, MU_HEADER_FROM,
-					     sizeof (MU_HEADER_FROM) - 1) == 0)
-	    
-		from = copy_trimmed_value (buffer + sizeof (MU_HEADER_FROM));
-	      else if (!env_from
-		       && mu_c_strncasecmp (buffer, MU_HEADER_ENV_SENDER,
-					    sizeof (MU_HEADER_ENV_SENDER) - 1) == 0)
-		env_from = copy_trimmed_value (buffer +
-					       sizeof (MU_HEADER_ENV_SENDER));
-	      else if (!env_date
-		       && mu_c_strncasecmp (buffer, MU_HEADER_ENV_DATE,
-					    sizeof (MU_HEADER_ENV_DATE) - 1) == 0)
-		env_date = copy_trimmed_value (buffer +
-					       sizeof (MU_HEADER_ENV_DATE));
-	    }
 	}
       offset += len;
     }
@@ -267,45 +225,6 @@ _message_open (mu_stream_t stream)
   if (rc)
     return rc;
   
-  if (str->construct_envelope)
-    {
-      if (!env_from)
-	{
-	  if (from)
-	    {
-	      mu_address_t addr;
-	      
-	      mu_address_create (&addr, from);
-	      if (addr)
-		{
-		  mu_address_aget_email (addr, 1, &env_from);
-		  mu_address_destroy (&addr);
-		}
-	    }
-
-	  if (!env_from)
-	    env_from = mu_get_user_email (NULL);
-	}
-      free (from);
-      
-      if (!env_date)
-	{
-	  struct tm *tm;
-	  time_t t;
-	  char date[MU_DATETIME_FROM_LENGTH+1];
-	  
-	  time (&t);
-	  tm = gmtime (&t);
-	  mu_strftime (date, sizeof (date), MU_DATETIME_FROM, tm);
-	  env_date = strdup (date);
-	  if (!env_date)
-	    return ENOMEM;
-	}
-      
-      str->from = env_from;
-      str->date = env_date;
-    }
-  
   str->body_start = body_start;
   str->body_end = body_end - 1;
   
@@ -325,8 +244,6 @@ _message_done (mu_stream_t stream)
   struct _mu_message_stream *s = (struct _mu_message_stream*) stream;
 
   free (s->envelope_string);
-  free (s->date);
-  free (s->from);
   mu_stream_destroy (&s->transport);
 }
 
@@ -436,28 +353,37 @@ mu_message_from_stream_with_envelope (mu_message_t *pmsg,
   
   mu_message_set_stream (msg, draftstream, draftstream);
 
+  sp = (struct _mu_message_stream *) draftstream;
+
   if (!env)
     {
-      /*
-       * FIXME: Currently the envelope *must* be owned by the message,
-       * otherwise _mu_message_free won't destroy it.
-       * The same holds true for attribute and body.
-       */
-      if ((rc = mu_envelope_create (&env, msg)))
+      if (sp->construct_envelope)
+	{
+	  /*
+	   * FIXME: Currently the envelope *must* be owned by the message,
+	   * otherwise _mu_message_free won't destroy it.
+	   * The same holds true for attribute and body.
+	   */
+	  rc = mu_message_reconstruct_envelope (msg, &env);
+	  sp->construct_envelope = 0;
+	}
+      else if ((rc = mu_envelope_create (&env, msg)) == 0)
+	{
+	  mu_envelope_set_date (env, _env_msg_date, msg);
+	  mu_envelope_set_sender (env, _env_msg_sender, msg);
+	}
+
+      if (rc)
 	{
 	  mu_message_destroy (&msg, draftstream);
 	  mu_stream_destroy (&draftstream);
 	  return rc;
 	}
-  
-      mu_envelope_set_date (env, _env_msg_date, msg);
-      mu_envelope_set_sender (env, _env_msg_sender, msg);
     }
   mu_message_set_envelope (msg, env, draftstream);
 
   mu_body_create (&body, msg);
-  /* FIXME: It would be cleaner to use ioctl here */
-  sp = (struct _mu_message_stream *) draftstream;
+
   rc = mu_streamref_create_abridged (&bstream, instream,
 				     sp->body_start, sp->body_end);
   if (rc)
diff --git a/libmailutils/tests/Makefile.am b/libmailutils/tests/Makefile.am
index 7b8fb7a08..bd0b781c0 100644
--- a/libmailutils/tests/Makefile.am
+++ b/libmailutils/tests/Makefile.am
@@ -56,6 +56,7 @@ noinst_PROGRAMS = \
  modmesg\
  parseopt\
  prop\
+ readmesg\
  recenv\
  scantime\
  strftime\
@@ -168,6 +169,7 @@ TESTSUITE_AT += \
  parseopt_help11.at\
  parseopt_help12.at\
  prop.at\
+ readmesg.at\
  recenv.at\
  scantime.at\
  strftime.at\
diff --git a/libmailutils/tests/readmesg.at b/libmailutils/tests/readmesg.at
new file mode 100644
index 000000000..805f70d08
--- /dev/null
+++ b/libmailutils/tests/readmesg.at
@@ -0,0 +1,72 @@
+# This file is part of GNU Mailutils. -*- Autotest -*-
+# Copyright (C) 2020 Free Software Foundation, Inc.
+#
+# GNU Mailutils is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 3, or (at
+# your option) any later version.
+#
+# GNU Mailutils is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with GNU Mailutils.  If not, see <http://www.gnu.org/licenses/>.
+
+AT_SETUP([Reading message from stream])
+AT_KEYWORDS([readmesg])
+
+AT_DATA([input.mbox],
+[[From gray@example.net Sat Aug  1 10:53:43 2020
+Return-Path: <gray@example.net>
+Organization: Mailutils-tests
+Received: from example.net (localhost [127.0.0.1])
+	by example.net with ESMTP id g6G9BZb00791
+	for <gray@example.net>; Sat,  1 Aug 2020 13:53:43 +0300
+Message-Id: <200207160911.g6G9BZb00791@example.net>
+To: Bin <bin@example.org>
+Subject: Read from stream
+Date: Sat,  1 Aug 2020 13:53:43 +0300
+From: Sergey Poznyakoff <gray@example.net>
+
+This is a sample message.
+]])
+
+AT_DATA([input.mesg],
+[[Return-Path: <gray@example.net>
+Organization: Mailutils-tests
+Received: from example.net (localhost [127.0.0.1])
+	by example.net with ESMTP id g6G9BZb00791
+	for <gray@example.net>; Sat,  1 Aug 2020 13:53:43 +0300
+Message-Id: <200207160911.g6G9BZb00791@example.net>
+To: Bin <bin@example.org>
+Subject: Read from stream
+Date: Sat,  1 Aug 2020 13:53:43 +0300
+From: Sergey Poznyakoff <gray@example.net>
+
+This is a sample message.
+]])
+
+AT_DATA([expout],
+[Sender: gray@example.net
+Date: Sat Aug  1 10:53:43 2020
+Size: 425
+Lines: 12
+Headers: 8
+Header size: 399
+Header lines: 11
+Body size: 26
+Body lines: 1
+This is a sample message.
+])
+
+AT_CHECK([readmesg input.mbox],
+[0],
+[expout])
+
+AT_CHECK([readmesg input.mesg],
+[0],
+[expout])
+
+AT_CLEANUP
\ No newline at end of file
diff --git a/libmailutils/tests/readmesg.c b/libmailutils/tests/readmesg.c
new file mode 100644
index 000000000..12349ee51
--- /dev/null
+++ b/libmailutils/tests/readmesg.c
@@ -0,0 +1,111 @@
+/* GNU Mailutils -- a suite of utilities for electronic mail
+   Copyright (C) 2020 Free Software Foundation, Inc.
+
+   GNU Mailutils is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3, or (at your option)
+   any later version.
+
+   GNU Mailutils is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with GNU Mailutils.  If not, see <http://www.gnu.org/licenses/>. */
+
+#ifdef HAVE_CONFIG_H
+# include <config.h>
+#endif
+#include <mailutils/mailutils.h>
+
+static void
+print_envelope (mu_message_t msg)
+{
+  mu_envelope_t env;
+  char const *sender, *date;
+
+  MU_ASSERT (mu_message_get_envelope (msg, &env));
+  MU_ASSERT (mu_envelope_sget_sender (env, &sender));
+  mu_printf ("Sender: %s\n", sender);  
+  MU_ASSERT (mu_envelope_sget_date (env, &date));
+  mu_printf ("Date: %s\n", date);
+}
+
+static void
+print_stats (mu_message_t msg)
+{
+  size_t n;
+      
+  MU_ASSERT (mu_message_size (msg, &n));
+  mu_printf ("Size: %lu\n", (unsigned long) n);
+  MU_ASSERT (mu_message_lines (msg, &n));
+  mu_printf ("Lines: %lu\n", (unsigned long) n);
+}
+
+static void
+print_header (mu_message_t msg)
+{
+  mu_header_t hdr;
+  size_t n;
+  
+  MU_ASSERT (mu_message_get_header (msg, &hdr));
+  MU_ASSERT (mu_header_get_field_count (hdr, &n));
+  mu_printf ("Headers: %lu\n", (unsigned long) n);
+  MU_ASSERT (mu_header_size (hdr, &n));
+  mu_printf ("Header size: %lu\n", (unsigned long) n);
+  MU_ASSERT (mu_header_lines (hdr, &n));
+  mu_printf ("Header lines: %lu\n", (unsigned long) n);
+}
+  
+static void
+print_body (mu_message_t msg)
+{
+  mu_body_t body;
+  mu_stream_t str;
+  size_t n;
+  
+  MU_ASSERT (mu_message_get_body (msg, &body));
+
+  MU_ASSERT (mu_body_size (body, &n));
+  mu_printf ("Body size: %lu\n", (unsigned long) n);
+  MU_ASSERT (mu_body_lines (body, &n));
+  mu_printf ("Body lines: %lu\n", (unsigned long) n);
+  
+  MU_ASSERT (mu_body_get_streamref (body, &str));
+  MU_ASSERT (mu_stream_copy (mu_strout, str, 0, NULL));
+  mu_stream_destroy (&str);
+}
+
+int
+main (int argc, char **argv)
+{
+  mu_stream_t instr;
+  mu_message_t msg;
+
+  mu_set_program_name (argv[0]);
+  mu_cli_simple (argc, argv,
+		 MU_CLI_OPTION_PROG_DOC, "Reads first message from FILE and "
+		 "prints its summary (envelope, stats, header and body).",
+		 MU_CLI_OPTION_PROG_ARGS, "FILE",
+		 MU_CLI_OPTION_RETURN_ARGC, &argc,
+		 MU_CLI_OPTION_RETURN_ARGV, &argv,
+		 MU_CLI_OPTION_END);
+
+  if (argc != 1)
+    {
+      mu_error ("required argument missing");
+      return 2;
+    }
+  
+  mu_set_user_email_domain ("localhost");
+  MU_ASSERT (mu_file_stream_create (&instr, argv[0], MU_STREAM_READ));  
+  MU_ASSERT (mu_stream_to_message (instr, &msg));
+
+  print_envelope (msg);
+  print_stats (msg);
+  print_header (msg);
+  print_body (msg);
+  
+  return 0;
+}
diff --git a/libmailutils/tests/testsuite.at b/libmailutils/tests/testsuite.at
index 2925bff9a..01bdfc33e 100644
--- a/libmailutils/tests/testsuite.at
+++ b/libmailutils/tests/testsuite.at
@@ -195,6 +195,8 @@ m4_include([exp.at])
 m4_include([url.at])
 m4_include([url-comp.at])
 
+m4_include([recenv.at])
+
 AT_BANNER(Mailcap)
 m4_include([ctm.at])
 m4_include([mailcap.at])
@@ -235,6 +237,8 @@ m4_include([debugspec.at])
 AT_BANNER([IMAP IO])
 m4_include([imapio.at])
 
+m4_include([readmesg.at])
+
 AT_BANNER(Message modification)
 m4_include([modmesg00.at])
 m4_include([modmesg01.at])
@@ -256,8 +260,6 @@ m4_include([msgset.at])
 m4_include([globtest.at])
 
 m4_include([linetrack.at])
-
-m4_include([recenv.at])
 
 m4_popdef([MU_TEST_GROUP])
 m4_popdef([MU_TEST_KEYWORDS])