mboxrb: Keep timestamps in normalized form

* include/mailutils/datetime.h (mu_timezone_offset): New proto.
* libmailutils/datetime/scantime.c: New conversion: %Z (timezone
abbreviation).
* libmailutils/datetime/parsedate.y (mu_timezone_offset): New function.
Returns time offset in seconds corresponding to the given timezone
abbreviation.

* include/mailutils/sys/mboxrb.h (mu_mboxrb_hdr): Remove unneeded
enum.
(mu_mboxrb_message): Remove env_date_start,
New member "date" holds the string representation of the envelope
timestamp in normalized form - ctime(3), UTC.
* libproto/mbox/mboxrb.c (scan_message_finalize): New function.
(scan_message_begin): New function.  When initializing the
message, timestamps in obsolete forms are converted to normalized
representation.
(mboxrb_rescan_unlocked): Use scan_message_finalize and
scan_message_begin.
* libproto/mbox/message.c (mboxrb_envelope_date): Get normalized
timestamp from the date member,
* libproto/mbox/tests/env.at: Reflect the above changes.

7 files changed, 216 insertions, 146 deletions

diff --git a/include/mailutils/datetime.h b/include/mailutils/datetime.h
index dd346ac29..cd8e3369c 100644
--- a/include/mailutils/datetime.h
+++ b/include/mailutils/datetime.h
@@ -82,6 +82,7 @@ int mu_parse_date_dtl (const char *p, const time_t *now,
 int mu_parse_date (const char *p, time_t *rettime, const time_t *now);
 
 int mu_utc_offset (void);
+int mu_timezone_offset (const char *buf, int *off);  
 void mu_datetime_tz_local (struct mu_timezone *tz);
 void mu_datetime_tz_utc (struct mu_timezone *tz);
 time_t mu_datetime_to_utc (struct tm *timeptr, struct mu_timezone *tz);
diff --git a/include/mailutils/sys/mboxrb.h b/include/mailutils/sys/mboxrb.h
index 0e3af71e8..22574330a 100644
--- a/include/mailutils/sys/mboxrb.h
+++ b/include/mailutils/sys/mboxrb.h
@@ -18,27 +18,21 @@
 #define _MAILUTILS_SYS_MBOX2_H
 
 # include <mailutils/types.h>
-
-enum mu_mboxrb_hdr
-  {
-    mu_mboxrb_hdr_status,
-    mu_mboxrb_hdr_x_imapbase,
-    mu_mboxrb_hdr_x_uid,
-    MU_MBOXRB_HDR_MAX
-  };
+# include <mailutils/datetime.h>
 
 struct mu_mboxrb_message
 {
   /* Offsets in the mailbox */
   mu_off_t message_start; /* Start of message */
   size_t from_length;     /* Length of the From_ line */
-  int env_sender_len;
-  int env_date_start;
+  int env_sender_len;     /* Length of the sender email in the From_ line */
   mu_off_t body_start;    /* Start of body */
   mu_off_t message_end;   /* Offset of the last byte of the message */
   /* Additional info */
   unsigned long uid;      /* IMAP-style uid.  */
 
+  char date[MU_DATETIME_FROM_LENGTH+1]; /* Envelope date in normalized form */
+  
   unsigned body_lines_scanned:1; /* True if body_lines is initialized */
   unsigned body_from_escaped:1;  /* True if body is from-escaped (valid if
 				    body_lines_scanned is true) */
diff --git a/libmailutils/datetime/parsedate.y b/libmailutils/datetime/parsedate.y
index 977713001..372cdb2eb 100644
--- a/libmailutils/datetime/parsedate.y
+++ b/libmailutils/datetime/parsedate.y
@@ -1209,6 +1209,20 @@ mu_parse_date (const char *p, time_t *rettime, const time_t *now)
   return mu_parse_date_dtl (p, now, rettime, NULL, NULL, NULL);
 }
 
+int
+mu_timezone_offset (const char *buf, int *off)
+{
+  SYMBOL const *tp;
+  
+  for (tp = tz_tab; tp->name; tp++)
+    if (mu_c_strcasecmp (buf, tp->name) == 0)
+      {
+	*off = - tp->value * 60;
+	return 0;
+      }
+  return -1;
+}
+
 #ifdef STANDALONE
 int
 main (int argc, char *argv[])
diff --git a/libmailutils/datetime/scantime.c b/libmailutils/datetime/scantime.c
index 0b118b765..8e19a803b 100644
--- a/libmailutils/datetime/scantime.c
+++ b/libmailutils/datetime/scantime.c
@@ -538,7 +538,31 @@ mu_scan_datetime (const char *input, const char *fmt,
 		  }
 	      }
 	      break;
-		      
+
+	    case 'Z':
+	      /* Time-zone in abbreviated form */
+	      {
+		char tzs[6];
+		p = mu_str_skip_class_comp (input, MU_CTYPE_SPACE);
+		n = p - input;
+		if (n > sizeof (tzs) - 1)
+		  {
+		    rc = MU_ERR_PARSE;
+		    break;
+		  }
+		memcpy (tzs, input, n);
+		tzs[n] = 0;
+		if (mu_timezone_offset (tzs, &n))
+		  {
+		    rc = MU_ERR_PARSE;
+		    break;
+		  }
+		if (tz)
+		  tz->utc_offset = n;
+		input = p;
+	      }
+	      break;
+	      
 	    case '%':
 	      if (*input == '%')
 		input++;
diff --git a/libproto/mbox/mboxrb.c b/libproto/mbox/mboxrb.c
index 412f58f61..a64a5f1ce 100644
--- a/libproto/mbox/mboxrb.c
+++ b/libproto/mbox/mboxrb.c
@@ -383,7 +383,8 @@ mboxrb_message_alloc_uid (struct mu_mboxrb_message *dmsg)
  * email address functions won't tolerate it).
  * 
  * Input: S - line read from the mailbox (with \n terminator).
- * Output: ZP - points to the time zone information in S.
+ * Output: ZP - points to the space character preceding the time zone
+ * information in S.  If there is no TZ, points to the terminating \n.
  *
  * Return value: If S is a valid From_ line, a pointer to the time
  * information in S.  Otherwise, NULL.
@@ -456,6 +457,152 @@ parse_from_line (char const *s, char **zp)
   return NULL;
 }
 
+/* Finalize current message */
+static inline int
+scan_message_finalize (struct mu_mboxrb_mailbox *dmp,
+		       struct mu_mboxrb_message *dmsg, mu_stream_t stream,
+		       size_t n, int *force_init_uids)
+{
+  int rc;
+  size_t count;
+  
+  rc = mu_stream_seek (stream, 0, MU_SEEK_CUR, &dmsg->message_end);
+  if (rc)
+    {
+      mu_debug (MU_DEBCAT_MAILBOX, MU_DEBUG_ERROR,
+		("%s:%s (%s): %s",
+		 __func__, "mu_stream_seek", dmp->name,
+		 mu_strerror (rc)));
+      return -1;
+    }
+  dmsg->message_end -= n + 1;
+  if (dmsg->uid == 0)
+    *force_init_uids = 1;
+  if (*force_init_uids)
+    mboxrb_message_alloc_uid (dmsg);
+
+  /* Every 100 mesgs update the lock, it should be every minute.  */
+  if (dmp->mailbox->locker && (dmp->mesg_count % 100) == 0)
+    mu_locker_touchlock (dmp->mailbox->locker);
+
+  count = dmp->mesg_count;
+  mboxrb_dispatch (dmp->mailbox, MU_EVT_MESSAGE_ADD, &count);
+  return 0;
+}
+
+static inline struct mu_mboxrb_message *
+scan_message_begin (struct mu_mboxrb_mailbox *dmp, mu_stream_t stream,
+		    char *buf, size_t n, char *ti, char *zn)
+{
+  int rc;
+  struct mu_mboxrb_message *dmsg;
+  
+  /* Create new message */
+  rc = mboxrb_alloc_message (dmp, &dmsg);
+  if (rc)
+    {
+      mu_debug (MU_DEBCAT_MAILBOX, MU_DEBUG_ERROR,
+		("%s:%s (%s): %s",
+		 __func__, "mboxrb_alloc_message", dmp->name,
+		 mu_strerror (rc)));
+      return NULL;
+    }
+  rc = mu_stream_seek (stream, 0, MU_SEEK_CUR, &dmsg->message_start);
+  if (rc)
+    {
+      mu_debug (MU_DEBCAT_MAILBOX, MU_DEBUG_ERROR,
+		("%s:%s (%s): %s",
+		 __func__, "mu_stream_seek", dmp->name,
+		 mu_strerror (rc)));
+      return NULL;
+    }
+  dmsg->message_start -= n;
+  dmsg->from_length = n;
+  dmsg->env_sender_len = ti - buf - 10;
+  while (dmsg->env_sender_len > 6 && buf[dmsg->env_sender_len-1] == ' ')
+    dmsg->env_sender_len--;
+  dmsg->env_sender_len -= 5;
+
+  if (zn[0] != '\n')
+    {
+      /*
+       * Ideally, From_ line should not contain time zone info.  If it does,
+       * zn points to the first space before the TZ.  The latter can be an
+       * abbreviated time zone or a numeric offset from UTC (see the comment
+       * to parse_from_line above).  Parse the date line and convert it to
+       * a normalized form (ctime(3)).
+       */
+      struct tm tm;
+      char const *fmt;
+      struct mu_timezone tz;
+      char *te;
+      time_t t;
+      
+      int numeric_zone = zn[1] == '+' || zn[1] == '-' || mu_isdigit (zn[1]);
+      if (zn[-3] == ':')
+	{
+	  if (zn[-6] == ':')
+	    {
+	      if (numeric_zone)
+		fmt = "%a %b %e %H:%M:%S %z %Y";
+	      else
+		fmt = "%a %b %e %H:%M:%S %Z %Y";
+	    }
+	  else
+	    {
+	      if (numeric_zone)
+		fmt = "%a %b %e %H:%M %z %Y";
+	      else
+		fmt = "%a %b %e %H:%M %Z %Y";
+	    }
+	}
+      else
+	{
+	  if (zn[-11] == ':')
+	    {
+	      if (numeric_zone)
+		fmt = "%a %b %e %H:%M:%S %Y %z";
+	      else
+		fmt = "%a %b %e %H:%M:%S %Y %Z";
+	    }
+	  else
+	    {
+	      if (numeric_zone)
+		fmt = "%a %b %e %H:%M %Y %z";
+	      else
+		fmt = "%a %b %e %H:%M %Y %Z";
+	    }
+	}
+	  
+      if (mu_scan_datetime (ti - 10, fmt, &tm, &tz, &te) == 0)
+	t = mu_datetime_to_utc (&tm, &tz);
+      else
+	t = time (NULL);
+      gmtime_r (&t, &tm);
+      mu_strftime (dmsg->date, sizeof (dmsg->date), MU_DATETIME_FROM, &tm);
+    }
+  else
+    {
+      /*
+       * No zone information in the timestamp.  Copy the timestamp to
+       * the date buffer.  The timestamp may or may not contain seconds.
+       * In the latter case, assume '0' seconds.
+       */
+      if (ti[6] == ':')
+	memcpy (dmsg->date, ti - 10, MU_DATETIME_FROM_LENGTH);
+      else
+	{
+	  memcpy (dmsg->date, ti - 10, 16);
+	  memcpy (dmsg->date + 16, ":00", 3);
+	  dmsg->date[19] = ' ';
+	  memcpy (dmsg->date + 20, ti + 7, 4);
+	}
+      dmsg->date[24] = 0;
+    }
+  
+  return dmsg;
+}
+
 /* Scan the mailbox starting from the given offset.
  *
  * Notes on the mailbox format:
@@ -528,7 +675,6 @@ mboxrb_rescan_unlocked (mu_mailbox_t mailbox, mu_off_t offset)
   char *zn, *ti;
   int force_init_uids = 0;
   size_t numlines = 0;
-  size_t count;
   
 # define IS_HEADER(h,b,n)			\
   ((n) > sizeof (h) - 1				\
@@ -578,32 +724,8 @@ mboxrb_rescan_unlocked (mu_mailbox_t mailbox, mu_off_t offset)
 	      rc = MU_ERR_PARSE;
 	      goto err;
 	    }
-
-	  rc = mboxrb_alloc_message (dmp, &dmsg);
-	  if (rc)
-	    {
-	      mu_debug (MU_DEBCAT_MAILBOX, MU_DEBUG_ERROR,
-			("%s:%s (%s): %s",
-			 __func__, "mboxrb_alloc_message", dmp->name,
-			 mu_strerror (rc)));
-	      goto err;
-	    }
-	  rc = mu_stream_seek (stream, 0, MU_SEEK_CUR, &dmsg->message_start);
-	  if (rc)
-	    {
-	      mu_debug (MU_DEBCAT_MAILBOX, MU_DEBUG_ERROR,
-			("%s:%s (%s): %s",
-			 __func__, "mu_stream_seek", dmp->name,
-			 mu_strerror (rc)));
-	      goto err;
-	    }
-	  dmsg->message_start -= n;
-	  dmsg->from_length = n;
-	  dmsg->env_date_start = ti - buf - 10;
-	  dmsg->env_sender_len = dmsg->env_date_start;
-	  while (dmsg->env_sender_len > 6 && buf[dmsg->env_sender_len-1] == ' ')
-	    dmsg->env_sender_len--;
-	  dmsg->env_sender_len -= 5;
+	  if ((dmsg = scan_message_begin (dmp, stream, buf, n, ti, zn)) == NULL)
+	    goto err;
 	  state = mboxrb_scan_header;
 	  break;
 
@@ -674,55 +796,10 @@ mboxrb_rescan_unlocked (mu_mailbox_t mailbox, mu_off_t offset)
 	case mboxrb_scan_empty_line:
 	  if ((ti = parse_from_line (buf, &zn)) != 0)
 	    {
-	      /* Finalize current message */
-	      rc = mu_stream_seek (stream, 0, MU_SEEK_CUR, &dmsg->message_end);
-	      if (rc)
-		{
-		  mu_debug (MU_DEBCAT_MAILBOX, MU_DEBUG_ERROR,
-			    ("%s:%s (%s): %s",
-			     __func__, "mu_stream_seek", dmp->name,
-			     mu_strerror (rc)));
-		  goto err;
-		}
-	      dmsg->message_end -= n + 1;
-	      if (dmsg->uid == 0)
-		force_init_uids = 1;
-	      if (force_init_uids)
-		mboxrb_message_alloc_uid (dmsg);
-
-              /* Every 100 mesgs update the lock, it should be every minute.  */
-              if (mailbox->locker && (dmp->mesg_count % 100) == 0)
-                mu_locker_touchlock (mailbox->locker);
-
-              count = dmp->mesg_count;
-              mboxrb_dispatch (mailbox, MU_EVT_MESSAGE_ADD, &count);
-	      
-	      /* Create new message */
-	      rc = mboxrb_alloc_message (dmp, &dmsg);
-	      if (rc)
-		{
-		  mu_debug (MU_DEBCAT_MAILBOX, MU_DEBUG_ERROR,
-			    ("%s:%s (%s): %s",
-			     __func__, "mboxrb_alloc_message", dmp->name,
-			     mu_strerror (rc)));
-		  goto err;
-		}
-	      rc = mu_stream_seek (stream, 0, MU_SEEK_CUR, &dmsg->message_start);
-	      if (rc)
-		{
-		  mu_debug (MU_DEBCAT_MAILBOX, MU_DEBUG_ERROR,
-			    ("%s:%s (%s): %s",
-			     __func__, "mu_stream_seek", dmp->name,
-			     mu_strerror (rc)));
-		  goto err;
-		}
-	      dmsg->message_start -= n;
-	      dmsg->from_length = n;
-	      dmsg->env_date_start = ti - buf - 10;
-	      dmsg->env_sender_len = dmsg->env_date_start;
-	      while (dmsg->env_sender_len > 6 && buf[dmsg->env_sender_len-1] == ' ')
-		dmsg->env_sender_len--;
-	      dmsg->env_sender_len -= 5;
+	      if (scan_message_finalize (dmp, dmsg, stream, n, &force_init_uids))
+		goto err;
+	      if ((dmsg = scan_message_begin (dmp, stream, buf, n, ti, zn)) == NULL)
+		goto err;
 	      state = mboxrb_scan_header;
 	    }
 	  else if (n == 1 && buf[0] == '\n')
@@ -736,24 +813,8 @@ mboxrb_rescan_unlocked (mu_mailbox_t mailbox, mu_off_t offset)
 
   if (dmsg)
     {
-      /* Finalize the last message */
-      rc = mu_stream_seek (stream, 0, MU_SEEK_CUR, &dmsg->message_end);
-      if (rc)
-	{
-	  mu_debug (MU_DEBCAT_MAILBOX, MU_DEBUG_ERROR,
-		    ("%s:%s (%s): %s",
-		     __func__, "mu_stream_seek", dmp->name,
-		     mu_strerror (rc)));
-	  goto err;
-	}
-      dmsg->message_end--;
-      if (dmsg->uid == 0)
-	force_init_uids = 1;
-      if (force_init_uids)
-	mboxrb_message_alloc_uid (dmsg);
-
-      count = dmp->mesg_count;
-      mboxrb_dispatch (mailbox, MU_EVT_MESSAGE_ADD, &count);
+      if (scan_message_finalize (dmp, dmsg, stream, 0, &force_init_uids))
+	goto err;
     }
   
  err:
diff --git a/libproto/mbox/message.c b/libproto/mbox/message.c
index d33694222..a0e8dd076 100644
--- a/libproto/mbox/message.c
+++ b/libproto/mbox/message.c
@@ -29,6 +29,7 @@
 #include <mailutils/attribute.h>
 #include <mailutils/envelope.h>
 #include <mailutils/io.h>
+#include <mailutils/util.h>
 
 void
 mu_mboxrb_message_free (struct mu_mboxrb_message *dmsg)
@@ -211,45 +212,20 @@ static int
 mboxrb_envelope_date (mu_envelope_t env, char *buf, size_t len,
 		      size_t *pnwrite)
 {
-  int rc = 0;
-  size_t date_len;  
   mu_message_t msg = mu_envelope_get_owner (env);
   struct mu_mboxrb_message *dmsg = mu_message_get_owner (msg);
-  if (!dmsg)
-    return EINVAL;
 
-  date_len = dmsg->from_length - dmsg->env_date_start - 1;
-  
   if (!buf || len <= 1)
     {
-      len = date_len;
+      len = MU_DATETIME_FROM_LENGTH;
     }
   else
     {
-      mu_stream_t stream;
-	
-      rc = mu_streamref_create_abridged (&stream,
-					 dmsg->mbox->mailbox->stream,
-					 dmsg->message_start + dmsg->env_date_start,
-					 dmsg->message_start + dmsg->from_length);
-      if (rc)
-	return rc;
-      len--;
-      if (len > date_len)
-	len = date_len;
-      rc = mu_stream_read (stream, buf, len, &len);
-      if (rc == 0)
-	{
-	  buf[len] = 0;
-	}
-      mu_stream_destroy (&stream);
-    }
-  if (rc == 0)
-    {
-      if (pnwrite)
-	*pnwrite = len;
+      len = mu_cpystr (buf, dmsg->date, len);
     }
-  return rc;
+  if (pnwrite)
+    *pnwrite = len;
+  return 0;
 }	
 
 static int
@@ -564,7 +540,7 @@ env_to_stream (struct mu_mboxrb_message const *dmsg,
       mu_stream_seek (dst, 0, MU_SEEK_CUR, &off);
       ref->from_length = off - ref->message_start;
       ref->env_sender_len = strlen (sender);
-      ref->env_date_start = strlen (sender) + 6;
+      strncpy (ref->date, date, sizeof (ref->date));
     }
   
   return rc;
diff --git a/libproto/mbox/tests/env.at b/libproto/mbox/tests/env.at
index 689cb4ba4..203f10969 100644
--- a/libproto/mbox/tests/env.at
+++ b/libproto/mbox/tests/env.at
@@ -92,7 +92,7 @@ I believe I can guess that
 AT_CHECK([mbop -r -m inbox 1 \; env_date \; env_sender],
 [0],
 [1 current message
-1 env_date: Wed Dec  2 05:53 1992
+1 env_date: Wed Dec  2 05:53:00 1992
 1 env_sender: hare@wonder.land
 ])
 
@@ -106,56 +106,56 @@ AT_CHECK([mbop -r -m inbox 2 \; env_date \; env_sender],
 AT_CHECK([mbop -r -m inbox 3 \; env_date \; env_sender],
 [0],
 [3 current message
-3 env_date: Wed Dec  2 05:53 PST 1992
+3 env_date: Wed Dec  2 13:53:00 1992
 3 env_sender: hare@wonder.land
 ])
 
 AT_CHECK([mbop -r -m inbox 4 \; env_date \; env_sender],
 [0],
 [4 current message
-4 env_date: Wed Dec  2 05:53:22 PST 1992
+4 env_date: Wed Dec  2 13:53:22 1992
 4 env_sender: alice@wonder.land
 ])
 
 AT_CHECK([mbop -r -m inbox 5 \; env_date \; env_sender],
 [0],
 [5 current message
-5 env_date: Wed Dec  2 05:53 -0700 1992
+5 env_date: Wed Dec  2 12:53:00 1992
 5 env_sender: hare@wonder.land
 ])
 
 AT_CHECK([mbop -r -m inbox 6 \; env_date \; env_sender],
 [0],
 [6 current message
-6 env_date: Wed Dec  2 05:53:22 -0700 1992
+6 env_date: Wed Dec  2 12:53:22 1992
 6 env_sender: alice@wonder.land
 ])
 
 AT_CHECK([mbop -r -m inbox 7 \; env_date \; env_sender],
 [0],
 [7 current message
-7 env_date: Wed Dec  2 05:53 1992 PST
+7 env_date: Wed Dec  2 13:53:00 1992
 7 env_sender: hare@wonder.land
 ])
 
 AT_CHECK([mbop -r -m inbox 8 \; env_date \; env_sender],
 [0],
 [8 current message
-8 env_date: Wed Dec  2 05:53:22 1992 PST
+8 env_date: Wed Dec  2 13:53:22 1992
 8 env_sender: alice@wonder.land
 ])
 
 AT_CHECK([mbop -r -m inbox 9 \; env_date \; env_sender],
 [0],
 [9 current message
-9 env_date: Wed Dec  2 05:53 1992 -0700
+9 env_date: Wed Dec  2 12:53:00 1992
 9 env_sender: hare@wonder.land
 ])
 
 AT_CHECK([mbop -r -m inbox 10 \; env_date \; env_sender],
 [0],
 [10 current message
-10 env_date: Wed Dec  2 05:53:22 1992 -0700
+10 env_date: Wed Dec  2 12:53:22 1992
 10 env_sender: alice@wonder.land
 ])